caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1bf05fbc987dddebe94de7b33d810d8221eda1a5
clang-p2996/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
alex-t 1a33294652 [AMDGPU] Filtering out the inactive lanes bits when lowering copy to SCC 
Normally, given that the DA results are kept consistent over the selection DAG, uniform comparisons get selected to S_CMP_* but divergent to V_CMP_*.  Sometimes, for the sake of efficiency,  SSA subgraphs may be converted to VALU to avoid repeatedly copying data back and forth. Hence we have to be able to sustain the correctness passing the i1 from VALU to SALU context and vice versa.

VALU operations only process the active lanes of the VGPR and ignore inactive ones.
Active lanes correspond to 1 bit in the EXEC mask register.
SALU represents i1 as just one bit but VALU as 64bits: 0/1 and 0/(0xffffffffffffffff & EXEC) respectively.
SALU uses one-bit conditional flag SCC but VALU - VCC that is a pair of 32-bit SGPRs

To expose SCC to the VALU context we need to convert the one-bit boolean value to the appropriate 64bit.
To return back to the SALU context we need to do the opposite.

To correctly convert 64bit VALU boolean to either 0 or 1 we need to filter out the bits corresponding to the inactive lanes.

Reviewed By: piotr

Differential Revision: https://reviews.llvm.org/D109900
2021-09-21 21:19:31 +03:00

919 lines
32 KiB
C++
Raw Blame History

//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Copies from VGPR to SGPR registers are illegal and the register coalescer
/// will sometimes generate these illegal copies in situations like this:
///
/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
///
/// BB0:
/// %0 <sgpr> = SCALAR_INST
/// %1 <vsrc> = COPY %0 <sgpr>
/// ...
/// BRANCH %cond BB1, BB2
/// BB1:
/// %2 <vgpr> = VECTOR_INST
/// %3 <vsrc> = COPY %2 <vgpr>
/// BB2:
/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
///
///
/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
/// code will look like this:
///
/// BB0:
/// %0 <sgpr> = SCALAR_INST
/// ...
/// BRANCH %cond BB1, BB2
/// BB1:
/// %2 <vgpr> = VECTOR_INST
/// %3 <vsrc> = COPY %2 <vgpr>
/// BB2:
/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
///
/// Now that the result of the PHI instruction is an SGPR, the register
/// allocator is now forced to constrain the register class of %3 to
/// <sgpr> so we end up with final code like this:
///
/// BB0:
/// %0 <sgpr> = SCALAR_INST
/// ...
/// BRANCH %cond BB1, BB2
/// BB1:
/// %2 <vgpr> = VECTOR_INST
/// %3 <sgpr> = COPY %2 <vgpr>
/// BB2:
/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
///
/// Now this code contains an illegal copy from a VGPR to an SGPR.
///
/// In order to avoid this problem, this pass searches for PHI instructions
/// which define a <vsrc> register and constrains its definition class to
/// <vgpr> if the user of the PHI's definition register is a vector instruction.
/// If the PHI's definition class is constrained to <vgpr> then the coalescer
/// will be unable to perform the COPY removal from the above example which
/// ultimately led to the creation of an illegal COPY.
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/InitializePasses.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
#define DEBUG_TYPE "si-fix-sgpr-copies"
static cl::opt<bool> EnableM0Merge(
"amdgpu-enable-merge-m0",
cl::desc("Merge and hoist M0 initializations"),
cl::init(true));
namespace {
class SIFixSGPRCopies : public MachineFunctionPass {
MachineDominatorTree *MDT;
public:
static char ID;
MachineRegisterInfo *MRI;
const SIRegisterInfo *TRI;
const SIInstrInfo *TII;
SIFixSGPRCopies() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override;
MachineBasicBlock *processPHINode(MachineInstr &MI);
StringRef getPassName() const override { return "SI Fix SGPR copies"; }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineDominatorTree>();
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
} // end anonymous namespace
INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
"SI Fix SGPR copies", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_END(SIFixSGPRCopies, DEBUG_TYPE,
"SI Fix SGPR copies", false, false)
char SIFixSGPRCopies::ID = 0;
char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
FunctionPass *llvm::createSIFixSGPRCopiesPass() {
return new SIFixSGPRCopies();
}
static bool hasVectorOperands(const MachineInstr &MI,
const SIRegisterInfo *TRI) {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual())
continue;
if (TRI->hasVectorRegisters(MRI.getRegClass(MI.getOperand(i).getReg())))
return true;
}
return false;
}
static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
getCopyRegClasses(const MachineInstr &Copy,
const SIRegisterInfo &TRI,
const MachineRegisterInfo &MRI) {
Register DstReg = Copy.getOperand(0).getReg();
Register SrcReg = Copy.getOperand(1).getReg();
const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
? MRI.getRegClass(SrcReg)
: TRI.getPhysRegClass(SrcReg);
// We don't really care about the subregister here.
// SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
const TargetRegisterClass *DstRC = DstReg.isVirtual()
? MRI.getRegClass(DstReg)
: TRI.getPhysRegClass(DstReg);
return std::make_pair(SrcRC, DstRC);
}
static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
const TargetRegisterClass *DstRC,
const SIRegisterInfo &TRI) {
return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
TRI.hasVectorRegisters(SrcRC);
}
static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
const TargetRegisterClass *DstRC,
const SIRegisterInfo &TRI) {
return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
TRI.hasVectorRegisters(DstRC);
}
static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
const SIRegisterInfo *TRI,
const SIInstrInfo *TII) {
MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
auto &Src = MI.getOperand(1);
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = Src.getReg();
if (!SrcReg.isVirtual() || !DstReg.isVirtual())
return false;
for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
const auto *UseMI = MO.getParent();
if (UseMI == &MI)
continue;
if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
return false;
unsigned OpIdx = UseMI->getOperandNo(&MO);
if (OpIdx >= UseMI->getDesc().getNumOperands() ||
!TII->isOperandLegal(*UseMI, OpIdx, &Src))
return false;
}
// Change VGPR to SGPR destination.
MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
return true;
}
// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
//
// SGPRx = ...
// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
// VGPRz = COPY SGPRy
//
// ==>
//
// VGPRx = COPY SGPRx
// VGPRz = REG_SEQUENCE VGPRx, sub0
//
// This exposes immediate folding opportunities when materializing 64-bit
// immediates.
static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
const SIRegisterInfo *TRI,
const SIInstrInfo *TII,
MachineRegisterInfo &MRI) {
assert(MI.isRegSequence());
Register DstReg = MI.getOperand(0).getReg();
if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
return false;
if (!MRI.hasOneUse(DstReg))
return false;
MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
if (!CopyUse.isCopy())
return false;
// It is illegal to have vreg inputs to a physreg defining reg_sequence.
if (CopyUse.getOperand(0).getReg().isPhysical())
return false;
const TargetRegisterClass *SrcRC, *DstRC;
std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
return false;
if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
return true;
// TODO: Could have multiple extracts?
unsigned SubReg = CopyUse.getOperand(1).getSubReg();
if (SubReg != AMDGPU::NoSubRegister)
return false;
MRI.setRegClass(DstReg, DstRC);
// SGPRx = ...
// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
// VGPRz = COPY SGPRy
// =>
// VGPRx = COPY SGPRx
// VGPRz = REG_SEQUENCE VGPRx, sub0
MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
bool IsAGPR = TRI->hasAGPRs(DstRC);
for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
Register SrcReg = MI.getOperand(I).getReg();
unsigned SrcSubReg = MI.getOperand(I).getSubReg();
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
assert(TRI->isSGPRClass(SrcRC) &&
"Expected SGPR REG_SEQUENCE to only have SGPR inputs");
SrcRC = TRI->getSubRegClass(SrcRC, SrcSubReg);
const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
TmpReg)
.add(MI.getOperand(I));
if (IsAGPR) {
const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
TmpAReg)
.addReg(TmpReg, RegState::Kill);
TmpReg = TmpAReg;
}
MI.getOperand(I).setReg(TmpReg);
}
CopyUse.eraseFromParent();
return true;
}
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
const MachineInstr *MoveImm,
const SIInstrInfo *TII,
unsigned &SMovOp,
int64_t &Imm) {
if (Copy->getOpcode() != AMDGPU::COPY)
return false;
if (!MoveImm->isMoveImmediate())
return false;
const MachineOperand *ImmOp =
TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
if (!ImmOp->isImm())
return false;
// FIXME: Handle copies with sub-regs.
if (Copy->getOperand(0).getSubReg())
return false;
switch (MoveImm->getOpcode()) {
default:
return false;
case AMDGPU::V_MOV_B32_e32:
SMovOp = AMDGPU::S_MOV_B32;
break;
case AMDGPU::V_MOV_B64_PSEUDO:
SMovOp = AMDGPU::S_MOV_B64;
break;
}
Imm = ImmOp->getImm();
return true;
}
template <class UnaryPredicate>
bool searchPredecessors(const MachineBasicBlock *MBB,
const MachineBasicBlock *CutOff,
UnaryPredicate Predicate) {
if (MBB == CutOff)
return false;
DenseSet<const MachineBasicBlock *> Visited;
SmallVector<MachineBasicBlock *, 4> Worklist(MBB->predecessors());
while (!Worklist.empty()) {
MachineBasicBlock *MBB = Worklist.pop_back_val();
if (!Visited.insert(MBB).second)
continue;
if (MBB == CutOff)
continue;
if (Predicate(MBB))
return true;
Worklist.append(MBB->pred_begin(), MBB->pred_end());
}
return false;
}
// Checks if there is potential path From instruction To instruction.
// If CutOff is specified and it sits in between of that path we ignore
// a higher portion of the path and report it is not reachable.
static bool isReachable(const MachineInstr *From,
const MachineInstr *To,
const MachineBasicBlock *CutOff,
MachineDominatorTree &MDT) {
if (MDT.dominates(From, To))
return true;
const MachineBasicBlock *MBBFrom = From->getParent();
const MachineBasicBlock *MBBTo = To->getParent();
// Do predecessor search.
// We should almost never get here since we do not usually produce M0 stores
// other than -1.
return searchPredecessors(MBBTo, CutOff, [MBBFrom]
(const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
}
// Return the first non-prologue instruction in the block.
static MachineBasicBlock::iterator
getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
++I;
return I;
}
// Hoist and merge identical SGPR initializations into a common predecessor.
// This is intended to combine M0 initializations, but can work with any
// SGPR. A VGPR cannot be processed since we cannot guarantee vector
// executioon.
static bool hoistAndMergeSGPRInits(unsigned Reg,
const MachineRegisterInfo &MRI,
const TargetRegisterInfo *TRI,
MachineDominatorTree &MDT,
const TargetInstrInfo *TII) {
// List of inits by immediate value.
using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
InitListMap Inits;
// List of clobbering instructions.
SmallVector<MachineInstr*, 8> Clobbers;
// List of instructions marked for deletion.
SmallSet<MachineInstr*, 8> MergedInstrs;
bool Changed = false;
for (auto &MI : MRI.def_instructions(Reg)) {
MachineOperand *Imm = nullptr;
for (auto &MO : MI.operands()) {
if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
(!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
Imm = nullptr;
break;
} else if (MO.isImm())
Imm = &MO;
}
if (Imm)
Inits[Imm->getImm()].push_front(&MI);
else
Clobbers.push_back(&MI);
}
for (auto &Init : Inits) {
auto &Defs = Init.second;
for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
MachineInstr *MI1 = *I1;
for (auto I2 = std::next(I1); I2 != E; ) {
MachineInstr *MI2 = *I2;
// Check any possible interference
auto interferes = [&](MachineBasicBlock::iterator From,
MachineBasicBlock::iterator To) -> bool {
assert(MDT.dominates(&*To, &*From));
auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
const MachineBasicBlock *MBBFrom = From->getParent();
const MachineBasicBlock *MBBTo = To->getParent();
bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
if (!MayClobberFrom && !MayClobberTo)
return false;
if ((MayClobberFrom && !MayClobberTo) ||
(!MayClobberFrom && MayClobberTo))
return true;
// Both can clobber, this is not an interference only if both are
// dominated by Clobber and belong to the same block or if Clobber
// properly dominates To, given that To >> From, so it dominates
// both and located in a common dominator.
return !((MBBFrom == MBBTo &&
MDT.dominates(Clobber, &*From) &&
MDT.dominates(Clobber, &*To)) ||
MDT.properlyDominates(Clobber->getParent(), MBBTo));
};
return (llvm::any_of(Clobbers, interferes)) ||
(llvm::any_of(Inits, [&](InitListMap::value_type &C) {
return C.first != Init.first &&
llvm::any_of(C.second, interferes);
}));
};
if (MDT.dominates(MI1, MI2)) {
if (!interferes(MI2, MI1)) {
LLVM_DEBUG(dbgs()
<< "Erasing from "
<< printMBBReference(*MI2->getParent()) << " " << *MI2);
MergedInstrs.insert(MI2);
Changed = true;
++I2;
continue;
}
} else if (MDT.dominates(MI2, MI1)) {
if (!interferes(MI1, MI2)) {
LLVM_DEBUG(dbgs()
<< "Erasing from "
<< printMBBReference(*MI1->getParent()) << " " << *MI1);
MergedInstrs.insert(MI1);
Changed = true;
++I1;
break;
}
} else {
auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
MI2->getParent());
if (!MBB) {
++I2;
continue;
}
MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
if (!interferes(MI1, I) && !interferes(MI2, I)) {
LLVM_DEBUG(dbgs()
<< "Erasing from "
<< printMBBReference(*MI1->getParent()) << " " << *MI1
<< "and moving from "
<< printMBBReference(*MI2->getParent()) << " to "
<< printMBBReference(*I->getParent()) << " " << *MI2);
I->getParent()->splice(I, MI2->getParent(), MI2);
MergedInstrs.insert(MI1);
Changed = true;
++I1;
break;
}
}
++I2;
}
++I1;
}
}
// Remove initializations that were merged into another.
for (auto &Init : Inits) {
auto &Defs = Init.second;
auto I = Defs.begin();
while (I != Defs.end()) {
if (MergedInstrs.count(*I)) {
(*I)->eraseFromParent();
I = Defs.erase(I);
} else
++I;
}
}
// Try to schedule SGPR initializations as early as possible in the MBB.
for (auto &Init : Inits) {
auto &Defs = Init.second;
for (auto MI : Defs) {
auto MBB = MI->getParent();
MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
MachineBasicBlock::reverse_iterator B(BoundaryMI);
// Check if B should actually be a boundary. If not set the previous
// instruction as the boundary instead.
if (!TII->isBasicBlockPrologue(*B))
B++;
auto R = std::next(MI->getReverseIterator());
const unsigned Threshold = 50;
// Search until B or Threshold for a place to insert the initialization.
for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
TII->isSchedulingBoundary(*R, MBB, *MBB->getParent()))
break;
// Move to directly after R.
if (&*--R != MI)
MBB->splice(*R, MBB, MI);
}
}
if (Changed)
MRI.clearKillFlags(Reg);
return Changed;
}
bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
// Only need to run this in SelectionDAG path.
if (MF.getProperties().hasProperty(
MachineFunctionProperties::Property::Selected))
return false;
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
TII = ST.getInstrInfo();
MDT = &getAnalysis<MachineDominatorTree>();
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
BI != BE; ++BI) {
MachineBasicBlock *MBB = &*BI;
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
++I) {
MachineInstr &MI = *I;
switch (MI.getOpcode()) {
default:
continue;
case AMDGPU::COPY:
case AMDGPU::WQM:
case AMDGPU::STRICT_WQM:
case AMDGPU::SOFT_WQM:
case AMDGPU::STRICT_WWM: {
Register DstReg = MI.getOperand(0).getReg();
const TargetRegisterClass *SrcRC, *DstRC;
std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
if (MI.isCopy()) {
Register SrcReg = MI.getOperand(1).getReg();
if (SrcReg == AMDGPU::SCC) {
Register SCCCopy = MRI->createVirtualRegister(
TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID));
I = BuildMI(*MI.getParent(),
std::next(MachineBasicBlock::iterator(MI)),
MI.getDebugLoc(),
TII->get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
: AMDGPU::S_CSELECT_B64),
SCCCopy)
.addImm(-1)
.addImm(0);
I = BuildMI(*MI.getParent(), std::next(I), I->getDebugLoc(),
TII->get(AMDGPU::COPY), DstReg)
.addReg(SCCCopy);
MI.eraseFromParent();
continue;
} else if (DstReg == AMDGPU::SCC) {
unsigned Opcode =
ST.isWave64() ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
Register Exec = ST.isWave64() ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
I = BuildMI(*MI.getParent(),
std::next(MachineBasicBlock::iterator(MI)),
MI.getDebugLoc(), TII->get(Opcode))
.addReg(Tmp, getDefRegState(true))
.addReg(SrcReg)
.addReg(Exec);
MI.eraseFromParent();
continue;
}
}
if (!DstReg.isVirtual()) {
// If the destination register is a physical register there isn't
// really much we can do to fix this.
// Some special instructions use M0 as an input. Some even only use
// the first lane. Insert a readfirstlane and hope for the best.
if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(SrcRC)) {
Register TmpReg
= MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
BuildMI(*MBB, MI, MI.getDebugLoc(),
TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
.add(MI.getOperand(1));
MI.getOperand(1).setReg(TmpReg);
}
continue;
}
if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) {
Register SrcReg = MI.getOperand(1).getReg();
if (!SrcReg.isVirtual()) {
MachineBasicBlock *NewBB = TII->moveToVALU(MI, MDT);
if (NewBB && NewBB != MBB) {
MBB = NewBB;
E = MBB->end();
BI = MachineFunction::iterator(MBB);
BE = MF.end();
}
assert((!NewBB || NewBB == I->getParent()) &&
"moveToVALU did not return the right basic block");
break;
}
MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
unsigned SMovOp;
int64_t Imm;
// If we are just copying an immediate, we can replace the copy with
// s_mov_b32.
if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
MI.getOperand(1).ChangeToImmediate(Imm);
MI.addImplicitDefUseOperands(MF);
MI.setDesc(TII->get(SMovOp));
break;
}
MachineBasicBlock *NewBB = TII->moveToVALU(MI, MDT);
if (NewBB && NewBB != MBB) {
MBB = NewBB;
E = MBB->end();
BI = MachineFunction::iterator(MBB);
BE = MF.end();
}
assert((!NewBB || NewBB == I->getParent()) &&
"moveToVALU did not return the right basic block");
} else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
tryChangeVGPRtoSGPRinCopy(MI, TRI, TII);
}
break;
}
case AMDGPU::PHI: {
MachineBasicBlock *NewBB = processPHINode(MI);
if (NewBB && NewBB != MBB) {
MBB = NewBB;
E = MBB->end();
BI = MachineFunction::iterator(MBB);
BE = MF.end();
}
assert((!NewBB || NewBB == I->getParent()) &&
"moveToVALU did not return the right basic block");
break;
}
case AMDGPU::REG_SEQUENCE: {
if (TRI->hasVectorRegisters(TII->getOpRegClass(MI, 0)) ||
!hasVectorOperands(MI, TRI)) {
foldVGPRCopyIntoRegSequence(MI, TRI, TII, *MRI);
continue;
}
LLVM_DEBUG(dbgs() << "Fixing REG_SEQUENCE: " << MI);
MachineBasicBlock *NewBB = TII->moveToVALU(MI, MDT);
if (NewBB && NewBB != MBB) {
MBB = NewBB;
E = MBB->end();
BI = MachineFunction::iterator(MBB);
BE = MF.end();
}
assert((!NewBB || NewBB == I->getParent()) &&
"moveToVALU did not return the right basic block");
break;
}
case AMDGPU::INSERT_SUBREG: {
const TargetRegisterClass *DstRC, *Src0RC, *Src1RC;
DstRC = MRI->getRegClass(MI.getOperand(0).getReg());
Src0RC = MRI->getRegClass(MI.getOperand(1).getReg());
Src1RC = MRI->getRegClass(MI.getOperand(2).getReg());
if (TRI->isSGPRClass(DstRC) &&
(TRI->hasVectorRegisters(Src0RC) ||
TRI->hasVectorRegisters(Src1RC))) {
LLVM_DEBUG(dbgs() << " Fixing INSERT_SUBREG: " << MI);
MachineBasicBlock *NewBB = TII->moveToVALU(MI, MDT);
if (NewBB && NewBB != MBB) {
MBB = NewBB;
E = MBB->end();
BI = MachineFunction::iterator(MBB);
BE = MF.end();
}
assert((!NewBB || NewBB == I->getParent()) &&
"moveToVALU did not return the right basic block");
}
break;
}
case AMDGPU::V_WRITELANE_B32: {
// Some architectures allow more than one constant bus access without
// SGPR restriction
if (ST.getConstantBusLimit(MI.getOpcode()) != 1)
break;
// Writelane is special in that it can use SGPR and M0 (which would
// normally count as using the constant bus twice - but in this case it
// is allowed since the lane selector doesn't count as a use of the
// constant bus). However, it is still required to abide by the 1 SGPR
// rule. Apply a fix here as we might have multiple SGPRs after
// legalizing VGPRs to SGPRs
int Src0Idx =
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
int Src1Idx =
AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
MachineOperand &Src0 = MI.getOperand(Src0Idx);
MachineOperand &Src1 = MI.getOperand(Src1Idx);
// Check to see if the instruction violates the 1 SGPR rule
if ((Src0.isReg() && TRI->isSGPRReg(*MRI, Src0.getReg()) &&
Src0.getReg() != AMDGPU::M0) &&
(Src1.isReg() && TRI->isSGPRReg(*MRI, Src1.getReg()) &&
Src1.getReg() != AMDGPU::M0)) {
// Check for trivially easy constant prop into one of the operands
// If this is the case then perform the operation now to resolve SGPR
// issue. If we don't do that here we will always insert a mov to m0
// that can't be resolved in later operand folding pass
bool Resolved = false;
for (MachineOperand *MO : {&Src0, &Src1}) {
if (MO->getReg().isVirtual()) {
MachineInstr *DefMI = MRI->getVRegDef(MO->getReg());
if (DefMI && TII->isFoldableCopy(*DefMI)) {
const MachineOperand &Def = DefMI->getOperand(0);
if (Def.isReg() &&
MO->getReg() == Def.getReg() &&
MO->getSubReg() == Def.getSubReg()) {
const MachineOperand &Copied = DefMI->getOperand(1);
if (Copied.isImm() &&
TII->isInlineConstant(APInt(64, Copied.getImm(), true))) {
MO->ChangeToImmediate(Copied.getImm());
Resolved = true;
break;
}
}
}
}
}
if (!Resolved) {
// Haven't managed to resolve by replacing an SGPR with an immediate
// Move src1 to be in M0
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
TII->get(AMDGPU::COPY), AMDGPU::M0)
.add(Src1);
Src1.ChangeToRegister(AMDGPU::M0, false);
}
}
break;
}
}
}
}
if (MF.getTarget().getOptLevel() > CodeGenOpt::None && EnableM0Merge)
hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
return true;
}
MachineBasicBlock *SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
unsigned numVGPRUses = 0;
bool AllAGPRUses = true;
SetVector<const MachineInstr *> worklist;
SmallSet<const MachineInstr *, 4> Visited;
SetVector<MachineInstr *> PHIOperands;
MachineBasicBlock *CreatedBB = nullptr;
worklist.insert(&MI);
Visited.insert(&MI);
while (!worklist.empty()) {
const MachineInstr *Instr = worklist.pop_back_val();
Register Reg = Instr->getOperand(0).getReg();
for (const auto &Use : MRI->use_operands(Reg)) {
const MachineInstr *UseMI = Use.getParent();
AllAGPRUses &= (UseMI->isCopy() &&
TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg())) ||
TRI->isAGPR(*MRI, Use.getReg());
if (UseMI->isCopy() || UseMI->isRegSequence()) {
if (UseMI->isCopy() &&
UseMI->getOperand(0).getReg().isPhysical() &&
!TRI->isSGPRReg(*MRI, UseMI->getOperand(0).getReg())) {
numVGPRUses++;
}
if (Visited.insert(UseMI).second)
worklist.insert(UseMI);
continue;
}
if (UseMI->isPHI()) {
const TargetRegisterClass *UseRC = MRI->getRegClass(Use.getReg());
if (!TRI->isSGPRReg(*MRI, Use.getReg()) &&
UseRC != &AMDGPU::VReg_1RegClass)
numVGPRUses++;
continue;
}
const TargetRegisterClass *OpRC =
TII->getOpRegClass(*UseMI, UseMI->getOperandNo(&Use));
if (!TRI->isSGPRClass(OpRC) && OpRC != &AMDGPU::VS_32RegClass &&
OpRC != &AMDGPU::VS_64RegClass) {
numVGPRUses++;
}
}
}
Register PHIRes = MI.getOperand(0).getReg();
const TargetRegisterClass *RC0 = MRI->getRegClass(PHIRes);
if (AllAGPRUses && numVGPRUses && !TRI->hasAGPRs(RC0)) {
LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0));
for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
MachineInstr *DefMI = MRI->getVRegDef(MI.getOperand(I).getReg());
if (DefMI && DefMI->isPHI())
PHIOperands.insert(DefMI);
}
}
bool hasVGPRInput = false;
for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
Register InputReg = MI.getOperand(i).getReg();
MachineInstr *Def = MRI->getVRegDef(InputReg);
if (TRI->isVectorRegister(*MRI, InputReg)) {
if (Def->isCopy()) {
Register SrcReg = Def->getOperand(1).getReg();
const TargetRegisterClass *RC =
TRI->getRegClassForReg(*MRI, SrcReg);
if (TRI->isSGPRClass(RC))
continue;
}
hasVGPRInput = true;
break;
}
else if (Def->isCopy() &&
TRI->isVectorRegister(*MRI, Def->getOperand(1).getReg())) {
Register SrcReg = Def->getOperand(1).getReg();
MachineInstr *SrcDef = MRI->getVRegDef(SrcReg);
unsigned SMovOp;
int64_t Imm;
if (!isSafeToFoldImmIntoCopy(Def, SrcDef, TII, SMovOp, Imm)) {
hasVGPRInput = true;
break;
} else {
// Formally, if we did not do this right away
// it would be done on the next iteration of the
// runOnMachineFunction main loop. But why not if we can?
MachineFunction *MF = MI.getParent()->getParent();
Def->getOperand(1).ChangeToImmediate(Imm);
Def->addImplicitDefUseOperands(*MF);
Def->setDesc(TII->get(SMovOp));
}
}
}
if ((!TRI->isVectorRegister(*MRI, PHIRes) &&
RC0 != &AMDGPU::VReg_1RegClass) &&
(hasVGPRInput || numVGPRUses > 1)) {
LLVM_DEBUG(dbgs() << "Fixing PHI: " << MI);
CreatedBB = TII->moveToVALU(MI);
}
else {
LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
TII->legalizeOperands(MI, MDT);
}
// Propagate register class back to PHI operands which are PHI themselves.
while (!PHIOperands.empty()) {
processPHINode(*PHIOperands.pop_back_val());
}
return CreatedBB;
}
Reference in New Issue View Git Blame Copy Permalink