75e528fdd9
buffer_load instructions that use TFE also need to zero initialize return values similar to how the image instructions currently work. Add support for this with standard zero init of all results + zero init of just TFE flag when enable-prt-strict-null subtarget feature is disabled.
3408 lines
150 KiB
TableGen
3408 lines
150 KiB
TableGen
//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">;
|
|
def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">;
|
|
|
|
def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
|
|
def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
|
|
|
|
def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">;
|
|
|
|
def BUFAddrKind {
|
|
int Offset = 0;
|
|
int OffEn = 1;
|
|
int IdxEn = 2;
|
|
int BothEn = 3;
|
|
int Addr64 = 4;
|
|
}
|
|
|
|
class getAddrName<int addrKind> {
|
|
string ret =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset), "offset",
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn), "offen",
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen",
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
|
|
"")))));
|
|
}
|
|
|
|
class MUBUFAddr64Table <bit is_addr64, string Name> {
|
|
bit IsAddr64 = is_addr64;
|
|
string OpName = Name;
|
|
}
|
|
|
|
class MTBUFAddr64Table <bit is_addr64, string Name> {
|
|
bit IsAddr64 = is_addr64;
|
|
string OpName = Name;
|
|
}
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// BUF class (base class for MTBUF and MUBUF pseudos)
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class BUF_Pseudo <string opName, dag outs, dag ins,
|
|
string asmOps, list<dag> pattern=[]> :
|
|
InstSI<outs, ins, "", pattern>,
|
|
SIMCInstr<opName, SIEncodingFamily.NONE> {
|
|
|
|
let isPseudo = 1;
|
|
let isCodeGenOnly = 1;
|
|
let Size = 8;
|
|
let UseNamedOperandTable = 1;
|
|
|
|
string Mnemonic = opName;
|
|
string AsmOperands = asmOps;
|
|
|
|
Instruction Opcode = !cast<Instruction>(NAME);
|
|
|
|
|
|
let VM_CNT = 1;
|
|
let EXP_CNT = 1;
|
|
|
|
let Uses = [EXEC];
|
|
let hasSideEffects = 0;
|
|
let SchedRW = [WriteVMEM];
|
|
|
|
|
|
|
|
bits<1> offen = 0;
|
|
bits<1> idxen = 0;
|
|
bits<1> addr64 = 0;
|
|
bits<1> lds = 0;
|
|
bits<1> has_vdata = !not(lds);
|
|
bits<1> has_vaddr = 1;
|
|
bits<1> has_glc = 1;
|
|
bits<1> has_dlc = 1;
|
|
bits<1> glc_value = 0; // the value for glc if no such operand
|
|
bits<1> dlc_value = 0; // the value for dlc if no such operand
|
|
bits<1> has_srsrc = 1;
|
|
bits<1> has_soffset = 1;
|
|
bits<1> has_offset = 1;
|
|
bits<1> has_slc = 1;
|
|
bits<1> tfe = 0;
|
|
bits<4> elements = 0;
|
|
bits<1> has_sccb = 1;
|
|
bits<1> sccb_value = 0;
|
|
bits<1> IsBufferInv = 0;
|
|
}
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF classes
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class MTBUFGetBaseOpcode<string Op> {
|
|
string ret = !subst("FORMAT_XY", "FORMAT_X",
|
|
!subst("FORMAT_XYZ", "FORMAT_X",
|
|
!subst("FORMAT_XYZW", "FORMAT_X", Op)));
|
|
}
|
|
|
|
|
|
class MTBUF_Pseudo <string opName, dag outs, dag ins,
|
|
string asmOps, list<dag> pattern=[]> :
|
|
BUF_Pseudo <opName, outs, ins, asmOps, pattern> {
|
|
|
|
Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
|
|
let MTBUF = 1;
|
|
}
|
|
|
|
class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> :
|
|
InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> {
|
|
|
|
let isPseudo = 0;
|
|
let isCodeGenOnly = 0;
|
|
|
|
let VM_CNT = 1;
|
|
let EXP_CNT = 1;
|
|
let MTBUF = 1;
|
|
|
|
// copy relevant pseudo op flags
|
|
let UseNamedOperandTable = ps.UseNamedOperandTable;
|
|
let SubtargetPredicate = ps.SubtargetPredicate;
|
|
let OtherPredicates = ps.OtherPredicates;
|
|
let AsmMatchConverter = ps.AsmMatchConverter;
|
|
let Constraints = ps.Constraints;
|
|
let DisableEncoding = ps.DisableEncoding;
|
|
let TSFlags = ps.TSFlags;
|
|
let SchedRW = ps.SchedRW;
|
|
let mayLoad = ps.mayLoad;
|
|
let mayStore = ps.mayStore;
|
|
let IsAtomicRet = ps.IsAtomicRet;
|
|
let IsAtomicNoRet = ps.IsAtomicNoRet;
|
|
|
|
bits<12> offset;
|
|
bits<5> cpol;
|
|
bits<7> format;
|
|
bits<8> vaddr;
|
|
bits<10> vdata;
|
|
bits<7> srsrc;
|
|
bits<8> soffset;
|
|
|
|
bits<4> dfmt = format{3-0};
|
|
bits<3> nfmt = format{6-4};
|
|
|
|
// GFX90A+ only: instruction uses AccVGPR for data
|
|
// Bit supersedes tfe.
|
|
bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);
|
|
}
|
|
|
|
class getMTBUFInsDA<list<RegisterClass> vdataList,
|
|
list<RegisterClass> vaddrList=[], bit hasRestrictedSOffset> {
|
|
RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
|
|
RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
|
|
RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
|
|
|
|
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset),
|
|
(ins SCSrc_b32:$soffset));
|
|
|
|
dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset,
|
|
(ins Offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz));
|
|
|
|
dag Inputs = !if(!empty(vaddrList),
|
|
NonVaddrInputs,
|
|
!con((ins vaddrClass:$vaddr), NonVaddrInputs));
|
|
dag ret = !if(!empty(vdataList),
|
|
Inputs,
|
|
!con((ins vdata_op:$vdata), Inputs));
|
|
}
|
|
|
|
class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasRestrictedSOffset> {
|
|
dag ret =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret,
|
|
(ins))))));
|
|
}
|
|
|
|
class getMTBUFAsmOps<int addrKind> {
|
|
string Pfx =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset",
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn),
|
|
"$vaddr, $srsrc,$format $soffset offen",
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn),
|
|
"$vaddr, $srsrc,$format $soffset idxen",
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn),
|
|
"$vaddr, $srsrc,$format $soffset idxen offen",
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64),
|
|
"$vaddr, $srsrc,$format $soffset addr64",
|
|
"")))));
|
|
string ret = " $vdata, " # Pfx # "$offset$cpol";
|
|
}
|
|
|
|
class MTBUF_SetupAddr<int addrKind> {
|
|
bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn),
|
|
!eq(addrKind, BUFAddrKind.BothEn));
|
|
|
|
bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn),
|
|
!eq(addrKind, BUFAddrKind.BothEn));
|
|
|
|
bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);
|
|
|
|
bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);
|
|
}
|
|
|
|
class MTBUF_Load_Pseudo <string opName,
|
|
int addrKind,
|
|
RegisterClass vdataClass,
|
|
int elems,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind>
|
|
: MTBUF_Pseudo<opName,
|
|
(outs getLdStRegisterOperand<vdataClass>.ret:$vdata),
|
|
getMTBUFIns<addrKindCopy, [], hasRestrictedSOffset>.ret,
|
|
getMTBUFAsmOps<addrKindCopy>.ret,
|
|
pattern>,
|
|
MTBUF_SetupAddr<addrKindCopy> {
|
|
let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
|
|
let mayLoad = 1;
|
|
let mayStore = 0;
|
|
let elements = elems;
|
|
}
|
|
|
|
multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass,
|
|
int elems, bit hasRestrictedSOffset> {
|
|
|
|
def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,
|
|
MTBUFAddr64Table<0, NAME>;
|
|
|
|
def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,
|
|
MTBUFAddr64Table<1, NAME>;
|
|
|
|
def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
|
|
let DisableWQM = 1 in {
|
|
def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
}
|
|
}
|
|
|
|
multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
|
|
int elems> {
|
|
defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>;
|
|
defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>;
|
|
}
|
|
|
|
class MTBUF_Store_Pseudo <string opName,
|
|
int addrKind,
|
|
RegisterClass vdataClass,
|
|
int elems,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind,
|
|
RegisterClass vdataClassCopy = vdataClass>
|
|
: MTBUF_Pseudo<opName,
|
|
(outs),
|
|
getMTBUFIns<addrKindCopy, [vdataClassCopy], hasRestrictedSOffset>.ret,
|
|
getMTBUFAsmOps<addrKindCopy>.ret,
|
|
pattern>,
|
|
MTBUF_SetupAddr<addrKindCopy> {
|
|
let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
|
|
let mayLoad = 0;
|
|
let mayStore = 1;
|
|
let elements = elems;
|
|
}
|
|
|
|
multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass,
|
|
int elems, bit hasRestrictedSOffset> {
|
|
|
|
def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,
|
|
MTBUFAddr64Table<0, NAME>;
|
|
|
|
def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,
|
|
MTBUFAddr64Table<1, NAME>;
|
|
|
|
def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
|
|
let DisableWQM = 1 in {
|
|
def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
|
|
}
|
|
}
|
|
|
|
multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
|
|
int elems> {
|
|
defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>;
|
|
defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF classes
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class MUBUFGetBaseOpcode<string Op> {
|
|
string ret = !subst("DWORDX2", "DWORD",
|
|
!subst("DWORDX3", "DWORD",
|
|
!subst("DWORDX4", "DWORD", Op)));
|
|
}
|
|
|
|
class MUBUF_Pseudo <string opName, dag outs, dag ins,
|
|
string asmOps, list<dag> pattern=[]> :
|
|
BUF_Pseudo <opName, outs, ins, asmOps, pattern> {
|
|
|
|
Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
|
|
let MUBUF = 1;
|
|
let AsmMatchConverter = "cvtMubuf";
|
|
let usesCustomInserter = 1;
|
|
}
|
|
|
|
class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> :
|
|
InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> {
|
|
|
|
let isPseudo = 0;
|
|
let isCodeGenOnly = 0;
|
|
|
|
let VM_CNT = 1;
|
|
let EXP_CNT = 1;
|
|
let MUBUF = 1;
|
|
|
|
// copy relevant pseudo op flags
|
|
let SubtargetPredicate = ps.SubtargetPredicate;
|
|
let AsmMatchConverter = ps.AsmMatchConverter;
|
|
let OtherPredicates = ps.OtherPredicates;
|
|
let Constraints = ps.Constraints;
|
|
let DisableEncoding = ps.DisableEncoding;
|
|
let TSFlags = ps.TSFlags;
|
|
let UseNamedOperandTable = ps.UseNamedOperandTable;
|
|
let SchedRW = ps.SchedRW;
|
|
let mayLoad = ps.mayLoad;
|
|
let mayStore = ps.mayStore;
|
|
let IsAtomicRet = ps.IsAtomicRet;
|
|
let IsAtomicNoRet = ps.IsAtomicNoRet;
|
|
let VALU = ps.VALU;
|
|
let LGKM_CNT = ps.LGKM_CNT;
|
|
|
|
bits<12> offset;
|
|
bits<5> cpol;
|
|
bits<8> vaddr;
|
|
bits<10> vdata;
|
|
bits<7> srsrc;
|
|
bits<8> soffset;
|
|
|
|
// GFX90A+ only: instruction uses AccVGPR for data
|
|
// Bit supersedes tfe.
|
|
bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);
|
|
}
|
|
|
|
// For cache invalidation instructions.
|
|
class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
|
|
MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
|
|
|
|
let AsmMatchConverter = "";
|
|
|
|
let hasSideEffects = 1;
|
|
let mayLoad = 0;
|
|
let mayStore = 0;
|
|
|
|
let IsBufferInv = 1;
|
|
// Set everything else to 0.
|
|
let offen = 0;
|
|
let idxen = 0;
|
|
let addr64 = 0;
|
|
let has_vdata = 0;
|
|
let has_vaddr = 0;
|
|
let has_glc = 0;
|
|
let has_dlc = 0;
|
|
let glc_value = 0;
|
|
let dlc_value = 0;
|
|
let has_srsrc = 0;
|
|
let has_soffset = 0;
|
|
let has_offset = 0;
|
|
let has_slc = 0;
|
|
let has_sccb = 0;
|
|
let sccb_value = 0;
|
|
}
|
|
|
|
class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> {
|
|
RegisterOperand tfeVDataOp =
|
|
!if(!eq(RC.Size, 32), AVLdSt_64,
|
|
!if(!eq(RC.Size, 64), AVLdSt_96,
|
|
!if(!eq(RC.Size, 96), AVLdSt_128,
|
|
!if(!eq(RC.Size, 128), AVLdSt_160,
|
|
RegisterOperand<VReg_1> // Invalid register.
|
|
))));
|
|
|
|
RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret);
|
|
}
|
|
|
|
class getMUBUFInsDA<list<RegisterClass> vdataList,
|
|
list<RegisterClass> vaddrList, bit isTFE, bit hasRestrictedSOffset> {
|
|
RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
|
|
RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
|
|
RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret;
|
|
|
|
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
|
|
dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz));
|
|
|
|
dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs));
|
|
dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs));
|
|
}
|
|
|
|
class getMUBUFElements<ValueType vt> {
|
|
int ret =
|
|
!if(!eq(vt, f16), 1,
|
|
!if(!eq(vt, v2f16), 2,
|
|
!if(!eq(vt, v3f16), 3,
|
|
!if(!eq(vt, v4f16), 4,
|
|
!if(!eq(vt.Size, 32), 1,
|
|
!if(!eq(vt.Size, 64), 2,
|
|
!if(!eq(vt.Size, 96), 3,
|
|
!if(!eq(vt.Size, 128), 4, 0)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
);
|
|
}
|
|
|
|
class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasRestrictedSOffset> {
|
|
dag ret =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret,
|
|
(ins))))));
|
|
}
|
|
|
|
class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> {
|
|
string Vdata = !if(noVdata, " ", " $vdata, ");
|
|
string Lds = !if(isLds, " lds", "");
|
|
string TFE = !if(isTFE, " tfe", "");
|
|
string MainArgs =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen",
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen",
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
|
|
"")))));
|
|
string Offset = "$offset";
|
|
string OtherArgs = "$cpol";
|
|
|
|
string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE;
|
|
}
|
|
|
|
class MUBUF_SetupAddr<int addrKind> {
|
|
bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn),
|
|
!eq(addrKind, BUFAddrKind.BothEn));
|
|
|
|
bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn),
|
|
!eq(addrKind, BUFAddrKind.BothEn));
|
|
|
|
bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);
|
|
|
|
bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);
|
|
}
|
|
|
|
class MUBUF_Load_Pseudo <string opName,
|
|
int addrKind,
|
|
ValueType vdata_vt,
|
|
bit HasTiedDest = 0,
|
|
bit isLds = 0,
|
|
bit isLdsOpc = 0,
|
|
bit isTFE = 0,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind,
|
|
RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret.RegClass,
|
|
RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret>
|
|
: MUBUF_Pseudo<opName,
|
|
!if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)),
|
|
!con(getMUBUFIns<addrKindCopy, [], isTFE, hasRestrictedSOffset>.ret,
|
|
!if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))),
|
|
getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret,
|
|
pattern>,
|
|
MUBUF_SetupAddr<addrKindCopy> {
|
|
let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") #
|
|
"_" # getAddrName<addrKindCopy>.ret;
|
|
let AsmMatchConverter = "cvtMubuf";
|
|
|
|
let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
|
|
let LGKM_CNT = isLds;
|
|
let has_vdata = !not(!or(isLds, isLdsOpc));
|
|
let mayLoad = 1;
|
|
let mayStore = isLds;
|
|
let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]);
|
|
let tfe = isTFE;
|
|
let lds = isLds;
|
|
let elements = getMUBUFElements<vdata_vt>.ret;
|
|
let VALU = isLds;
|
|
}
|
|
|
|
class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat <
|
|
(load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),
|
|
(load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset))
|
|
>;
|
|
|
|
class MUBUF_Addr64_Load_Pat <Instruction inst,
|
|
ValueType load_vt = i32,
|
|
SDPatternOperator ld = null_frag> : GCNPat <
|
|
(load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),
|
|
(load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset))
|
|
>;
|
|
|
|
multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
|
|
def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
|
|
def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>;
|
|
}
|
|
defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt,
|
|
bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> {
|
|
defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt);
|
|
|
|
def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,
|
|
MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
|
|
|
|
def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,
|
|
MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
|
|
|
|
def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
|
|
let DisableWQM = 1 in {
|
|
def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32,
|
|
bit TiedDest = 0, bit isLds = 0> {
|
|
defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>;
|
|
defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>;
|
|
|
|
if !not(isLds) then {
|
|
defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>;
|
|
defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> {
|
|
defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>;
|
|
defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName,
|
|
ValueType load_vt = i32,
|
|
bit TiedDest = 0,
|
|
bit isLds = 0,
|
|
bit isLdsOpc = 1> {
|
|
|
|
defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt);
|
|
|
|
def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>;
|
|
def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
|
|
def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
|
|
def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
|
|
|
|
def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
|
|
def _VBUFFER_OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
|
|
def _VBUFFER_IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
|
|
def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
|
|
}
|
|
|
|
class MUBUF_Store_Pseudo <string opName,
|
|
int addrKind,
|
|
ValueType store_vt,
|
|
bit isTFE = 0,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind>
|
|
: MUBUF_Pseudo<opName,
|
|
(outs),
|
|
getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret.RegClass], isTFE, hasRestrictedSOffset>.ret,
|
|
getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret,
|
|
pattern>,
|
|
MUBUF_SetupAddr<addrKindCopy> {
|
|
let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") #
|
|
getAddrName<addrKindCopy>.ret;
|
|
let mayLoad = 0;
|
|
let mayStore = 1;
|
|
let elements = getMUBUFElements<store_vt>.ret;
|
|
let tfe = isTFE;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> {
|
|
|
|
def : GCNPat <
|
|
(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)),
|
|
(!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>;
|
|
|
|
def : GCNPat <
|
|
(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),
|
|
(!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>;
|
|
}
|
|
defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>;
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt,
|
|
bit isTFE, bit hasRestrictedSOffset> {
|
|
defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt);
|
|
|
|
def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>,
|
|
MUBUFAddr64Table<0, NAME>;
|
|
|
|
def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset>,
|
|
MUBUFAddr64Table<1, NAME>;
|
|
|
|
def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
|
|
let DisableWQM = 1 in {
|
|
def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> {
|
|
defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>;
|
|
defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>;
|
|
|
|
defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>;
|
|
defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>;
|
|
}
|
|
|
|
class MUBUF_Pseudo_Store_Lds<string opName>
|
|
: MUBUF_Pseudo<opName,
|
|
(outs),
|
|
(ins SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset, CPol:$cpol, i1imm:$swz),
|
|
" $srsrc, $soffset$offset lds$cpol"> {
|
|
let LGKM_CNT = 1;
|
|
let mayLoad = 1;
|
|
let mayStore = 1;
|
|
|
|
let has_vdata = 0;
|
|
let has_vaddr = 0;
|
|
let lds = 1;
|
|
let VALU = 1;
|
|
|
|
let Uses = [EXEC, M0];
|
|
let AsmMatchConverter = "cvtMubuf";
|
|
}
|
|
|
|
class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasRestrictedSOffset,
|
|
list<RegisterClass> vaddrList=[]> {
|
|
RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
|
|
RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
|
|
|
|
dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata));
|
|
dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr)));
|
|
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
|
|
dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset));
|
|
dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol),
|
|
(ins CPol_NonGLC_WithDefault:$cpol));
|
|
|
|
dag ret = !con(Data, MainInputs, CPol);
|
|
}
|
|
|
|
class getMUBUFAtomicIns<int addrKind,
|
|
RegisterClass vdataClass,
|
|
bit vdata_in,
|
|
bit hasRestrictedSOffset,
|
|
// Workaround bug bz30254
|
|
RegisterClass vdataClassCopy=vdataClass> {
|
|
dag ret =
|
|
!if(!eq(addrKind, BUFAddrKind.Offset),
|
|
getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.OffEn),
|
|
getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.IdxEn),
|
|
getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.BothEn),
|
|
getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret,
|
|
!if(!eq(addrKind, BUFAddrKind.Addr64),
|
|
getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret,
|
|
(ins))))));
|
|
}
|
|
|
|
class MUBUF_Atomic_Pseudo<string opName,
|
|
int addrKind,
|
|
dag outs,
|
|
dag ins,
|
|
string asmOps,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind>
|
|
: MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
|
|
MUBUF_SetupAddr<addrKindCopy> {
|
|
let mayStore = 1;
|
|
let mayLoad = 1;
|
|
let hasPostISelHook = 1;
|
|
let hasSideEffects = 1;
|
|
let DisableWQM = 1;
|
|
let has_glc = 0;
|
|
let has_dlc = 0;
|
|
let has_sccb = 1;
|
|
let AsmMatchConverter = "cvtMubufAtomic";
|
|
}
|
|
|
|
class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
|
|
RegisterClass vdataClass,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind,
|
|
RegisterClass vdataClassCopy = vdataClass>
|
|
: MUBUF_Atomic_Pseudo<opName, addrKindCopy,
|
|
(outs),
|
|
getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasRestrictedSOffset>.ret,
|
|
getMUBUFAsmOps<addrKindCopy>.ret,
|
|
pattern> {
|
|
let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
|
|
let glc_value = 0;
|
|
let dlc_value = 0;
|
|
let sccb_value = 0;
|
|
let IsAtomicNoRet = 1;
|
|
}
|
|
|
|
class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
|
|
RegisterClass vdataClass,
|
|
bit hasRestrictedSOffset = 0,
|
|
list<dag> pattern=[],
|
|
// Workaround bug bz30254
|
|
int addrKindCopy = addrKind,
|
|
RegisterClass vdataClassCopy = vdataClass,
|
|
RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret>
|
|
: MUBUF_Atomic_Pseudo<opName, addrKindCopy,
|
|
(outs vdata_op:$vdata),
|
|
getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasRestrictedSOffset>.ret,
|
|
getMUBUFAsmOps<addrKindCopy>.ret,
|
|
pattern> {
|
|
let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
|
|
let glc_value = 1;
|
|
let dlc_value = 0;
|
|
let sccb_value = 0;
|
|
let IsAtomicRet = 1;
|
|
let Constraints = "$vdata = $vdata_in";
|
|
let DisableEncoding = "$vdata_in";
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
|
|
RegisterClass vdataClass,
|
|
ValueType vdataType> {
|
|
let FPAtomic = vdataType.isFP in {
|
|
def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>,
|
|
MUBUFAddr64Table <0, NAME>;
|
|
def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>,
|
|
MUBUFAddr64Table <1, NAME>;
|
|
def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>;
|
|
def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>;
|
|
def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;
|
|
|
|
def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>,
|
|
MUBUFAddr64Table <0, NAME # "_VBUFFER">;
|
|
def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>,
|
|
MUBUFAddr64Table <1, NAME # "_VBUFFER">;
|
|
def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;
|
|
def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;
|
|
def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
|
|
RegisterClass vdataClass,
|
|
ValueType vdataType,
|
|
SDPatternOperator atomic> {
|
|
let FPAtomic = vdataType.isFP in {
|
|
def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0,
|
|
[(set vdataType:$vdata,
|
|
(atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),
|
|
vdataType:$vdata_in))]>,
|
|
MUBUFAddr64Table <0, NAME # "_RTN">;
|
|
|
|
def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0,
|
|
[(set vdataType:$vdata,
|
|
(atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
|
|
vdataType:$vdata_in))]>,
|
|
MUBUFAddr64Table <1, NAME # "_RTN">;
|
|
|
|
def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>;
|
|
def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>;
|
|
def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;
|
|
|
|
def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1,
|
|
[(set vdataType:$vdata,
|
|
(atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),
|
|
vdataType:$vdata_in))]>,
|
|
MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">;
|
|
|
|
def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1,
|
|
[(set vdataType:$vdata,
|
|
(atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
|
|
vdataType:$vdata_in))]>,
|
|
MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">;
|
|
|
|
def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;
|
|
def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;
|
|
def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Pseudo_Atomics <string opName,
|
|
RegisterClass vdataClass,
|
|
ValueType vdataType,
|
|
SDPatternOperator atomic = null_frag> :
|
|
MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>,
|
|
MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF Instructions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_format_x", f32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_xy", v2f32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_xyz", v3f32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_xyzw", v4f32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_x", f32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_xy", v2f32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_xyz", v3f32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_xyzw", v4f32
|
|
>;
|
|
|
|
let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in {
|
|
let TiedSourceNotRead = 1 in {
|
|
defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_x", i32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xy", v2i32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xyz", v3i32
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xyzw", v4i32
|
|
>;
|
|
}
|
|
defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_x", i32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xy", v2i32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xyz", v3i32
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xyzw", v4i32
|
|
>;
|
|
} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1.
|
|
|
|
let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in {
|
|
let TiedSourceNotRead = 1 in {
|
|
defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_x", f16
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xy", v2f16
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xyz", v3f16
|
|
>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_xyzw", v4f16
|
|
>;
|
|
}
|
|
defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_x", f16
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xy", v2f16
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xyz", v3f16
|
|
>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_xyzw", v4f16
|
|
>;
|
|
} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1.
|
|
|
|
defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_ubyte", i32
|
|
>;
|
|
defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_sbyte", i32
|
|
>;
|
|
defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_ushort", i32
|
|
>;
|
|
defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_sshort", i32
|
|
>;
|
|
defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
|
|
"buffer_load_dword", i32
|
|
>;
|
|
defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_dwordx2", v2i32
|
|
>;
|
|
defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_dwordx3", v3i32
|
|
>;
|
|
defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_dwordx4", v4i32
|
|
>;
|
|
|
|
defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_b32", i32
|
|
>;
|
|
defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_format_x", f32
|
|
>;
|
|
defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_i8", i32
|
|
>;
|
|
defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_i16", i32
|
|
>;
|
|
defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_u8", i32
|
|
>;
|
|
defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc <
|
|
"buffer_load_lds_u16", i32
|
|
>;
|
|
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
|
|
|
|
foreach vt = Reg32Types.types in {
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>;
|
|
}
|
|
|
|
foreach vt = VReg_64.RegTypes in {
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>;
|
|
}
|
|
|
|
foreach vt = VReg_96.RegTypes in {
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>;
|
|
}
|
|
|
|
foreach vt = VReg_128.RegTypes in {
|
|
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>;
|
|
}
|
|
|
|
defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
|
|
"buffer_store_byte", i32
|
|
>;
|
|
defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
|
|
"buffer_store_short", i32
|
|
>;
|
|
defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
|
|
"buffer_store_dword", i32
|
|
>;
|
|
defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_dwordx2", v2i32
|
|
>;
|
|
defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_dwordx3", v3i32
|
|
>;
|
|
defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
|
|
"buffer_store_dwordx4", v4i32
|
|
>;
|
|
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>;
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>;
|
|
|
|
foreach vt = Reg32Types.types in {
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>;
|
|
}
|
|
|
|
foreach vt = VReg_64.RegTypes in {
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>;
|
|
}
|
|
|
|
foreach vt = VReg_96.RegTypes in {
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>;
|
|
}
|
|
|
|
foreach vt = VReg_128.RegTypes in {
|
|
defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>;
|
|
}
|
|
|
|
defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_swap", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_cmpswap", VReg_64, v2i32
|
|
>;
|
|
defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_add", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_sub", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_smin", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_umin", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_smax", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_umax", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_and", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_or", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_xor", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_inc", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_dec", VGPR_32, i32
|
|
>;
|
|
defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_swap_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_cmpswap_x2", VReg_128, v2i64
|
|
>;
|
|
defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_add_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_sub_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_smin_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_umin_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_smax_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_umax_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_and_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_or_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_xor_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_inc_x2", VReg_64, i64
|
|
>;
|
|
defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_dec_x2", VReg_64, i64
|
|
>;
|
|
|
|
let OtherPredicates = [HasGFX10_BEncoding] in {
|
|
defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub
|
|
>;
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX8GFX9 in {
|
|
def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
|
|
/*
|
|
defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
|
|
defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
|
|
*/
|
|
|
|
def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
|
|
int_amdgcn_buffer_wbinvl1_sc>;
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX6GFX7GFX10Plus in {
|
|
|
|
defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag
|
|
>;
|
|
defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fmin", VGPR_32, f32, null_frag
|
|
>;
|
|
defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fmax", VGPR_32, f32, null_frag
|
|
>;
|
|
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX6GFX7GFX10 in {
|
|
|
|
defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag
|
|
>;
|
|
defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fmin_x2", VReg_64, f64, null_frag
|
|
>;
|
|
defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_fmax_x2", VReg_64, f64, null_frag
|
|
>;
|
|
|
|
}
|
|
|
|
let SubtargetPredicate = HasD16LoadStore in {
|
|
let TiedSourceNotRead = 1 in {
|
|
|
|
defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_ubyte_d16", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
|
|
"buffer_load_ubyte_d16_hi", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_sbyte_d16", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
|
|
"buffer_load_sbyte_d16_hi", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
|
|
"buffer_load_short_d16", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
|
|
"buffer_load_short_d16_hi", i32, 1
|
|
>;
|
|
|
|
defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
|
|
"buffer_load_format_d16_hi_x", i32
|
|
>;
|
|
} // End TiedSourceNotRead
|
|
|
|
defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
|
|
"buffer_store_byte_d16_hi", i32
|
|
>;
|
|
|
|
defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
|
|
"buffer_store_short_d16_hi", i32
|
|
>;
|
|
|
|
defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
|
|
"buffer_store_format_d16_hi_x", i32
|
|
>;
|
|
|
|
} // End HasD16LoadStore
|
|
|
|
let SubtargetPredicate = isNotGFX12Plus in
|
|
def BUFFER_WBINVL1 : MUBUF_Invalidate <
|
|
"buffer_wbinvl1", int_amdgcn_buffer_wbinvl1
|
|
>;
|
|
|
|
let SubtargetPredicate = HasAtomicFaddNoRtnInsts in
|
|
defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN<
|
|
"buffer_atomic_add_f32", VGPR_32, f32
|
|
>;
|
|
|
|
let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in
|
|
defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
|
|
"buffer_atomic_pk_add_f16", VGPR_32, v2f16
|
|
>;
|
|
|
|
let OtherPredicates = [HasAtomicFaddRtnInsts] in
|
|
defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN<
|
|
"buffer_atomic_add_f32", VGPR_32, f32, null_frag
|
|
>;
|
|
|
|
let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in
|
|
defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN <
|
|
"buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag
|
|
>;
|
|
|
|
let SubtargetPredicate = isGFX12Plus in {
|
|
defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_cond_sub_u32", VGPR_32, i32
|
|
>;
|
|
|
|
let FPAtomic = 1 in
|
|
defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics <
|
|
"buffer_atomic_pk_add_bf16", VGPR_32, v2bf16
|
|
>;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF Instructions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>;
|
|
defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>;
|
|
defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>;
|
|
defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>;
|
|
defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>;
|
|
defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
|
|
|
|
let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
|
|
let TiedSourceNotRead = 1 in {
|
|
defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>;
|
|
}
|
|
defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
|
|
let TiedSourceNotRead = 1 in {
|
|
defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>;
|
|
}
|
|
defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
|
|
} // End HasPackedD16VMem.
|
|
|
|
let SubtargetPredicate = isGFX7Plus in {
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Instruction definitions for CI and newer.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
|
|
int_amdgcn_buffer_wbinvl1_vol>;
|
|
|
|
} // End let SubtargetPredicate = isGFX7Plus
|
|
|
|
let SubtargetPredicate = isGFX90APlus in {
|
|
def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> {
|
|
let has_glc = 1;
|
|
let has_sccb = 1;
|
|
let InOperandList = (ins CPol_0:$cpol);
|
|
let AsmOperands = "$cpol";
|
|
}
|
|
def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> {
|
|
let SubtargetPredicate = isGFX90AOnly;
|
|
}
|
|
} // End SubtargetPredicate = isGFX90APlus
|
|
|
|
let SubtargetPredicate = HasBufferFlatGlobalAtomicsF64 in {
|
|
defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>;
|
|
defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>;
|
|
defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>;
|
|
} // End SubtargetPredicate = HasBufferFlatGlobalAtomicsF64
|
|
|
|
def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> {
|
|
let SubtargetPredicate = isGFX940Plus;
|
|
let has_glc = 1;
|
|
let has_sccb = 1;
|
|
let InOperandList = (ins CPol_0:$cpol);
|
|
let AsmOperands = "$cpol";
|
|
}
|
|
|
|
def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
|
|
def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF Patterns
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// buffer_load/store_format patterns
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>);
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0)),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0)),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm)),
|
|
(!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm)),
|
|
(!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt>{
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>;
|
|
}
|
|
defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
|
|
}
|
|
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
|
|
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">;
|
|
|
|
let OtherPredicates = [HasUnpackedD16VMem] in {
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
|
|
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
let OtherPredicates = [HasPackedD16VMem] in {
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
|
|
} // End HasPackedD16VMem.
|
|
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
|
|
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">;
|
|
|
|
multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>);
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm),
|
|
(!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm),
|
|
(!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
|
|
getVregSrcForVT<vt>.ret:$vdata,
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary),
|
|
(extract_swz $auxiliary))
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>;
|
|
}
|
|
defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
|
|
}
|
|
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
|
|
|
|
let OtherPredicates = [HasUnpackedD16VMem] in {
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
|
|
defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
let OtherPredicates = [HasPackedD16VMem] in {
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
|
|
} // End HasPackedD16VMem.
|
|
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
|
|
defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// buffer_atomic patterns
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> {
|
|
foreach RtnMode = ["ret", "noret"] in {
|
|
|
|
defvar Op = !cast<SDPatternOperator>(OpPrefix
|
|
# !if(!eq(RtnMode, "ret"), "", "_noret")
|
|
# !if(isIntr, "", "_" # vt.Size));
|
|
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
|
|
|
|
let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
|
|
def : GCNPat<
|
|
(vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in,
|
|
SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset)
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
|
|
vt:$vdata_in)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in,
|
|
VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset)
|
|
>;
|
|
} // end let AddedComplexity
|
|
|
|
} // end foreach RtnMode
|
|
}
|
|
|
|
multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>;
|
|
}
|
|
defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>;
|
|
}
|
|
|
|
multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> {
|
|
defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>;
|
|
}
|
|
|
|
multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> {
|
|
foreach RtnMode = ["ret", "noret"] in {
|
|
|
|
defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global"
|
|
# !if(!eq(RtnMode, "ret"), "", "_noret")
|
|
# "_" # vt.Size);
|
|
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
|
|
defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass;
|
|
|
|
let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
|
|
defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)
|
|
data_vt_RC:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset,
|
|
Offset:$offset);
|
|
def : GCNPat<
|
|
(vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, data_vt_RC)),
|
|
!if(!eq(vt, i32), sub0, sub0_sub1)),
|
|
OffsetResDag)
|
|
>;
|
|
|
|
defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix)
|
|
data_vt_RC:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc,
|
|
SCSrc_b32:$soffset, Offset:$offset);
|
|
def : GCNPat<
|
|
(vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
|
|
data_vt:$vdata_in)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, data_vt_RC)),
|
|
!if(!eq(vt, i32), sub0, sub0_sub1)),
|
|
Addr64ResDag)
|
|
>;
|
|
} // end let AddedComplexity
|
|
|
|
} // end foreach RtnMode
|
|
}
|
|
|
|
multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>;
|
|
}
|
|
defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">;
|
|
}
|
|
|
|
|
|
foreach Ty = [i32, i64] in {
|
|
|
|
defvar Suffix = !if(!eq(Ty, i64), "_X2", "");
|
|
|
|
defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>;
|
|
defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>;
|
|
|
|
} // end foreach Ty
|
|
|
|
defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;
|
|
defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;
|
|
|
|
multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst,
|
|
list<string> RtnModes = ["ret", "noret"]> {
|
|
foreach RtnMode = RtnModes in {
|
|
|
|
defvar Op = !cast<SDPatternOperator>(OpPrefix
|
|
# !if(!eq(RtnMode, "ret"), "", "_noret"));
|
|
|
|
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
|
|
defvar CachePolicy = !if(!eq(RtnMode, "ret"),
|
|
(extract_cpol_set_glc $auxiliary), (extract_cpol $auxiliary));
|
|
|
|
let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
|
|
def : GCNPat<
|
|
(vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset),
|
|
timm:$offset, timm:$auxiliary, 0)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)
|
|
getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, CachePolicy)
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset),
|
|
timm:$offset, timm:$auxiliary, timm)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix)
|
|
getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc,
|
|
SCSrc_b32:$soffset, timm:$offset, CachePolicy)
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset,
|
|
(BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, 0)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix)
|
|
getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc,
|
|
SCSrc_b32:$soffset, timm:$offset, CachePolicy)
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset,
|
|
(BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, timm)),
|
|
(!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix)
|
|
getVregSrcForVT<vt>.ret:$vdata_in,
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy)
|
|
>;
|
|
} // end let AddedComplexity
|
|
|
|
} // end foreach RtnMode
|
|
}
|
|
|
|
multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst,
|
|
list<string> RtnModes = ["ret", "noret"]> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>;
|
|
}
|
|
defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>;
|
|
}
|
|
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">;
|
|
|
|
let OtherPredicates = [HasAtomicCSubNoRtnInsts] in
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>;
|
|
|
|
let SubtargetPredicate = isGFX12Plus in {
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd_bf16", v2bf16, "BUFFER_ATOMIC_PK_ADD_BF16_VBUFFER">;
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["ret"]>;
|
|
|
|
let OtherPredicates = [HasAtomicCSubNoRtnInsts] in
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>;
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX6GFX7GFX10Plus in {
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">;
|
|
}
|
|
let SubtargetPredicate = isGFX6GFX7GFX10 in {
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">;
|
|
}
|
|
|
|
class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag <
|
|
(ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7),
|
|
(vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> {
|
|
let HasNoUse = true;
|
|
}
|
|
|
|
multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt,
|
|
string opcode> {
|
|
def : GCNPat<
|
|
(NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,
|
|
0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (extract_cpol $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
|
|
0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm),
|
|
(!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (extract_cpol $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,
|
|
i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0),
|
|
(!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (extract_cpol $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
|
|
i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm),
|
|
(!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
|
|
getVregSrcForVT<vt>.ret:$vdata_in,
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary))
|
|
>;
|
|
}
|
|
|
|
multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
|
|
string opcode> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>;
|
|
}
|
|
defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">;
|
|
}
|
|
|
|
let OtherPredicates = [HasAtomicFaddNoRtnInsts] in
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>;
|
|
|
|
let OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] in {
|
|
let SubtargetPredicate = isGFX9Only in
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>;
|
|
|
|
let SubtargetPredicate = isGFX12Plus in
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16_VBUFFER", ["noret"]>;
|
|
} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts]
|
|
|
|
let OtherPredicates = [HasAtomicFaddRtnInsts] in
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>;
|
|
|
|
let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in {
|
|
let SubtargetPredicate = isGFX9Only in
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>;
|
|
|
|
let SubtargetPredicate = isGFX12Plus in
|
|
defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16_VBUFFER", ["ret"]>;
|
|
} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts]
|
|
|
|
let OtherPredicates = [HasBufferFlatGlobalAtomicsF64] in {
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">;
|
|
defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">;
|
|
} // End SubtargetPredicate = HasBufferFlatGlobalAtomicsF64
|
|
|
|
multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> {
|
|
foreach RtnMode = ["ret", "noret"] in {
|
|
defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap
|
|
# !if(!eq(RtnMode, "ret"), "", "_noret"));
|
|
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
|
|
defvar CachePolicy = !if(!eq(RtnMode, "ret"),
|
|
(extract_cpol_set_glc $auxiliary),
|
|
(extract_cpol $auxiliary));
|
|
defvar SrcRC = getVregSrcForVT<vt>.ret;
|
|
defvar DataRC = getVregSrcForVT<data_vt>.ret.RegClass;
|
|
defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1);
|
|
defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3);
|
|
|
|
defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)
|
|
(REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);
|
|
def : GCNPat<
|
|
(vt (Op
|
|
vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset),
|
|
timm:$offset, timm:$auxiliary, 0)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG OffsetResDag, SubLo),
|
|
OffsetResDag)
|
|
>;
|
|
|
|
defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix)
|
|
(REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
|
|
VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
CachePolicy);
|
|
def : GCNPat<
|
|
(vt (Op
|
|
vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,
|
|
0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG IdxenResDag, SubLo),
|
|
IdxenResDag)
|
|
>;
|
|
|
|
defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix)
|
|
(REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
|
|
VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
CachePolicy);
|
|
def : GCNPat<
|
|
(vt (Op
|
|
vt:$data, vt:$cmp, v4i32:$rsrc, 0,
|
|
i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, 0)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG OffenResDag, SubLo),
|
|
OffenResDag)
|
|
>;
|
|
|
|
defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix)
|
|
(REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);
|
|
def : GCNPat<
|
|
(vt (Op
|
|
vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,
|
|
i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$auxiliary, timm)),
|
|
!if(!eq(RtnMode, "ret"),
|
|
(EXTRACT_SUBREG BothenResDag, SubLo),
|
|
BothenResDag)
|
|
>;
|
|
} // end foreach RtnMode
|
|
}
|
|
|
|
multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>;
|
|
}
|
|
defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">;
|
|
}
|
|
|
|
defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;
|
|
defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;
|
|
|
|
class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
|
|
PatFrag constant_ld> : GCNPat <
|
|
(vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
|
|
i32:$offset))),
|
|
(Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
|
|
>;
|
|
|
|
multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
|
|
ValueType vt, PatFrag atomic_ld> {
|
|
def : GCNPat <
|
|
(vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),
|
|
(Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
|
|
>;
|
|
|
|
def : GCNPat <
|
|
(vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))),
|
|
(Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))
|
|
>;
|
|
}
|
|
|
|
let SubtargetPredicate = isGFX6GFX7 in {
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
|
|
def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
|
|
|
|
defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
|
|
defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
|
|
} // End SubtargetPredicate = isGFX6GFX7
|
|
|
|
multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt,
|
|
PatFrag ld> {
|
|
def : GCNPat <
|
|
(vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),
|
|
(!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset)
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt,
|
|
PatFrag ld> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>;
|
|
}
|
|
defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>;
|
|
}
|
|
|
|
let OtherPredicates = [Has16BitInsts] in {
|
|
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>;
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>;
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>;
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>;
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>;
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>;
|
|
|
|
defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>;
|
|
|
|
} // End OtherPredicates = [Has16BitInsts]
|
|
|
|
multiclass MUBUFScratchLoadPat_Common <string Instr,
|
|
ValueType vt, PatFrag ld> {
|
|
def : GCNPat <
|
|
(vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
|
|
i32:$soffset, i32:$offset))),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0)
|
|
>;
|
|
|
|
def : GCNPat <
|
|
(vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0)
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUFScratchLoadPat <string Instr,
|
|
ValueType vt, PatFrag ld> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>;
|
|
}
|
|
defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>;
|
|
}
|
|
|
|
// XXX - Is it possible to have a complex pattern in a PatFrag?
|
|
multiclass MUBUFScratchLoadPat_D16_Common <string Instr,
|
|
ValueType vt, PatFrag ld_frag> {
|
|
def : GCNPat <
|
|
(ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in)
|
|
>;
|
|
|
|
def : GCNPat <
|
|
(ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in)
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUFScratchLoadPat_D16 <string Instr,
|
|
ValueType vt, PatFrag ld_frag> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>;
|
|
}
|
|
defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>;
|
|
}
|
|
|
|
let OtherPredicates = [DisableFlatScratch] in {
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>;
|
|
|
|
foreach vt = Reg32Types.types in {
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>;
|
|
}
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>;
|
|
defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>;
|
|
|
|
let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in {
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>;
|
|
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>;
|
|
defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>;
|
|
}
|
|
|
|
} // End OtherPredicates = [DisableFlatScratch]
|
|
|
|
multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
|
|
ValueType vt, PatFrag atomic_st> {
|
|
def : GCNPat <
|
|
(atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),
|
|
(Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)
|
|
>;
|
|
|
|
def : GCNPat <
|
|
(atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)),
|
|
(Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))
|
|
>;
|
|
}
|
|
let SubtargetPredicate = isGFX6GFX7 in {
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>;
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>;
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>;
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>;
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>;
|
|
defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>;
|
|
} // End Predicates = isGFX6GFX7
|
|
|
|
|
|
multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt,
|
|
PatFrag st> {
|
|
|
|
def : GCNPat <
|
|
(st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)),
|
|
(!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset)
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt,
|
|
PatFrag st> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>;
|
|
}
|
|
defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>;
|
|
}
|
|
|
|
defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>;
|
|
defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>;
|
|
|
|
multiclass MUBUFScratchStorePat_Common <string Instr,
|
|
ValueType vt, PatFrag st,
|
|
RegisterClass rc = VGPR_32> {
|
|
def : GCNPat <
|
|
(st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
|
|
i32:$soffset, i32:$offset)),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0)
|
|
>;
|
|
|
|
def : GCNPat <
|
|
(st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
|
|
i32:$offset)),
|
|
(!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0)
|
|
>;
|
|
}
|
|
|
|
multiclass MUBUFScratchStorePat <string Instr,
|
|
ValueType vt, PatFrag st,
|
|
RegisterClass rc = VGPR_32> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>;
|
|
}
|
|
defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>;
|
|
}
|
|
|
|
let OtherPredicates = [DisableFlatScratch] in {
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>;
|
|
|
|
foreach vt = Reg32Types.types in {
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>;
|
|
}
|
|
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>;
|
|
|
|
|
|
let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in {
|
|
// Hiding the extract high pattern in the PatFrag seems to not
|
|
// automatically increase the complexity.
|
|
let AddedComplexity = 1 in {
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>;
|
|
defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>;
|
|
}
|
|
}
|
|
} // End OtherPredicates = [DisableFlatScratch]
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF Patterns
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// tbuffer_load/store_format patterns
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>);
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, 0)),
|
|
(!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, timm)),
|
|
(!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, 0)),
|
|
(!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, timm)),
|
|
(!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
|
|
(as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
}
|
|
|
|
multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>;
|
|
}
|
|
defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
|
|
}
|
|
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
|
|
|
|
let OtherPredicates = [HasUnpackedD16VMem] in {
|
|
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
|
|
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
|
|
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
|
|
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
|
|
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
let OtherPredicates = [HasPackedD16VMem] in {
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
|
|
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
|
|
} // End HasPackedD16VMem.
|
|
|
|
multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>);
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, 0),
|
|
(!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, timm),
|
|
(!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,
|
|
timm:$format, timm:$auxiliary, 0),
|
|
(!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
|
|
timm:$offset, (as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
|
|
def : GCNPat<
|
|
(st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset),
|
|
timm:$offset, timm:$format, timm:$auxiliary, timm),
|
|
(!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
|
|
getVregSrcForVT<vt>.ret:$vdata,
|
|
(REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
|
|
SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format),
|
|
(extract_cpol $auxiliary), (extract_swz $auxiliary))
|
|
>;
|
|
}
|
|
|
|
multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
|
|
string opcode, ValueType memoryVt = vt> {
|
|
let SubtargetPredicate = HasUnrestrictedSOffset in {
|
|
defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>;
|
|
}
|
|
defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
|
|
}
|
|
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
|
|
|
|
let OtherPredicates = [HasUnpackedD16VMem] in {
|
|
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
|
|
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
|
|
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
|
|
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
|
|
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
let OtherPredicates = [HasPackedD16VMem] in {
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
|
|
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
|
|
} // End HasPackedD16VMem.
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Target-specific instruction encodings.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef,
|
|
string real_name = ps.Mnemonic> :
|
|
MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{31-26} = 0x38;
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx11<bits<8> op, string real_name = !cast<MUBUF_Pseudo>(NAME).Mnemonic> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx11 : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> {
|
|
let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
// In GFX11 dlc is applicable to all loads/stores/atomics.
|
|
let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value);
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{25-18} = op;
|
|
let Inst{53} = ps.tfe;
|
|
let Inst{54} = ps.offen;
|
|
let Inst{55} = ps.idxen;
|
|
let AssemblerPredicate = isGFX11Only;
|
|
let DecoderNamespace = "GFX11";
|
|
}
|
|
}
|
|
|
|
class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
|
|
Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> {
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{16} = ps.lds;
|
|
let Inst{24-18} = op;
|
|
let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{55} = ps.tfe;
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx10<bits<8> op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx10 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
|
|
let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);
|
|
let Inst{25} = op{7};
|
|
let AssemblerPredicate = isGFX10Only;
|
|
let DecoderNamespace = "GFX10";
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx6_gfx7<bits<8> op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx6_gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
|
|
let Inst{15} = ps.addr64;
|
|
let AssemblerPredicate = isGFX6GFX7;
|
|
let DecoderNamespace = "GFX6GFX7";
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx6<bits<8> op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx6 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
|
|
let Inst{15} = ps.addr64;
|
|
let AssemblerPredicate = isGFX6;
|
|
let DecoderNamespace = "GFX6";
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx7<bits<8> op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
|
|
let Inst{15} = ps.addr64;
|
|
let AssemblerPredicate = isGFX7Only;
|
|
let DecoderNamespace = "GFX7";
|
|
}
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Base ENC_VBUFFER for GFX12.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> :
|
|
InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 {
|
|
|
|
let isPseudo = 0;
|
|
let isCodeGenOnly = 0;
|
|
|
|
let VM_CNT = 1;
|
|
let EXP_CNT = 1;
|
|
|
|
// copy relevant pseudo op flags
|
|
let SubtargetPredicate = ps.SubtargetPredicate;
|
|
let AsmMatchConverter = ps.AsmMatchConverter;
|
|
let OtherPredicates = ps.OtherPredicates;
|
|
let Constraints = ps.Constraints;
|
|
let DisableEncoding = ps.DisableEncoding;
|
|
let TSFlags = ps.TSFlags;
|
|
let UseNamedOperandTable = ps.UseNamedOperandTable;
|
|
let SchedRW = ps.SchedRW;
|
|
let mayLoad = ps.mayLoad;
|
|
let mayStore = ps.mayStore;
|
|
let IsAtomicRet = ps.IsAtomicRet;
|
|
let IsAtomicNoRet = ps.IsAtomicNoRet;
|
|
let VALU = ps.VALU;
|
|
let LGKM_CNT = ps.LGKM_CNT;
|
|
let MUBUF = ps.MUBUF;
|
|
let MTBUF = ps.MTBUF;
|
|
|
|
bits<24> offset;
|
|
bits<8> vaddr;
|
|
bits<10> vdata;
|
|
|
|
bits<7> srsrc;
|
|
bits<7> soffset;
|
|
bits<6> cpol;
|
|
|
|
let Inst{95-72} = !if(ps.has_offset, offset, ?);
|
|
let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
|
|
let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);
|
|
let Inst{49-48} = 0b00;
|
|
let Inst{6-0} = !if(ps.has_soffset, soffset, ?);
|
|
let Inst{21-14} = op;
|
|
let Inst{22} = ps.tfe;
|
|
let Inst{62} = ps.offen;
|
|
let Inst{63} = ps.idxen;
|
|
|
|
let Inst{54-53} = cpol{2-1}; // th{2-1}
|
|
let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0}
|
|
let Inst{51-50} = cpol{4-3}; // scope
|
|
|
|
let Inst{31-26} = 0b110001;
|
|
}
|
|
|
|
class VBUFFER_Real_gfx12<bits<8> op, BUF_Pseudo ps, string real_name> :
|
|
VBUFFER_Real<op, ps, real_name>,
|
|
SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> {
|
|
let AssemblerPredicate = isGFX12Only;
|
|
let DecoderNamespace = "GFX12";
|
|
}
|
|
|
|
multiclass VBUFFER_MUBUF_Real_gfx12<bits<8> op, string real_name> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx12 : VBUFFER_Real_gfx12<op, ps, real_name> {
|
|
// Set the last bit of format to 1 to avoid round-trip issues, as some tools
|
|
// print BUF_FMT_INVALID for format 0.
|
|
let Inst{55} = 0b1;
|
|
}
|
|
}
|
|
|
|
multiclass VBUFFER_MTBUF_Real_gfx12<bits<4> op, string real_name> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> {
|
|
bits<7> format;
|
|
let Inst{61-55} = format;
|
|
}
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF - GFX11, GFX12.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Shortcut to default Mnemonic from BUF_Pseudo. Hides the cast to the
|
|
// specific pseudo (bothen in this case) since any of them will work.
|
|
class get_BUF_ps<string name> {
|
|
string Mnemonic = !cast<BUF_Pseudo>(name # "_BOTHEN").Mnemonic;
|
|
}
|
|
|
|
// gfx11 instruction that accept both old and new assembler name.
|
|
class Mnem_gfx11_gfx12 <string mnemonic, string real_name> :
|
|
MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>;
|
|
|
|
class Mnem_gfx11 <string mnemonic, string real_name> :
|
|
MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Only]>;
|
|
|
|
class Mnem_gfx12 <string mnemonic, string real_name> :
|
|
MnemonicAlias<mnemonic, real_name>, Requires<[isGFX12Plus]>;
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx11_Impl2<bits<8> op, string real_name> {
|
|
defm _BOTHEN : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _IDXEN : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFEN : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFSET : MUBUF_Real_gfx11<op, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx12_Impl2<bits<8> op, string real_name> {
|
|
defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<bits<8> op, string real_name> :
|
|
MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>,
|
|
MUBUF_Real_AllAddr_gfx12_Impl2<op, real_name>;
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, bit hasTFE,
|
|
string real_name = get_BUF_ps<NAME>.Mnemonic> {
|
|
defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>;
|
|
if hasTFE then
|
|
defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<8> op, bit hasTFE,
|
|
string real_name> {
|
|
defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>;
|
|
if hasTFE then
|
|
defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>;
|
|
}
|
|
|
|
// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
|
|
multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> :
|
|
MUBUF_Real_AllAddr_gfx11_Impl<op, hasTFE>;
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op,
|
|
string real_name = get_BUF_ps<NAME>.Mnemonic> :
|
|
MUBUF_Real_AllAddr_gfx11_gfx12_Impl<op, /*hasTFE=*/1, real_name> {
|
|
defvar ps = get_BUF_ps<NAME>;
|
|
if !ne(ps.Mnemonic, real_name) then
|
|
def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx11_impl<bits<8> op, bit is_return,
|
|
string real_name> {
|
|
defvar Rtn = !if(is_return, "_RTN", "");
|
|
defm _BOTHEN#Rtn : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _IDXEN#Rtn : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFEN#Rtn : MUBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFSET#Rtn : MUBUF_Real_gfx11<op, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx12_impl<bits<8> op, bit is_return,
|
|
string real_name = get_BUF_ps<NAME>.Mnemonic> {
|
|
defvar Rtn = !if(is_return, "_RTN", "");
|
|
defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_IDXEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl<bits<8> op, bit is_return,
|
|
string real_name> :
|
|
MUBUF_Real_Atomic_gfx11_impl<op, is_return, real_name>,
|
|
MUBUF_Real_Atomic_gfx12_impl<op, is_return, real_name>;
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> :
|
|
MUBUF_Real_Atomic_gfx12_impl<op, 0>,
|
|
MUBUF_Real_Atomic_gfx12_impl<op, 1>;
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx11<bits<8> op, string real_name> :
|
|
MUBUF_Real_Atomic_gfx11_impl<op, 0, real_name>,
|
|
MUBUF_Real_Atomic_gfx11_impl<op, 1, real_name> {
|
|
defvar ps = get_BUF_ps<NAME>;
|
|
def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op,
|
|
string gfx12_name = get_BUF_ps<NAME>.Mnemonic,
|
|
string gfx11_name = gfx12_name> :
|
|
MUBUF_Real_Atomic_gfx11_impl<op, 0, gfx11_name>,
|
|
MUBUF_Real_Atomic_gfx11_impl<op, 1, gfx11_name>,
|
|
MUBUF_Real_Atomic_gfx12_impl<op, 0, gfx12_name>,
|
|
MUBUF_Real_Atomic_gfx12_impl<op, 1, gfx12_name> {
|
|
defvar ps = get_BUF_ps<NAME>;
|
|
if !ne(ps.Mnemonic, gfx11_name) then
|
|
def : Mnem_gfx11<ps.Mnemonic, gfx11_name>;
|
|
if !ne(ps.Mnemonic, gfx12_name) then
|
|
def : Mnem_gfx12<ps.Mnemonic, gfx12_name>;
|
|
if !ne(gfx11_name, gfx12_name) then
|
|
def : Mnem_gfx12<gfx11_name, gfx12_name>;
|
|
}
|
|
|
|
defm BUFFER_GL0_INV : MUBUF_Real_gfx11<0x02B>;
|
|
defm BUFFER_GL1_INV : MUBUF_Real_gfx11<0x02C>;
|
|
|
|
defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">;
|
|
defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">;
|
|
defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">;
|
|
defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">;
|
|
defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">;
|
|
defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">;
|
|
defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">;
|
|
defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">;
|
|
defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">;
|
|
defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">;
|
|
defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">;
|
|
defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">;
|
|
defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>;
|
|
defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>;
|
|
defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>;
|
|
defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>;
|
|
defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">;
|
|
defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">;
|
|
defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">;
|
|
defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">;
|
|
defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>;
|
|
defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>;
|
|
defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>;
|
|
defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>;
|
|
defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>;
|
|
defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>;
|
|
defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">;
|
|
defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">;
|
|
defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">;
|
|
defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">;
|
|
defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">;
|
|
defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">;
|
|
defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">;
|
|
defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">;
|
|
defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">;
|
|
defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">;
|
|
defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">;
|
|
defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>;
|
|
defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>;
|
|
defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>;
|
|
defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>;
|
|
defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>;
|
|
defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">;
|
|
defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">;
|
|
defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">;
|
|
defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">;
|
|
defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">;
|
|
defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">;
|
|
defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">;
|
|
defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>;
|
|
defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
|
|
defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">;
|
|
defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">;
|
|
defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">;
|
|
defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">;
|
|
defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
|
|
defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">;
|
|
defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">;
|
|
defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">;
|
|
defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">;
|
|
defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
|
|
defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">;
|
|
defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">;
|
|
defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">;
|
|
defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">;
|
|
defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">;
|
|
defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">;
|
|
defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">;
|
|
defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">;
|
|
defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">;
|
|
defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">;
|
|
defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">;
|
|
defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">;
|
|
defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_gfx12<0x059>;
|
|
defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF - GFX10.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> {
|
|
defm _BOTHEN : MUBUF_Real_gfx10<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFSET : MUBUF_Real_gfx10<op>;
|
|
}
|
|
multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
|
|
defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>;
|
|
defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>;
|
|
}
|
|
multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> {
|
|
defm _OFFSET : MUBUF_Real_gfx10<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx10<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx10<op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx10<op>;
|
|
|
|
if !not(isTFE) then {
|
|
defm _LDS_OFFSET : MUBUF_Real_gfx10<op>;
|
|
defm _LDS_OFFEN : MUBUF_Real_gfx10<op>;
|
|
defm _LDS_IDXEN : MUBUF_Real_gfx10<op>;
|
|
defm _LDS_BOTHEN : MUBUF_Real_gfx10<op>;
|
|
}
|
|
}
|
|
multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> {
|
|
defm _BOTHEN_RTN : MUBUF_Real_gfx10<op>;
|
|
defm _IDXEN_RTN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFEN_RTN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFSET_RTN : MUBUF_Real_gfx10<op>;
|
|
}
|
|
multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
|
|
MUBUF_Real_Atomics_RTN_gfx10<op> {
|
|
defm _BOTHEN : MUBUF_Real_gfx10<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx10<op>;
|
|
defm _OFFSET : MUBUF_Real_gfx10<op>;
|
|
}
|
|
|
|
defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>;
|
|
defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>;
|
|
defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>;
|
|
defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>;
|
|
defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>;
|
|
defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>;
|
|
defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>;
|
|
defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>;
|
|
defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>;
|
|
defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
|
|
defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>;
|
|
defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>;
|
|
defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
|
|
|
|
defm BUFFER_GL0_INV : MUBUF_Real_gfx10<0x071>;
|
|
defm BUFFER_GL1_INV : MUBUF_Real_gfx10<0x072>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MUBUF - GFX6, GFX7, GFX10.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> {
|
|
defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>;
|
|
}
|
|
multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
|
|
defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>;
|
|
defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>;
|
|
}
|
|
multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> {
|
|
defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
|
|
if !not(isTFE) then {
|
|
defm _LDS_OFFSET : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _LDS_ADDR64 : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _LDS_OFFEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _LDS_IDXEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _LDS_BOTHEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
}
|
|
}
|
|
multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> {
|
|
defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>;
|
|
|
|
defm _ADDR64_RTN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _BOTHEN_RTN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _IDXEN_RTN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFEN_RTN : MUBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFSET_RTN : MUBUF_Real_gfx6_gfx7<op>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
|
|
MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
|
|
|
|
multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> :
|
|
MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>,
|
|
MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>;
|
|
|
|
multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> {
|
|
defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>;
|
|
defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
|
|
MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
|
|
|
|
// FIXME-GFX6: Following instructions are available only on GFX6.
|
|
//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>;
|
|
//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>;
|
|
|
|
defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
|
|
defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
|
|
defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
|
|
defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
|
|
defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
|
|
defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
|
|
defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
|
|
defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
|
|
defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
|
|
defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
|
|
defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
|
|
defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
|
|
defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
|
|
defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
|
|
defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
|
|
defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
|
|
defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
|
|
defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
|
|
defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
|
|
defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
|
|
defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
|
|
defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
|
|
|
|
defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
|
|
defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
|
|
defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
|
|
defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
|
|
defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
|
|
defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
|
|
defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
|
|
defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
|
|
defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
|
|
defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
|
|
defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
|
|
defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
|
|
defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
|
|
defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
|
|
defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
|
|
defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
|
|
defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
|
|
defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
|
|
defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
|
|
defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
|
|
defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
|
|
defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
|
|
defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
|
|
defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
|
|
defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
|
|
defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
|
|
defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
|
|
defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
|
|
defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
|
|
// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
|
|
defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
|
|
defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
|
|
defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
|
|
|
|
defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>;
|
|
|
|
defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>;
|
|
defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>;
|
|
defm BUFFER_WBINVL1 : MUBUF_Real_gfx6_gfx7<0x071>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef,
|
|
string real_name = ps.Mnemonic> :
|
|
MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{31-26} = 0x3a; //encoding
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
|
|
multiclass MTBUF_Real_gfx11<bits<4> op, string real_name> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _gfx11 : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> {
|
|
let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);
|
|
let Inst{18-15} = op;
|
|
let Inst{25-19} = format;
|
|
let Inst{53} = ps.tfe;
|
|
let Inst{54} = ps.offen;
|
|
let Inst{55} = ps.idxen;
|
|
let AssemblerPredicate = isGFX11Only;
|
|
let DecoderNamespace = "GFX11";
|
|
}
|
|
}
|
|
|
|
class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
|
|
Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> {
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{18-16} = op;
|
|
let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{55} = ps.tfe;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF - GFX11.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, string real_name> {
|
|
defm _BOTHEN : MTBUF_Real_gfx11<op, real_name>;
|
|
defm _IDXEN : MTBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFEN : MTBUF_Real_gfx11<op, real_name>;
|
|
defm _OFFSET : MTBUF_Real_gfx11<op, real_name>;
|
|
|
|
defm _VBUFFER_BOTHEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_IDXEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;
|
|
defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12<op, real_name>;
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op,
|
|
string real_name = get_BUF_ps<NAME>.Mnemonic>
|
|
: MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> {
|
|
defvar ps = get_BUF_ps<NAME>;
|
|
if !ne(ps.Mnemonic, real_name) then
|
|
def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;
|
|
}
|
|
|
|
defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">;
|
|
defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>;
|
|
defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>;
|
|
defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">;
|
|
defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>;
|
|
defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>;
|
|
defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>;
|
|
defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF - GFX10.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MTBUF_Real_gfx10<bits<4> op> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _gfx10 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
|
|
let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);
|
|
let Inst{25-19} = format;
|
|
let Inst{53} = op{3};
|
|
let AssemblerPredicate = isGFX10Only;
|
|
let DecoderNamespace = "GFX10";
|
|
}
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
|
|
defm _BOTHEN : MTBUF_Real_gfx10<op>;
|
|
defm _IDXEN : MTBUF_Real_gfx10<op>;
|
|
defm _OFFEN : MTBUF_Real_gfx10<op>;
|
|
defm _OFFSET : MTBUF_Real_gfx10<op>;
|
|
}
|
|
|
|
defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>;
|
|
defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF - GFX6, GFX7, GFX10.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
multiclass MTBUF_Real_gfx6_gfx7<bits<4> op> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _gfx6_gfx7 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
|
|
let Inst{15} = ps.addr64;
|
|
let Inst{22-19} = dfmt;
|
|
let Inst{25-23} = nfmt;
|
|
let AssemblerPredicate = isGFX6GFX7;
|
|
let DecoderNamespace = "GFX6GFX7";
|
|
}
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
|
|
defm _ADDR64 : MTBUF_Real_gfx6_gfx7<op>;
|
|
defm _BOTHEN : MTBUF_Real_gfx6_gfx7<op>;
|
|
defm _IDXEN : MTBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFEN : MTBUF_Real_gfx6_gfx7<op>;
|
|
defm _OFFSET : MTBUF_Real_gfx6_gfx7<op>;
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
|
|
MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
|
|
|
|
defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
|
|
defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
|
|
defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
|
|
defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
|
|
defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
|
|
defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// GFX8, GFX9 (VI).
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc,
|
|
bit has_sccb = ps.has_sccb> :
|
|
MUBUF_Real<ps>,
|
|
Enc64,
|
|
SIMCInstr<ps.PseudoInstr, Enc> {
|
|
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
|
|
let Inst{16} = ps.lds;
|
|
let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{24-18} = op;
|
|
let Inst{31-26} = 0x38; //encoding
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
|
|
multiclass MUBUF_Real_vi <bits<7> op,
|
|
bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _vi : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> {
|
|
let AssemblerPredicate = isGFX8GFX9NotGFX90A;
|
|
let DecoderNamespace = "GFX8";
|
|
|
|
let Inst{55} = ps.tfe;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx90a <bits<7> op,
|
|
bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx90a : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> {
|
|
let AssemblerPredicate = isGFX90APlus;
|
|
let DecoderNamespace = "GFX90A";
|
|
let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""),
|
|
ps.AsmOperands);
|
|
|
|
let Inst{55} = acc;
|
|
}
|
|
}
|
|
|
|
class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> :
|
|
MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> {
|
|
let AssemblerPredicate = isGFX940Plus;
|
|
let DecoderNamespace = "GFX9";
|
|
let AsmString = ps.Mnemonic # ps.AsmOperands;
|
|
|
|
let Inst{55} = acc;
|
|
}
|
|
|
|
multiclass MUBUF_Real_vi_gfx90a<bits<7> op, bit isTFE = 0> : MUBUF_Real_vi<op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
|
|
if !not(isTFE) then {
|
|
if !not(ps.FPAtomic) then
|
|
defm NAME : MUBUF_Real_gfx90a<op>;
|
|
}
|
|
|
|
if ps.FPAtomic then {
|
|
let SubtargetPredicate = isGFX90AOnly,
|
|
AssemblerPredicate = isGFX90AOnly in
|
|
defm NAME : MUBUF_Real_gfx90a<op, 0>;
|
|
def _gfx940 : MUBUF_Real_gfx940<op, ps>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> {
|
|
defm _OFFSET : MUBUF_Real_vi_gfx90a <op, isTFE>;
|
|
defm _OFFEN : MUBUF_Real_vi_gfx90a <op, isTFE>;
|
|
defm _IDXEN : MUBUF_Real_vi_gfx90a <op, isTFE>;
|
|
defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, isTFE>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> {
|
|
defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>;
|
|
if hasTFE then
|
|
defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> {
|
|
defm _OFFSET : MUBUF_Real_vi <op>;
|
|
defm _OFFEN : MUBUF_Real_vi <op>;
|
|
defm _IDXEN : MUBUF_Real_vi <op>;
|
|
defm _BOTHEN : MUBUF_Real_vi <op>;
|
|
|
|
if !not(isTFE) then {
|
|
defm _LDS_OFFSET : MUBUF_Real_vi <op>;
|
|
defm _LDS_OFFEN : MUBUF_Real_vi <op>;
|
|
defm _LDS_IDXEN : MUBUF_Real_vi <op>;
|
|
defm _LDS_BOTHEN : MUBUF_Real_vi <op>;
|
|
|
|
defm _OFFSET : MUBUF_Real_gfx90a <op>;
|
|
defm _OFFEN : MUBUF_Real_gfx90a <op>;
|
|
defm _IDXEN : MUBUF_Real_gfx90a <op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx90a <op>;
|
|
|
|
defm _LDS_OFFSET : MUBUF_Real_gfx90a <op>;
|
|
defm _LDS_OFFEN : MUBUF_Real_gfx90a <op>;
|
|
defm _LDS_IDXEN : MUBUF_Real_gfx90a <op>;
|
|
defm _LDS_BOTHEN : MUBUF_Real_gfx90a <op>;
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
|
|
defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>;
|
|
defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_gfx80 <bits<7> op> {
|
|
defvar ps = !cast<MUBUF_Pseudo>(NAME);
|
|
def _gfx80 : MUBUF_Real<ps>,
|
|
Enc64,
|
|
SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
|
|
let AssemblerPredicate=HasUnpackedD16VMem;
|
|
let DecoderNamespace="GFX80_UNPACKED";
|
|
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{16} = ps.lds;
|
|
let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{24-18} = op;
|
|
let Inst{31-26} = 0x38; //encoding
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{55} = ps.tfe;
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> {
|
|
defm _OFFSET : MUBUF_Real_gfx80 <op>;
|
|
defm _OFFEN : MUBUF_Real_gfx80 <op>;
|
|
defm _IDXEN : MUBUF_Real_gfx80 <op>;
|
|
defm _BOTHEN : MUBUF_Real_gfx80 <op>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
|
|
defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>;
|
|
defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>;
|
|
}
|
|
|
|
multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
|
|
MUBUF_Real_AllAddr_vi<op, 0> {
|
|
defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op>;
|
|
defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op>;
|
|
defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op>;
|
|
defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op>;
|
|
}
|
|
|
|
defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>;
|
|
defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>;
|
|
defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>;
|
|
defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>;
|
|
defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>;
|
|
defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>;
|
|
defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>;
|
|
defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>;
|
|
let SubtargetPredicate = HasUnpackedD16VMem in {
|
|
defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>;
|
|
defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>;
|
|
defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>;
|
|
} // End HasUnpackedD16VMem.
|
|
let SubtargetPredicate = HasPackedD16VMem in {
|
|
defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>;
|
|
defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>;
|
|
defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>;
|
|
defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>;
|
|
defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>;
|
|
} // End HasPackedD16VMem.
|
|
defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>;
|
|
defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>;
|
|
defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>;
|
|
defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>;
|
|
defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>;
|
|
defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>;
|
|
defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>;
|
|
defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>;
|
|
defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>;
|
|
defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>;
|
|
defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>;
|
|
defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>;
|
|
defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>;
|
|
defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>;
|
|
defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>;
|
|
defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>;
|
|
|
|
defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>;
|
|
defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>;
|
|
defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>;
|
|
defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>;
|
|
defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>;
|
|
defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>;
|
|
|
|
defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>;
|
|
defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
|
|
|
|
defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>;
|
|
defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>;
|
|
defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>;
|
|
defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>;
|
|
defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>;
|
|
defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>;
|
|
defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>;
|
|
defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>;
|
|
defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>;
|
|
defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>;
|
|
defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>;
|
|
defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>;
|
|
defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>;
|
|
|
|
defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>;
|
|
defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>;
|
|
defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>;
|
|
defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>;
|
|
defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>;
|
|
defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>;
|
|
defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>;
|
|
defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>;
|
|
defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>;
|
|
defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>;
|
|
defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>;
|
|
defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>;
|
|
defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>;
|
|
|
|
defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d>;
|
|
|
|
let AssemblerPredicate = isGFX8GFX9 in {
|
|
defm BUFFER_WBINVL1 : MUBUF_Real_vi <0x3e>;
|
|
defm BUFFER_WBINVL1_VOL : MUBUF_Real_vi <0x3f>;
|
|
} // End AssemblerPredicate = isGFX8GFX9
|
|
|
|
|
|
defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>;
|
|
|
|
let SubtargetPredicate = HasAtomicFaddNoRtnInsts in {
|
|
defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>;
|
|
} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts
|
|
|
|
let SubtargetPredicate = isGFX90APlus in {
|
|
defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>;
|
|
defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>;
|
|
defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>;
|
|
} // End SubtargetPredicate = isGFX90APlus
|
|
|
|
let AsmString = BUFFER_WBL2.Mnemonic, // drop flags
|
|
AssemblerPredicate = isGFX90AOnly,
|
|
SubtargetPredicate = isGFX90AOnly in
|
|
defm BUFFER_WBL2 : MUBUF_Real_gfx90a<0x28>;
|
|
defm BUFFER_INVL2 : MUBUF_Real_gfx90a<0x29>;
|
|
|
|
let SubtargetPredicate = isGFX940Plus in {
|
|
def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>;
|
|
def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>;
|
|
}
|
|
|
|
class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> :
|
|
MTBUF_Real<ps>,
|
|
Enc64,
|
|
SIMCInstr<ps.PseudoInstr, Enc> {
|
|
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{18-15} = op;
|
|
let Inst{22-19} = dfmt;
|
|
let Inst{25-23} = nfmt;
|
|
let Inst{31-26} = 0x3a; //encoding
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
|
|
let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{55} = ps.tfe;
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
|
|
class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
|
|
MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> {
|
|
let AssemblerPredicate = isGFX8GFX9NotGFX90A;
|
|
let DecoderNamespace = "GFX8";
|
|
|
|
let Inst{55} = ps.tfe;
|
|
}
|
|
|
|
class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> :
|
|
MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> {
|
|
let AssemblerPredicate = isGFX90APlus;
|
|
let DecoderNamespace = "GFX90A";
|
|
let AsmString = ps.Mnemonic # ps.AsmOperands;
|
|
|
|
let Inst{55} = acc;
|
|
}
|
|
|
|
multiclass MTBUF_Real_vi_gfx90a<bits<4> op> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _vi : MTBUF_Real_vi<op, ps>;
|
|
def _gfx90a : MTBUF_Real_gfx90a<op, ps>;
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
|
|
defm _OFFSET : MTBUF_Real_vi_gfx90a <op>;
|
|
defm _OFFEN : MTBUF_Real_vi_gfx90a <op>;
|
|
defm _IDXEN : MTBUF_Real_vi_gfx90a <op>;
|
|
defm _BOTHEN : MTBUF_Real_vi_gfx90a <op>;
|
|
}
|
|
|
|
multiclass MTBUF_Real_gfx80 <bits<4> op> {
|
|
defvar ps = !cast<MTBUF_Pseudo>(NAME);
|
|
def _gfx80 : MTBUF_Real<ps>,
|
|
Enc64,
|
|
SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
|
|
let AssemblerPredicate=HasUnpackedD16VMem;
|
|
let DecoderNamespace="GFX80_UNPACKED";
|
|
|
|
let Inst{11-0} = !if(ps.has_offset, offset, ?);
|
|
let Inst{12} = ps.offen;
|
|
let Inst{13} = ps.idxen;
|
|
let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
|
|
let Inst{18-15} = op;
|
|
let Inst{22-19} = dfmt;
|
|
let Inst{25-23} = nfmt;
|
|
let Inst{31-26} = 0x3a; //encoding
|
|
let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
|
|
let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
|
|
let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
|
|
let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
|
|
let Inst{55} = ps.tfe;
|
|
let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
|
|
}
|
|
}
|
|
|
|
multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
|
|
defm _OFFSET : MTBUF_Real_gfx80 <op>;
|
|
defm _OFFEN : MTBUF_Real_gfx80 <op>;
|
|
defm _IDXEN : MTBUF_Real_gfx80 <op>;
|
|
defm _BOTHEN : MTBUF_Real_gfx80 <op>;
|
|
}
|
|
|
|
defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>;
|
|
defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>;
|
|
defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>;
|
|
defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>;
|
|
defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>;
|
|
defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>;
|
|
defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
|
|
let SubtargetPredicate = HasUnpackedD16VMem in {
|
|
defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>;
|
|
defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
|
|
} // End HasUnpackedD16VMem.
|
|
let SubtargetPredicate = HasPackedD16VMem in {
|
|
defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>;
|
|
defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>;
|
|
defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>;
|
|
defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
|
|
} // End HasUnpackedD16VMem.
|
|
|
|
def MUBUFInfoTable : GenericTable {
|
|
let FilterClass = "MUBUF_Pseudo";
|
|
let CppTypeName = "MUBUFInfo";
|
|
let Fields = [
|
|
"Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset",
|
|
"IsBufferInv", "tfe"
|
|
];
|
|
|
|
let PrimaryKey = ["Opcode"];
|
|
let PrimaryKeyName = "getMUBUFOpcodeHelper";
|
|
}
|
|
|
|
def getMUBUFInfoFromOpcode : SearchIndex {
|
|
let Table = MUBUFInfoTable;
|
|
let Key = ["Opcode"];
|
|
}
|
|
|
|
def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
|
|
let Table = MUBUFInfoTable;
|
|
let Key = ["BaseOpcode", "elements"];
|
|
}
|
|
|
|
def MTBUFInfoTable : GenericTable {
|
|
let FilterClass = "MTBUF_Pseudo";
|
|
let CppTypeName = "MTBUFInfo";
|
|
let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
|
|
|
|
let PrimaryKey = ["Opcode"];
|
|
let PrimaryKeyName = "getMTBUFOpcodeHelper";
|
|
}
|
|
|
|
def getMTBUFInfoFromOpcode : SearchIndex {
|
|
let Table = MTBUFInfoTable;
|
|
let Key = ["Opcode"];
|
|
}
|
|
|
|
def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
|
|
let Table = MTBUFInfoTable;
|
|
let Key = ["BaseOpcode", "elements"];
|
|
}
|