f7a46aa8fb
This patch implements shouldFoldSelectWithIdentityConstant for RISCV. It would try to generate vmerge after the binary instruction and let them folded to maksed instruction later. Reviewed By: craig.topper Differential Revision: https://reviews.llvm.org/D131551
762 lines
31 KiB
C++
762 lines
31 KiB
C++
//===-- RISCVISelLowering.h - RISCV DAG Lowering Interface ------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file defines the interfaces that RISCV uses to lower LLVM code into a
|
|
// selection DAG.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_LIB_TARGET_RISCV_RISCVISELLOWERING_H
|
|
#define LLVM_LIB_TARGET_RISCV_RISCVISELLOWERING_H
|
|
|
|
#include "RISCV.h"
|
|
#include "llvm/CodeGen/CallingConvLower.h"
|
|
#include "llvm/CodeGen/SelectionDAG.h"
|
|
#include "llvm/CodeGen/TargetLowering.h"
|
|
#include "llvm/Support/TargetParser.h"
|
|
#include <optional>
|
|
|
|
namespace llvm {
|
|
class RISCVSubtarget;
|
|
struct RISCVRegisterInfo;
|
|
namespace RISCVISD {
|
|
enum NodeType : unsigned {
|
|
FIRST_NUMBER = ISD::BUILTIN_OP_END,
|
|
RET_FLAG,
|
|
URET_FLAG,
|
|
SRET_FLAG,
|
|
MRET_FLAG,
|
|
CALL,
|
|
/// Select with condition operator - This selects between a true value and
|
|
/// a false value (ops #3 and #4) based on the boolean result of comparing
|
|
/// the lhs and rhs (ops #0 and #1) of a conditional expression with the
|
|
/// condition code in op #2, a XLenVT constant from the ISD::CondCode enum.
|
|
/// The lhs and rhs are XLenVT integers. The true and false values can be
|
|
/// integer or floating point.
|
|
SELECT_CC,
|
|
BR_CC,
|
|
BuildPairF64,
|
|
SplitF64,
|
|
TAIL,
|
|
|
|
// Add the Lo 12 bits from an address. Selected to ADDI.
|
|
ADD_LO,
|
|
// Get the Hi 20 bits from an address. Selected to LUI.
|
|
HI,
|
|
|
|
// Represents an AUIPC+ADDI pair. Selected to PseudoLLA.
|
|
LLA,
|
|
|
|
// Selected as PseudoAddTPRel. Used to emit a TP-relative relocation.
|
|
ADD_TPREL,
|
|
|
|
// Load address.
|
|
LA_TLS_GD,
|
|
|
|
// Multiply high for signedxunsigned.
|
|
MULHSU,
|
|
// RV64I shifts, directly matching the semantics of the named RISC-V
|
|
// instructions.
|
|
SLLW,
|
|
SRAW,
|
|
SRLW,
|
|
// 32-bit operations from RV64M that can't be simply matched with a pattern
|
|
// at instruction selection time. These have undefined behavior for division
|
|
// by 0 or overflow (divw) like their target independent counterparts.
|
|
DIVW,
|
|
DIVUW,
|
|
REMUW,
|
|
// RV64IB rotates, directly matching the semantics of the named RISC-V
|
|
// instructions.
|
|
ROLW,
|
|
RORW,
|
|
// RV64IZbb bit counting instructions directly matching the semantics of the
|
|
// named RISC-V instructions.
|
|
CLZW,
|
|
CTZW,
|
|
|
|
// RV64IZbb absolute value for i32. Expanded to (max (negw X), X) during isel.
|
|
ABSW,
|
|
|
|
// FPR<->GPR transfer operations when the FPR is smaller than XLEN, needed as
|
|
// XLEN is the only legal integer width.
|
|
//
|
|
// FMV_H_X matches the semantics of the FMV.H.X.
|
|
// FMV_X_ANYEXTH is similar to FMV.X.H but has an any-extended result.
|
|
// FMV_X_SIGNEXTH is similar to FMV.X.H and has a sign-extended result.
|
|
// FMV_W_X_RV64 matches the semantics of the FMV.W.X.
|
|
// FMV_X_ANYEXTW_RV64 is similar to FMV.X.W but has an any-extended result.
|
|
//
|
|
// This is a more convenient semantic for producing dagcombines that remove
|
|
// unnecessary GPR->FPR->GPR moves.
|
|
FMV_H_X,
|
|
FMV_X_ANYEXTH,
|
|
FMV_X_SIGNEXTH,
|
|
FMV_W_X_RV64,
|
|
FMV_X_ANYEXTW_RV64,
|
|
// FP to XLen int conversions. Corresponds to fcvt.l(u).s/d/h on RV64 and
|
|
// fcvt.w(u).s/d/h on RV32. Unlike FP_TO_S/UINT these saturate out of
|
|
// range inputs. These are used for FP_TO_S/UINT_SAT lowering. Rounding mode
|
|
// is passed as a TargetConstant operand using the RISCVFPRndMode enum.
|
|
FCVT_X,
|
|
FCVT_XU,
|
|
// FP to 32 bit int conversions for RV64. These are used to keep track of the
|
|
// result being sign extended to 64 bit. These saturate out of range inputs.
|
|
// Used for FP_TO_S/UINT and FP_TO_S/UINT_SAT lowering. Rounding mode
|
|
// is passed as a TargetConstant operand using the RISCVFPRndMode enum.
|
|
FCVT_W_RV64,
|
|
FCVT_WU_RV64,
|
|
|
|
// Rounds an FP value to its corresponding integer in the same FP format.
|
|
// First operand is the value to round, the second operand is the largest
|
|
// integer that can be represented exactly in the FP format. This will be
|
|
// expanded into multiple instructions and basic blocks with a custom
|
|
// inserter.
|
|
FROUND,
|
|
|
|
// READ_CYCLE_WIDE - A read of the 64-bit cycle CSR on a 32-bit target
|
|
// (returns (Lo, Hi)). It takes a chain operand.
|
|
READ_CYCLE_WIDE,
|
|
// brev8, orc.b, zip, and unzip from Zbb and Zbkb. All operands are i32 or
|
|
// XLenVT.
|
|
BREV8,
|
|
ORC_B,
|
|
ZIP,
|
|
UNZIP,
|
|
// Vector Extension
|
|
// VMV_V_X_VL matches the semantics of vmv.v.x but includes an extra operand
|
|
// for the VL value to be used for the operation. The first operand is
|
|
// passthru operand.
|
|
VMV_V_X_VL,
|
|
// VFMV_V_F_VL matches the semantics of vfmv.v.f but includes an extra operand
|
|
// for the VL value to be used for the operation. The first operand is
|
|
// passthru operand.
|
|
VFMV_V_F_VL,
|
|
// VMV_X_S matches the semantics of vmv.x.s. The result is always XLenVT sign
|
|
// extended from the vector element size.
|
|
VMV_X_S,
|
|
// VMV_S_X_VL matches the semantics of vmv.s.x. It carries a VL operand.
|
|
VMV_S_X_VL,
|
|
// VFMV_S_F_VL matches the semantics of vfmv.s.f. It carries a VL operand.
|
|
VFMV_S_F_VL,
|
|
// Splats an 64-bit value that has been split into two i32 parts. This is
|
|
// expanded late to two scalar stores and a stride 0 vector load.
|
|
// The first operand is passthru operand.
|
|
SPLAT_VECTOR_SPLIT_I64_VL,
|
|
// Read VLENB CSR
|
|
READ_VLENB,
|
|
// Truncates a RVV integer vector by one power-of-two. Carries both an extra
|
|
// mask and VL operand.
|
|
TRUNCATE_VECTOR_VL,
|
|
// Matches the semantics of vslideup/vslidedown. The first operand is the
|
|
// pass-thru operand, the second is the source vector, the third is the
|
|
// XLenVT index (either constant or non-constant), the fourth is the mask
|
|
// and the fifth the VL.
|
|
VSLIDEUP_VL,
|
|
VSLIDEDOWN_VL,
|
|
// Matches the semantics of vslide1up/slide1down. The first operand is
|
|
// passthru operand, the second is source vector, third is the XLenVT scalar
|
|
// value. The fourth and fifth operands are the mask and VL operands.
|
|
VSLIDE1UP_VL,
|
|
VSLIDE1DOWN_VL,
|
|
// Matches the semantics of the vid.v instruction, with a mask and VL
|
|
// operand.
|
|
VID_VL,
|
|
// Matches the semantics of the vfcnvt.rod function (Convert double-width
|
|
// float to single-width float, rounding towards odd). Takes a double-width
|
|
// float vector and produces a single-width float vector. Also has a mask and
|
|
// VL operand.
|
|
VFNCVT_ROD_VL,
|
|
// These nodes match the semantics of the corresponding RVV vector reduction
|
|
// instructions. They produce a vector result which is the reduction
|
|
// performed over the second vector operand plus the first element of the
|
|
// third vector operand. The first operand is the pass-thru operand. The
|
|
// second operand is an unconstrained vector type, and the result, first, and
|
|
// third operand's types are expected to be the corresponding full-width
|
|
// LMUL=1 type for the second operand:
|
|
// nxv8i8 = vecreduce_add nxv8i8, nxv32i8, nxv8i8
|
|
// nxv2i32 = vecreduce_add nxv2i32, nxv8i32, nxv2i32
|
|
// The different in types does introduce extra vsetvli instructions but
|
|
// similarly it reduces the number of registers consumed per reduction.
|
|
// Also has a mask and VL operand.
|
|
VECREDUCE_ADD_VL,
|
|
VECREDUCE_UMAX_VL,
|
|
VECREDUCE_SMAX_VL,
|
|
VECREDUCE_UMIN_VL,
|
|
VECREDUCE_SMIN_VL,
|
|
VECREDUCE_AND_VL,
|
|
VECREDUCE_OR_VL,
|
|
VECREDUCE_XOR_VL,
|
|
VECREDUCE_FADD_VL,
|
|
VECREDUCE_SEQ_FADD_VL,
|
|
VECREDUCE_FMIN_VL,
|
|
VECREDUCE_FMAX_VL,
|
|
|
|
// Vector binary ops with a merge as a third operand, a mask as a fourth
|
|
// operand, and VL as a fifth operand.
|
|
ADD_VL,
|
|
AND_VL,
|
|
MUL_VL,
|
|
OR_VL,
|
|
SDIV_VL,
|
|
SHL_VL,
|
|
SREM_VL,
|
|
SRA_VL,
|
|
SRL_VL,
|
|
SUB_VL,
|
|
UDIV_VL,
|
|
UREM_VL,
|
|
XOR_VL,
|
|
SMIN_VL,
|
|
SMAX_VL,
|
|
UMIN_VL,
|
|
UMAX_VL,
|
|
|
|
SADDSAT_VL,
|
|
UADDSAT_VL,
|
|
SSUBSAT_VL,
|
|
USUBSAT_VL,
|
|
|
|
MULHS_VL,
|
|
MULHU_VL,
|
|
FADD_VL,
|
|
FSUB_VL,
|
|
FMUL_VL,
|
|
FDIV_VL,
|
|
FMINNUM_VL,
|
|
FMAXNUM_VL,
|
|
|
|
// Vector unary ops with a mask as a second operand and VL as a third operand.
|
|
FNEG_VL,
|
|
FABS_VL,
|
|
FSQRT_VL,
|
|
FCOPYSIGN_VL, // Has a merge operand
|
|
VFCVT_RTZ_X_F_VL,
|
|
VFCVT_RTZ_XU_F_VL,
|
|
VFCVT_X_F_VL,
|
|
VFROUND_NOEXCEPT_VL,
|
|
VFCVT_RM_X_F_VL, // Has a rounding mode operand.
|
|
SINT_TO_FP_VL,
|
|
UINT_TO_FP_VL,
|
|
FP_ROUND_VL,
|
|
FP_EXTEND_VL,
|
|
|
|
// Vector FMA ops with a mask as a fourth operand and VL as a fifth operand.
|
|
VFMADD_VL,
|
|
VFNMADD_VL,
|
|
VFMSUB_VL,
|
|
VFNMSUB_VL,
|
|
|
|
// Widening instructions with a merge value a third operand, a mask as a
|
|
// fourth operand, and VL as a fifth operand.
|
|
VWMUL_VL,
|
|
VWMULU_VL,
|
|
VWMULSU_VL,
|
|
VWADD_VL,
|
|
VWADDU_VL,
|
|
VWSUB_VL,
|
|
VWSUBU_VL,
|
|
VWADD_W_VL,
|
|
VWADDU_W_VL,
|
|
VWSUB_W_VL,
|
|
VWSUBU_W_VL,
|
|
|
|
VNSRL_VL,
|
|
|
|
// Vector compare producing a mask. Fourth operand is input mask. Fifth
|
|
// operand is VL.
|
|
SETCC_VL,
|
|
|
|
// Vector select with an additional VL operand. This operation is unmasked.
|
|
VSELECT_VL,
|
|
// Vector select with operand #2 (the value when the condition is false) tied
|
|
// to the destination and an additional VL operand. This operation is
|
|
// unmasked.
|
|
VP_MERGE_VL,
|
|
|
|
// Mask binary operators.
|
|
VMAND_VL,
|
|
VMOR_VL,
|
|
VMXOR_VL,
|
|
|
|
// Set mask vector to all zeros or ones.
|
|
VMCLR_VL,
|
|
VMSET_VL,
|
|
|
|
// Matches the semantics of vrgather.vx and vrgather.vv with extra operands
|
|
// for passthru and VL. Operands are (src, index, mask, passthru, vl).
|
|
VRGATHER_VX_VL,
|
|
VRGATHER_VV_VL,
|
|
VRGATHEREI16_VV_VL,
|
|
|
|
// Vector sign/zero extend with additional mask & VL operands.
|
|
VSEXT_VL,
|
|
VZEXT_VL,
|
|
|
|
// vcpop.m with additional mask and VL operands.
|
|
VCPOP_VL,
|
|
|
|
// Reads value of CSR.
|
|
// The first operand is a chain pointer. The second specifies address of the
|
|
// required CSR. Two results are produced, the read value and the new chain
|
|
// pointer.
|
|
READ_CSR,
|
|
// Write value to CSR.
|
|
// The first operand is a chain pointer, the second specifies address of the
|
|
// required CSR and the third is the value to write. The result is the new
|
|
// chain pointer.
|
|
WRITE_CSR,
|
|
// Read and write value of CSR.
|
|
// The first operand is a chain pointer, the second specifies address of the
|
|
// required CSR and the third is the value to write. Two results are produced,
|
|
// the value read before the modification and the new chain pointer.
|
|
SWAP_CSR,
|
|
|
|
// FP to 32 bit int conversions for RV64. These are used to keep track of the
|
|
// result being sign extended to 64 bit. These saturate out of range inputs.
|
|
STRICT_FCVT_W_RV64 = ISD::FIRST_TARGET_STRICTFP_OPCODE,
|
|
STRICT_FCVT_WU_RV64,
|
|
|
|
// WARNING: Do not add anything in the end unless you want the node to
|
|
// have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
|
|
// opcodes will be thought as target memory ops!
|
|
|
|
// Load address.
|
|
LA = ISD::FIRST_TARGET_MEMORY_OPCODE,
|
|
LA_TLS_IE,
|
|
};
|
|
} // namespace RISCVISD
|
|
|
|
class RISCVTargetLowering : public TargetLowering {
|
|
const RISCVSubtarget &Subtarget;
|
|
|
|
public:
|
|
explicit RISCVTargetLowering(const TargetMachine &TM,
|
|
const RISCVSubtarget &STI);
|
|
|
|
const RISCVSubtarget &getSubtarget() const { return Subtarget; }
|
|
|
|
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
|
|
MachineFunction &MF,
|
|
unsigned Intrinsic) const override;
|
|
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
|
|
unsigned AS,
|
|
Instruction *I = nullptr) const override;
|
|
bool isLegalICmpImmediate(int64_t Imm) const override;
|
|
bool isLegalAddImmediate(int64_t Imm) const override;
|
|
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override;
|
|
bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
|
|
bool isZExtFree(SDValue Val, EVT VT2) const override;
|
|
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override;
|
|
bool signExtendConstant(const ConstantInt *CI) const override;
|
|
bool isCheapToSpeculateCttz(Type *Ty) const override;
|
|
bool isCheapToSpeculateCtlz(Type *Ty) const override;
|
|
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
|
|
bool hasAndNotCompare(SDValue Y) const override;
|
|
bool hasBitTest(SDValue X, SDValue Y) const override;
|
|
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
|
|
SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
|
|
unsigned OldShiftOpcode, unsigned NewShiftOpcode,
|
|
SelectionDAG &DAG) const override;
|
|
/// Return true if the (vector) instruction I will be lowered to an instruction
|
|
/// with a scalar splat operand for the given Operand number.
|
|
bool canSplatOperand(Instruction *I, int Operand) const;
|
|
/// Return true if a vector instruction will lower to a target instruction
|
|
/// able to splat the given operand.
|
|
bool canSplatOperand(unsigned Opcode, int Operand) const;
|
|
bool shouldSinkOperands(Instruction *I,
|
|
SmallVectorImpl<Use *> &Ops) const override;
|
|
bool shouldScalarizeBinop(SDValue VecOp) const override;
|
|
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
|
|
bool isFPImmLegal(const APFloat &Imm, EVT VT,
|
|
bool ForCodeSize) const override;
|
|
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
|
|
unsigned Index) const override;
|
|
|
|
bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
|
|
|
|
bool softPromoteHalfType() const override { return true; }
|
|
|
|
/// Return the register type for a given MVT, ensuring vectors are treated
|
|
/// as a series of gpr sized integers.
|
|
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
|
|
EVT VT) const override;
|
|
|
|
/// Return the number of registers for a given MVT, ensuring vectors are
|
|
/// treated as a series of gpr sized integers.
|
|
unsigned getNumRegistersForCallingConv(LLVMContext &Context,
|
|
CallingConv::ID CC,
|
|
EVT VT) const override;
|
|
|
|
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
|
|
EVT VT) const override;
|
|
|
|
/// Return true if the given shuffle mask can be codegen'd directly, or if it
|
|
/// should be stack expanded.
|
|
bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
|
|
|
|
bool hasBitPreservingFPLogic(EVT VT) const override;
|
|
bool
|
|
shouldExpandBuildVectorWithShuffles(EVT VT,
|
|
unsigned DefinedValues) const override;
|
|
|
|
// Provide custom lowering hooks for some operations.
|
|
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
|
|
void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
|
|
|
|
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
|
|
const APInt &DemandedElts,
|
|
TargetLoweringOpt &TLO) const override;
|
|
|
|
void computeKnownBitsForTargetNode(const SDValue Op,
|
|
KnownBits &Known,
|
|
const APInt &DemandedElts,
|
|
const SelectionDAG &DAG,
|
|
unsigned Depth) const override;
|
|
unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
|
|
const APInt &DemandedElts,
|
|
const SelectionDAG &DAG,
|
|
unsigned Depth) const override;
|
|
|
|
const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
|
|
|
|
// This method returns the name of a target specific DAG node.
|
|
const char *getTargetNodeName(unsigned Opcode) const override;
|
|
|
|
ConstraintType getConstraintType(StringRef Constraint) const override;
|
|
|
|
unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override;
|
|
|
|
std::pair<unsigned, const TargetRegisterClass *>
|
|
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
|
|
StringRef Constraint, MVT VT) const override;
|
|
|
|
void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
|
|
std::vector<SDValue> &Ops,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
MachineBasicBlock *
|
|
EmitInstrWithCustomInserter(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const override;
|
|
|
|
void AdjustInstrPostInstrSelection(MachineInstr &MI,
|
|
SDNode *Node) const override;
|
|
|
|
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
|
|
EVT VT) const override;
|
|
|
|
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
|
|
return VT.isScalarInteger();
|
|
}
|
|
bool convertSelectOfConstantsToMath(EVT VT) const override { return true; }
|
|
|
|
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
|
|
return isa<LoadInst>(I) || isa<StoreInst>(I);
|
|
}
|
|
Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
|
|
AtomicOrdering Ord) const override;
|
|
Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
|
|
AtomicOrdering Ord) const override;
|
|
|
|
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
|
|
EVT VT) const override;
|
|
|
|
ISD::NodeType getExtendForAtomicOps() const override {
|
|
return ISD::SIGN_EXTEND;
|
|
}
|
|
|
|
ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
|
|
return ISD::SIGN_EXTEND;
|
|
}
|
|
|
|
bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override {
|
|
if (DAG.getMachineFunction().getFunction().hasMinSize())
|
|
return false;
|
|
return true;
|
|
}
|
|
bool isDesirableToCommuteWithShift(const SDNode *N,
|
|
CombineLevel Level) const override;
|
|
|
|
/// If a physical register, this returns the register that receives the
|
|
/// exception address on entry to an EH pad.
|
|
Register
|
|
getExceptionPointerRegister(const Constant *PersonalityFn) const override;
|
|
|
|
/// If a physical register, this returns the register that receives the
|
|
/// exception typeid on entry to a landing pad.
|
|
Register
|
|
getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
|
|
|
|
bool shouldExtendTypeInLibCall(EVT Type) const override;
|
|
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
|
|
|
|
/// Returns the register with the specified architectural or ABI name. This
|
|
/// method is necessary to lower the llvm.read_register.* and
|
|
/// llvm.write_register.* intrinsics. Allocatable registers must be reserved
|
|
/// with the clang -ffixed-xX flag for access to be allowed.
|
|
Register getRegisterByName(const char *RegName, LLT VT,
|
|
const MachineFunction &MF) const override;
|
|
|
|
// Lower incoming arguments, copy physregs into vregs
|
|
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
|
|
bool IsVarArg,
|
|
const SmallVectorImpl<ISD::InputArg> &Ins,
|
|
const SDLoc &DL, SelectionDAG &DAG,
|
|
SmallVectorImpl<SDValue> &InVals) const override;
|
|
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
|
|
bool IsVarArg,
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
|
LLVMContext &Context) const override;
|
|
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
|
const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
|
|
SelectionDAG &DAG) const override;
|
|
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
|
|
SmallVectorImpl<SDValue> &InVals) const override;
|
|
|
|
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
|
|
Type *Ty) const override;
|
|
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
|
|
bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
|
|
bool shouldConsiderGEPOffsetSplit() const override { return true; }
|
|
|
|
bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
|
|
SDValue C) const override;
|
|
|
|
bool isMulAddWithConstProfitable(SDValue AddNode,
|
|
SDValue ConstNode) const override;
|
|
|
|
TargetLowering::AtomicExpansionKind
|
|
shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
|
|
Value *emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI,
|
|
Value *AlignedAddr, Value *Incr,
|
|
Value *Mask, Value *ShiftAmt,
|
|
AtomicOrdering Ord) const override;
|
|
TargetLowering::AtomicExpansionKind
|
|
shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override;
|
|
Value *emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder,
|
|
AtomicCmpXchgInst *CI,
|
|
Value *AlignedAddr, Value *CmpVal,
|
|
Value *NewVal, Value *Mask,
|
|
AtomicOrdering Ord) const override;
|
|
|
|
/// Returns true if the target allows unaligned memory accesses of the
|
|
/// specified type.
|
|
bool allowsMisalignedMemoryAccesses(
|
|
EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1),
|
|
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
|
|
unsigned *Fast = nullptr) const override;
|
|
|
|
bool splitValueIntoRegisterParts(
|
|
SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
|
|
unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC)
|
|
const override;
|
|
|
|
SDValue joinRegisterPartsIntoValue(
|
|
SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts,
|
|
unsigned NumParts, MVT PartVT, EVT ValueVT,
|
|
std::optional<CallingConv::ID> CC) const override;
|
|
|
|
static RISCVII::VLMUL getLMUL(MVT VT);
|
|
inline static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize,
|
|
unsigned MinSize) {
|
|
// Original equation:
|
|
// VLMAX = (VectorBits / EltSize) * LMUL
|
|
// where LMUL = MinSize / RISCV::RVVBitsPerBlock
|
|
// The following equations have been reordered to prevent loss of precision
|
|
// when calculating fractional LMUL.
|
|
return ((VectorBits / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
|
|
};
|
|
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul);
|
|
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index);
|
|
static unsigned getRegClassIDForVecVT(MVT VT);
|
|
static std::pair<unsigned, unsigned>
|
|
decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT,
|
|
unsigned InsertExtractIdx,
|
|
const RISCVRegisterInfo *TRI);
|
|
MVT getContainerForFixedLengthVector(MVT VT) const;
|
|
|
|
bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const override;
|
|
|
|
bool isLegalElementTypeForRVV(Type *ScalarTy) const;
|
|
|
|
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
|
|
|
|
unsigned getJumpTableEncoding() const override;
|
|
|
|
const MCExpr *LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
|
|
const MachineBasicBlock *MBB,
|
|
unsigned uid,
|
|
MCContext &Ctx) const override;
|
|
|
|
bool isVScaleKnownToBeAPowerOfTwo() const override;
|
|
|
|
bool isLegalScaleForGatherScatter(uint64_t Scale,
|
|
uint64_t ElemSize) const override {
|
|
// Scaled addressing not supported on indexed load/stores
|
|
return Scale == 1;
|
|
}
|
|
|
|
private:
|
|
/// RISCVCCAssignFn - This target-specific function extends the default
|
|
/// CCValAssign with additional information used to lower RISC-V calling
|
|
/// conventions.
|
|
typedef bool RISCVCCAssignFn(const DataLayout &DL, RISCVABI::ABI,
|
|
unsigned ValNo, MVT ValVT, MVT LocVT,
|
|
CCValAssign::LocInfo LocInfo,
|
|
ISD::ArgFlagsTy ArgFlags, CCState &State,
|
|
bool IsFixed, bool IsRet, Type *OrigTy,
|
|
const RISCVTargetLowering &TLI,
|
|
std::optional<unsigned> FirstMaskArgument);
|
|
|
|
void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
|
|
const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
|
|
RISCVCCAssignFn Fn) const;
|
|
void analyzeOutputArgs(MachineFunction &MF, CCState &CCInfo,
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
|
bool IsRet, CallLoweringInfo *CLI,
|
|
RISCVCCAssignFn Fn) const;
|
|
|
|
template <class NodeTy>
|
|
SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
|
|
SDValue getStaticTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG,
|
|
bool UseGOT) const;
|
|
SDValue getDynamicTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG) const;
|
|
|
|
SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, bool IsSRA) const;
|
|
SDValue lowerSPLAT_VECTOR_PARTS(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVectorMaskSplat(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
|
|
int64_t ExtTrueVal) const;
|
|
SDValue lowerVectorMaskTruncLike(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVectorTruncLike(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVectorFPExtendOrRoundLike(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVPREDUCE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVectorMaskVecReduction(SDValue Op, SelectionDAG &DAG,
|
|
bool IsVP) const;
|
|
SDValue lowerFPVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerSTEP_VECTOR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVECTOR_REVERSE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerABS(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerMaskedLoad(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerMaskedStore(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorFCOPYSIGNToRVV(SDValue Op,
|
|
SelectionDAG &DAG) const;
|
|
SDValue lowerMaskedGather(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerMaskedScatter(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorLoadToRVV(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorStoreToRVV(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorSetccToRVV(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorLogicOpToRVV(SDValue Op, SelectionDAG &DAG,
|
|
unsigned MaskOpc,
|
|
unsigned VecOpc) const;
|
|
SDValue lowerFixedLengthVectorShiftToRVV(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorSelectToRVV(SDValue Op,
|
|
SelectionDAG &DAG) const;
|
|
SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc,
|
|
bool HasMergeOp = false, bool HasMask = true) const;
|
|
SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc,
|
|
bool HasMergeOp = false) const;
|
|
SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, unsigned MaskOpc,
|
|
unsigned VecOpc) const;
|
|
SDValue lowerVPExtMaskOp(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVPSetCCMaskOp(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVPFPIntConvOp(SDValue Op, SelectionDAG &DAG,
|
|
unsigned RISCVISDOpc) const;
|
|
SDValue lowerVPStridedLoad(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerVPStridedStore(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFixedLengthVectorExtendToRVV(SDValue Op, SelectionDAG &DAG,
|
|
unsigned ExtendOpc) const;
|
|
SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
SDValue expandUnalignedRVVLoad(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue expandUnalignedRVVStore(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
bool isEligibleForTailCallOptimization(
|
|
CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
|
|
const SmallVector<CCValAssign, 16> &ArgLocs) const;
|
|
|
|
/// Generate error diagnostics if any register used by CC has been marked
|
|
/// reserved.
|
|
void validateCCReservedRegs(
|
|
const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
|
|
MachineFunction &MF) const;
|
|
|
|
bool useRVVForFixedLengthVectorVT(MVT VT) const;
|
|
|
|
MVT getVPExplicitVectorLengthTy() const override;
|
|
|
|
/// RVV code generation for fixed length vectors does not lower all
|
|
/// BUILD_VECTORs. This makes BUILD_VECTOR legalisation a source of stores to
|
|
/// merge. However, merging them creates a BUILD_VECTOR that is just as
|
|
/// illegal as the original, thus leading to an infinite legalisation loop.
|
|
/// NOTE: Once BUILD_VECTOR can be custom lowered for all legal vector types,
|
|
/// this override can be removed.
|
|
bool mergeStoresAfterLegalization(EVT VT) const override;
|
|
|
|
/// Disable normalizing
|
|
/// select(N0&N1, X, Y) => select(N0, select(N1, X, Y), Y) and
|
|
/// select(N0|N1, X, Y) => select(N0, select(N1, X, Y, Y))
|
|
/// RISCV doesn't have flags so it's better to perform the and/or in a GPR.
|
|
bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override {
|
|
return false;
|
|
};
|
|
};
|
|
namespace RISCVVIntrinsicsTable {
|
|
|
|
struct RISCVVIntrinsicInfo {
|
|
unsigned IntrinsicID;
|
|
uint8_t ScalarOperand;
|
|
uint8_t VLOperand;
|
|
bool hasScalarOperand() const {
|
|
// 0xF is not valid. See NoScalarOperand in IntrinsicsRISCV.td.
|
|
return ScalarOperand != 0xF;
|
|
}
|
|
bool hasVLOperand() const {
|
|
// 0x1F is not valid. See NoVLOperand in IntrinsicsRISCV.td.
|
|
return VLOperand != 0x1F;
|
|
}
|
|
};
|
|
|
|
using namespace RISCV;
|
|
|
|
#define GET_RISCVVIntrinsicsTable_DECL
|
|
#include "RISCVGenSearchableTables.inc"
|
|
|
|
} // end namespace RISCVVIntrinsicsTable
|
|
|
|
} // end namespace llvm
|
|
|
|
#endif
|