70ffcfe94a
An unordered_set include was removed from a header in 04ebd1907c
"[MC][NFC] Statically allocate storage for decoded pseudo probes and
function records (#102789)", but SPIRVEmitIntrinsics was getting the
definition from that transitive include. Fix the build by including the
header explicitly.
1764 lines
66 KiB
C++
1764 lines
66 KiB
C++
//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// The pass emits SPIRV intrinsics keeping essential high-level information for
|
|
// the translation of LLVM IR to SPIR-V.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "SPIRV.h"
|
|
#include "SPIRVBuiltins.h"
|
|
#include "SPIRVMetadata.h"
|
|
#include "SPIRVSubtarget.h"
|
|
#include "SPIRVTargetMachine.h"
|
|
#include "SPIRVUtils.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/InstIterator.h"
|
|
#include "llvm/IR/InstVisitor.h"
|
|
#include "llvm/IR/IntrinsicsSPIRV.h"
|
|
#include "llvm/IR/TypedPointerType.h"
|
|
|
|
#include <queue>
|
|
#include <unordered_set>
|
|
|
|
// This pass performs the following transformation on LLVM IR level required
|
|
// for the following translation to SPIR-V:
|
|
// - replaces direct usages of aggregate constants with target-specific
|
|
// intrinsics;
|
|
// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
|
|
// with a target-specific intrinsics;
|
|
// - emits intrinsics for the global variable initializers since IRTranslator
|
|
// doesn't handle them and it's not very convenient to translate them
|
|
// ourselves;
|
|
// - emits intrinsics to keep track of the string names assigned to the values;
|
|
// - emits intrinsics to keep track of constants (this is necessary to have an
|
|
// LLVM IR constant after the IRTranslation is completed) for their further
|
|
// deduplication;
|
|
// - emits intrinsics to keep track of original LLVM types of the values
|
|
// to be able to emit proper SPIR-V types eventually.
|
|
//
|
|
// TODO: consider removing spv.track.constant in favor of spv.assign.type.
|
|
|
|
using namespace llvm;
|
|
|
|
namespace llvm {
|
|
namespace SPIRV {
|
|
#define GET_BuiltinGroup_DECL
|
|
#include "SPIRVGenTables.inc"
|
|
} // namespace SPIRV
|
|
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
|
|
} // namespace llvm
|
|
|
|
namespace {
|
|
|
|
inline MetadataAsValue *buildMD(Value *Arg) {
|
|
LLVMContext &Ctx = Arg->getContext();
|
|
return MetadataAsValue::get(
|
|
Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg)));
|
|
}
|
|
|
|
class SPIRVEmitIntrinsics
|
|
: public ModulePass,
|
|
public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
|
|
SPIRVTargetMachine *TM = nullptr;
|
|
SPIRVGlobalRegistry *GR = nullptr;
|
|
Function *F = nullptr;
|
|
bool TrackConstants = true;
|
|
DenseMap<Instruction *, Constant *> AggrConsts;
|
|
DenseMap<Instruction *, Type *> AggrConstTypes;
|
|
DenseSet<Instruction *> AggrStores;
|
|
SPIRV::InstructionSet::InstructionSet InstrSet;
|
|
|
|
// a register of Instructions that don't have a complete type definition
|
|
SmallPtrSet<Value *, 8> UncompleteTypeInfo;
|
|
SmallVector<Instruction *> PostprocessWorklist;
|
|
|
|
// well known result types of builtins
|
|
enum WellKnownTypes { Event };
|
|
|
|
// deduce element type of untyped pointers
|
|
Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
|
|
Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
|
|
Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8);
|
|
Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
|
|
bool UnknownElemTypeI8);
|
|
Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
|
|
std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8);
|
|
Type *deduceElementTypeByUsersDeep(Value *Op,
|
|
std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8);
|
|
void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
|
|
bool UnknownElemTypeI8);
|
|
|
|
// deduce nested types of composites
|
|
Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
|
|
Type *deduceNestedTypeHelper(User *U, Type *Ty,
|
|
std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8);
|
|
|
|
// deduce Types of operands of the Instruction if possible
|
|
void deduceOperandElementType(Instruction *I, Instruction *AskOp = 0,
|
|
Type *AskTy = 0, CallInst *AssignCI = 0);
|
|
|
|
void preprocessCompositeConstants(IRBuilder<> &B);
|
|
void preprocessUndefs(IRBuilder<> &B);
|
|
|
|
CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
|
|
Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms,
|
|
IRBuilder<> &B) {
|
|
SmallVector<Value *, 4> Args;
|
|
Args.push_back(Arg2);
|
|
Args.push_back(buildMD(Arg));
|
|
for (auto *Imm : Imms)
|
|
Args.push_back(Imm);
|
|
return B.CreateIntrinsic(IntrID, {Types}, Args);
|
|
}
|
|
|
|
void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg);
|
|
void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg);
|
|
void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType);
|
|
|
|
void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
|
|
void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
|
|
bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
|
|
bool UnknownElemTypeI8);
|
|
void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
|
|
void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
|
|
IRBuilder<> &B);
|
|
void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
|
|
Type *ExpectedElementType,
|
|
unsigned OperandToReplace,
|
|
IRBuilder<> &B);
|
|
void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
|
|
void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
|
|
void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
|
|
void processParamTypes(Function *F, IRBuilder<> &B);
|
|
void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
|
|
Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
|
|
Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
|
|
std::unordered_set<Function *> &FVisited);
|
|
|
|
public:
|
|
static char ID;
|
|
SPIRVEmitIntrinsics() : ModulePass(ID) {
|
|
initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
|
|
}
|
|
SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : ModulePass(ID), TM(_TM) {
|
|
initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
|
|
}
|
|
Instruction *visitInstruction(Instruction &I) { return &I; }
|
|
Instruction *visitSwitchInst(SwitchInst &I);
|
|
Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
|
|
Instruction *visitBitCastInst(BitCastInst &I);
|
|
Instruction *visitInsertElementInst(InsertElementInst &I);
|
|
Instruction *visitExtractElementInst(ExtractElementInst &I);
|
|
Instruction *visitInsertValueInst(InsertValueInst &I);
|
|
Instruction *visitExtractValueInst(ExtractValueInst &I);
|
|
Instruction *visitLoadInst(LoadInst &I);
|
|
Instruction *visitStoreInst(StoreInst &I);
|
|
Instruction *visitAllocaInst(AllocaInst &I);
|
|
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
|
|
Instruction *visitUnreachableInst(UnreachableInst &I);
|
|
Instruction *visitCallInst(CallInst &I);
|
|
|
|
StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
|
|
|
|
bool runOnModule(Module &M) override;
|
|
bool runOnFunction(Function &F);
|
|
bool postprocessTypes();
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
ModulePass::getAnalysisUsage(AU);
|
|
}
|
|
};
|
|
|
|
bool isConvergenceIntrinsic(const Instruction *I) {
|
|
const auto *II = dyn_cast<IntrinsicInst>(I);
|
|
if (!II)
|
|
return false;
|
|
|
|
return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
|
|
II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
|
|
II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
|
|
}
|
|
} // namespace
|
|
|
|
char SPIRVEmitIntrinsics::ID = 0;
|
|
|
|
INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
|
|
false, false)
|
|
|
|
static inline bool isAssignTypeInstr(const Instruction *I) {
|
|
return isa<IntrinsicInst>(I) &&
|
|
cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
|
|
}
|
|
|
|
static bool isMemInstrToReplace(Instruction *I) {
|
|
return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
|
|
isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
|
|
}
|
|
|
|
static bool isAggrConstForceInt32(const Value *V) {
|
|
return isa<ConstantArray>(V) || isa<ConstantStruct>(V) ||
|
|
isa<ConstantDataArray>(V) ||
|
|
(isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
|
|
}
|
|
|
|
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
|
|
if (isa<PHINode>(I))
|
|
B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
|
|
else
|
|
B.SetInsertPoint(I);
|
|
}
|
|
|
|
static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
|
|
B.SetCurrentDebugLocation(I->getDebugLoc());
|
|
if (I->getType()->isVoidTy())
|
|
B.SetInsertPoint(I->getNextNode());
|
|
else
|
|
B.SetInsertPoint(*I->getInsertionPointAfterDef());
|
|
}
|
|
|
|
static bool requireAssignType(Instruction *I) {
|
|
IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
|
|
if (Intr) {
|
|
switch (Intr->getIntrinsicID()) {
|
|
case Intrinsic::invariant_start:
|
|
case Intrinsic::invariant_end:
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static inline void reportFatalOnTokenType(const Instruction *I) {
|
|
if (I->getType()->isTokenTy())
|
|
report_fatal_error("A token is encountered but SPIR-V without extensions "
|
|
"does not support token type",
|
|
false);
|
|
}
|
|
|
|
static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
|
|
return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
|
|
isPointerTy(SI->getValueOperand()->getType()) &&
|
|
isa<Argument>(SI->getValueOperand());
|
|
}
|
|
|
|
// Maybe restore original function return type.
|
|
static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
|
|
Type *Ty) {
|
|
CallInst *CI = dyn_cast<CallInst>(I);
|
|
if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
|
|
!CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
|
|
return Ty;
|
|
if (Type *OriginalTy = GR->findMutated(CI->getCalledFunction()))
|
|
return OriginalTy;
|
|
return Ty;
|
|
}
|
|
|
|
// Reconstruct type with nested element types according to deduced type info.
|
|
// Return nullptr if no detailed type info is available.
|
|
static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) {
|
|
Type *Ty = Op->getType();
|
|
if (!isUntypedPointerTy(Ty))
|
|
return Ty;
|
|
// try to find the pointee type
|
|
if (Type *NestedTy = GR->findDeducedElementType(Op))
|
|
return getTypedPointerWrapper(NestedTy, getPointerAddressSpace(Ty));
|
|
// not a pointer according to the type info (e.g., Event object)
|
|
CallInst *CI = GR->findAssignPtrTypeInstr(Op);
|
|
if (!CI)
|
|
return nullptr;
|
|
MetadataAsValue *MD = cast<MetadataAsValue>(CI->getArgOperand(1));
|
|
return cast<ConstantAsMetadata>(MD->getMetadata())->getType();
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
|
|
Value *Arg) {
|
|
Value *OfType = PoisonValue::get(Ty);
|
|
CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
|
|
{Arg->getType()}, OfType, Arg, {}, B);
|
|
GR->addAssignPtrTypeInstr(Arg, AssignCI);
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,
|
|
Value *Arg) {
|
|
Value *OfType = PoisonValue::get(ElemTy);
|
|
CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg);
|
|
if (AssignPtrTyCI == nullptr ||
|
|
AssignPtrTyCI->getParent()->getParent() != F) {
|
|
AssignPtrTyCI = buildIntrWithMD(
|
|
Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,
|
|
{B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
|
|
GR->addDeducedElementType(AssignPtrTyCI, ElemTy);
|
|
GR->addDeducedElementType(Arg, ElemTy);
|
|
GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI);
|
|
} else {
|
|
updateAssignType(AssignPtrTyCI, Arg, OfType);
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg,
|
|
Value *OfType) {
|
|
AssignCI->setArgOperand(1, buildMD(OfType));
|
|
if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() !=
|
|
Intrinsic::spv_assign_ptr_type)
|
|
return;
|
|
|
|
// update association with the pointee type
|
|
Type *ElemTy = OfType->getType();
|
|
GR->addDeducedElementType(AssignCI, ElemTy);
|
|
GR->addDeducedElementType(Arg, ElemTy);
|
|
}
|
|
|
|
// Set element pointer type to the given value of ValueTy and tries to
|
|
// specify this type further (recursively) by Operand value, if needed.
|
|
Type *
|
|
SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
|
|
bool UnknownElemTypeI8) {
|
|
std::unordered_set<Value *> Visited;
|
|
return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
|
|
UnknownElemTypeI8);
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
|
|
Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8) {
|
|
Type *Ty = ValueTy;
|
|
if (Operand) {
|
|
if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
|
|
if (Type *NestedTy =
|
|
deduceElementTypeHelper(Operand, Visited, UnknownElemTypeI8))
|
|
Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
|
|
} else {
|
|
Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited,
|
|
UnknownElemTypeI8);
|
|
}
|
|
}
|
|
return Ty;
|
|
}
|
|
|
|
// Traverse User instructions to deduce an element pointer type of the operand.
|
|
Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
|
|
Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
|
|
if (!Op || !isPointerTy(Op->getType()))
|
|
return nullptr;
|
|
|
|
if (auto ElemTy = getPointeeType(Op->getType()))
|
|
return ElemTy;
|
|
|
|
// maybe we already know operand's element type
|
|
if (Type *KnownTy = GR->findDeducedElementType(Op))
|
|
return KnownTy;
|
|
|
|
for (User *OpU : Op->users()) {
|
|
if (Instruction *Inst = dyn_cast<Instruction>(OpU)) {
|
|
if (Type *Ty = deduceElementTypeHelper(Inst, Visited, UnknownElemTypeI8))
|
|
return Ty;
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
// Implements what we know in advance about intrinsics and builtin calls
|
|
// TODO: consider feasibility of this particular case to be generalized by
|
|
// encoding knowledge about intrinsics and builtin calls by corresponding
|
|
// specification rules
|
|
static Type *getPointeeTypeByCallInst(StringRef DemangledName,
|
|
Function *CalledF, unsigned OpIdx) {
|
|
if ((DemangledName.starts_with("__spirv_ocl_printf(") ||
|
|
DemangledName.starts_with("printf(")) &&
|
|
OpIdx == 0)
|
|
return IntegerType::getInt8Ty(CalledF->getContext());
|
|
return nullptr;
|
|
}
|
|
|
|
// Deduce and return a successfully deduced Type of the Instruction,
|
|
// or nullptr otherwise.
|
|
Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
|
|
bool UnknownElemTypeI8) {
|
|
std::unordered_set<Value *> Visited;
|
|
return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
|
|
bool UnknownElemTypeI8) {
|
|
if (isUntypedPointerTy(RefTy)) {
|
|
if (!UnknownElemTypeI8)
|
|
return;
|
|
if (auto *I = dyn_cast<Instruction>(Op)) {
|
|
UncompleteTypeInfo.insert(I);
|
|
PostprocessWorklist.push_back(I);
|
|
}
|
|
}
|
|
Ty = RefTy;
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
|
|
Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
|
|
// allow to pass nullptr as an argument
|
|
if (!I)
|
|
return nullptr;
|
|
|
|
// maybe already known
|
|
if (Type *KnownTy = GR->findDeducedElementType(I))
|
|
return KnownTy;
|
|
|
|
// maybe a cycle
|
|
if (Visited.find(I) != Visited.end())
|
|
return nullptr;
|
|
Visited.insert(I);
|
|
|
|
// fallback value in case when we fail to deduce a type
|
|
Type *Ty = nullptr;
|
|
// look for known basic patterns of type inference
|
|
if (auto *Ref = dyn_cast<AllocaInst>(I)) {
|
|
maybeAssignPtrType(Ty, I, Ref->getAllocatedType(), UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
|
|
Ty = Ref->getResultElementType();
|
|
} else if (auto *Ref = dyn_cast<GlobalValue>(I)) {
|
|
Ty = deduceElementTypeByValueDeep(
|
|
Ref->getValueType(),
|
|
Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr, Visited,
|
|
UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
|
|
Type *RefTy = deduceElementTypeHelper(Ref->getPointerOperand(), Visited,
|
|
UnknownElemTypeI8);
|
|
maybeAssignPtrType(Ty, I, RefTy, UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<BitCastInst>(I)) {
|
|
if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
|
|
isPointerTy(Src) && isPointerTy(Dest))
|
|
Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited,
|
|
UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
|
|
Value *Op = Ref->getNewValOperand();
|
|
if (isPointerTy(Op->getType()))
|
|
Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
|
|
Value *Op = Ref->getValOperand();
|
|
if (isPointerTy(Op->getType()))
|
|
Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8);
|
|
} else if (auto *Ref = dyn_cast<PHINode>(I)) {
|
|
for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
|
|
Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited,
|
|
UnknownElemTypeI8);
|
|
if (Ty)
|
|
break;
|
|
}
|
|
} else if (auto *Ref = dyn_cast<SelectInst>(I)) {
|
|
for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
|
|
Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
|
|
if (Ty)
|
|
break;
|
|
}
|
|
} else if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
static StringMap<unsigned> ResTypeByArg = {
|
|
{"to_global", 0},
|
|
{"to_local", 0},
|
|
{"to_private", 0},
|
|
{"__spirv_GenericCastToPtr_ToGlobal", 0},
|
|
{"__spirv_GenericCastToPtr_ToLocal", 0},
|
|
{"__spirv_GenericCastToPtr_ToPrivate", 0},
|
|
{"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
|
|
{"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
|
|
{"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
|
|
// TODO: maybe improve performance by caching demangled names
|
|
if (Function *CalledF = CI->getCalledFunction()) {
|
|
std::string DemangledName =
|
|
getOclOrSpirvBuiltinDemangledName(CalledF->getName());
|
|
if (DemangledName.length() > 0)
|
|
DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName);
|
|
auto AsArgIt = ResTypeByArg.find(DemangledName);
|
|
if (AsArgIt != ResTypeByArg.end()) {
|
|
Ty = deduceElementTypeHelper(CI->getArgOperand(AsArgIt->second),
|
|
Visited, UnknownElemTypeI8);
|
|
}
|
|
}
|
|
}
|
|
|
|
// remember the found relationship
|
|
if (Ty) {
|
|
// specify nested types if needed, otherwise return unchanged
|
|
GR->addDeducedElementType(I, Ty);
|
|
}
|
|
|
|
return Ty;
|
|
}
|
|
|
|
// Re-create a type of the value if it has untyped pointer fields, also nested.
|
|
// Return the original value type if no corrections of untyped pointer
|
|
// information is found or needed.
|
|
Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
|
|
bool UnknownElemTypeI8) {
|
|
std::unordered_set<Value *> Visited;
|
|
return deduceNestedTypeHelper(U, U->getType(), Visited, UnknownElemTypeI8);
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
|
|
User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
|
|
bool UnknownElemTypeI8) {
|
|
if (!U)
|
|
return OrigTy;
|
|
|
|
// maybe already known
|
|
if (Type *KnownTy = GR->findDeducedCompositeType(U))
|
|
return KnownTy;
|
|
|
|
// maybe a cycle
|
|
if (Visited.find(U) != Visited.end())
|
|
return OrigTy;
|
|
Visited.insert(U);
|
|
|
|
if (dyn_cast<StructType>(OrigTy)) {
|
|
SmallVector<Type *> Tys;
|
|
bool Change = false;
|
|
for (unsigned i = 0; i < U->getNumOperands(); ++i) {
|
|
Value *Op = U->getOperand(i);
|
|
Type *OpTy = Op->getType();
|
|
Type *Ty = OpTy;
|
|
if (Op) {
|
|
if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
|
|
if (Type *NestedTy =
|
|
deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
|
|
Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
|
|
} else {
|
|
Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
|
|
UnknownElemTypeI8);
|
|
}
|
|
}
|
|
Tys.push_back(Ty);
|
|
Change |= Ty != OpTy;
|
|
}
|
|
if (Change) {
|
|
Type *NewTy = StructType::create(Tys);
|
|
GR->addDeducedCompositeType(U, NewTy);
|
|
return NewTy;
|
|
}
|
|
} else if (auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) {
|
|
if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) {
|
|
Type *OpTy = ArrTy->getElementType();
|
|
Type *Ty = OpTy;
|
|
if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
|
|
if (Type *NestedTy =
|
|
deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
|
|
Ty = TypedPointerType::get(NestedTy, PtrTy->getAddressSpace());
|
|
} else {
|
|
Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
|
|
UnknownElemTypeI8);
|
|
}
|
|
if (Ty != OpTy) {
|
|
Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
|
|
GR->addDeducedCompositeType(U, NewTy);
|
|
return NewTy;
|
|
}
|
|
}
|
|
} else if (auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
|
|
if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) {
|
|
Type *OpTy = VecTy->getElementType();
|
|
Type *Ty = OpTy;
|
|
if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
|
|
if (Type *NestedTy =
|
|
deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
|
|
Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
|
|
} else {
|
|
Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
|
|
UnknownElemTypeI8);
|
|
}
|
|
if (Ty != OpTy) {
|
|
Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
|
|
GR->addDeducedCompositeType(U, NewTy);
|
|
return NewTy;
|
|
}
|
|
}
|
|
}
|
|
|
|
return OrigTy;
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
|
|
if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
|
|
return Ty;
|
|
if (!UnknownElemTypeI8)
|
|
return nullptr;
|
|
if (auto *Instr = dyn_cast<Instruction>(I)) {
|
|
UncompleteTypeInfo.insert(Instr);
|
|
PostprocessWorklist.push_back(Instr);
|
|
}
|
|
return IntegerType::getInt8Ty(I->getContext());
|
|
}
|
|
|
|
static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
|
|
Value *PointerOperand) {
|
|
Type *PointeeTy = GR->findDeducedElementType(PointerOperand);
|
|
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
|
|
return nullptr;
|
|
auto *PtrTy = dyn_cast<PointerType>(I->getType());
|
|
if (!PtrTy)
|
|
return I->getType();
|
|
if (Type *NestedTy = GR->findDeducedElementType(I))
|
|
return getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
|
|
return nullptr;
|
|
}
|
|
|
|
// If the Instruction has Pointer operands with unresolved types, this function
|
|
// tries to deduce them. If the Instruction has Pointer operands with known
|
|
// types which differ from expected, this function tries to insert a bitcast to
|
|
// resolve the issue.
|
|
void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I,
|
|
Instruction *AskOp,
|
|
Type *AskTy,
|
|
CallInst *AskCI) {
|
|
SmallVector<std::pair<Value *, unsigned>> Ops;
|
|
Type *KnownElemTy = nullptr;
|
|
// look for known basic patterns of type inference
|
|
if (auto *Ref = dyn_cast<PHINode>(I)) {
|
|
if (!isPointerTy(I->getType()) ||
|
|
!(KnownElemTy = GR->findDeducedElementType(I)))
|
|
return;
|
|
for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
|
|
Value *Op = Ref->getIncomingValue(i);
|
|
if (isPointerTy(Op->getType()))
|
|
Ops.push_back(std::make_pair(Op, i));
|
|
}
|
|
} else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
|
|
KnownElemTy = GR->findDeducedElementType(I);
|
|
if (!KnownElemTy)
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(), 0));
|
|
} else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
|
|
KnownElemTy = Ref->getSourceElementType();
|
|
if (isUntypedPointerTy(KnownElemTy))
|
|
return;
|
|
Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
|
|
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
|
|
GetElementPtrInst::getPointerOperandIndex()));
|
|
} else if (auto *Ref = dyn_cast<LoadInst>(I)) {
|
|
KnownElemTy = I->getType();
|
|
if (isUntypedPointerTy(KnownElemTy))
|
|
return;
|
|
Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
|
|
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
|
|
LoadInst::getPointerOperandIndex()));
|
|
} else if (auto *Ref = dyn_cast<StoreInst>(I)) {
|
|
if (IsKernelArgInt8(Ref->getParent()->getParent(), Ref))
|
|
return;
|
|
if (!(KnownElemTy = reconstructType(GR, Ref->getValueOperand())))
|
|
return;
|
|
Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand());
|
|
if (PointeeTy && !isUntypedPointerTy(PointeeTy))
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
|
|
StoreInst::getPointerOperandIndex()));
|
|
} else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) {
|
|
KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
|
|
if (!KnownElemTy)
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
|
|
AtomicCmpXchgInst::getPointerOperandIndex()));
|
|
} else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) {
|
|
KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand());
|
|
if (!KnownElemTy)
|
|
return;
|
|
Ops.push_back(std::make_pair(Ref->getPointerOperand(),
|
|
AtomicRMWInst::getPointerOperandIndex()));
|
|
} else if (auto *Ref = dyn_cast<SelectInst>(I)) {
|
|
if (!isPointerTy(I->getType()) ||
|
|
!(KnownElemTy = GR->findDeducedElementType(I)))
|
|
return;
|
|
for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
|
|
Value *Op = Ref->getOperand(i);
|
|
if (isPointerTy(Op->getType()))
|
|
Ops.push_back(std::make_pair(Op, i));
|
|
}
|
|
} else if (auto *Ref = dyn_cast<ReturnInst>(I)) {
|
|
Type *RetTy = F->getReturnType();
|
|
if (!isPointerTy(RetTy))
|
|
return;
|
|
Value *Op = Ref->getReturnValue();
|
|
if (!Op)
|
|
return;
|
|
if (!(KnownElemTy = GR->findDeducedElementType(F))) {
|
|
if (Type *OpElemTy = GR->findDeducedElementType(Op)) {
|
|
GR->addDeducedElementType(F, OpElemTy);
|
|
TypedPointerType *DerivedTy =
|
|
TypedPointerType::get(OpElemTy, getPointerAddressSpace(RetTy));
|
|
GR->addReturnType(F, DerivedTy);
|
|
}
|
|
return;
|
|
}
|
|
Ops.push_back(std::make_pair(Op, 0));
|
|
} else if (auto *Ref = dyn_cast<ICmpInst>(I)) {
|
|
if (!isPointerTy(Ref->getOperand(0)->getType()))
|
|
return;
|
|
Value *Op0 = Ref->getOperand(0);
|
|
Value *Op1 = Ref->getOperand(1);
|
|
Type *ElemTy0 = GR->findDeducedElementType(Op0);
|
|
Type *ElemTy1 = GR->findDeducedElementType(Op1);
|
|
if (ElemTy0) {
|
|
KnownElemTy = ElemTy0;
|
|
Ops.push_back(std::make_pair(Op1, 1));
|
|
} else if (ElemTy1) {
|
|
KnownElemTy = ElemTy1;
|
|
Ops.push_back(std::make_pair(Op0, 0));
|
|
}
|
|
} else if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
if (Function *CalledF = CI->getCalledFunction()) {
|
|
std::string DemangledName =
|
|
getOclOrSpirvBuiltinDemangledName(CalledF->getName());
|
|
if (DemangledName.length() > 0 &&
|
|
!StringRef(DemangledName).starts_with("llvm.")) {
|
|
auto [Grp, Opcode, ExtNo] =
|
|
SPIRV::mapBuiltinToOpcode(DemangledName, InstrSet);
|
|
if (Opcode == SPIRV::OpGroupAsyncCopy) {
|
|
for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2;
|
|
++i) {
|
|
Value *Op = CI->getArgOperand(i);
|
|
if (!isPointerTy(Op->getType()))
|
|
continue;
|
|
++PtrCnt;
|
|
if (Type *ElemTy = GR->findDeducedElementType(Op))
|
|
KnownElemTy = ElemTy; // src will rewrite dest if both are defined
|
|
Ops.push_back(std::make_pair(Op, i));
|
|
}
|
|
} else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
|
|
if (CI->arg_size() < 2)
|
|
return;
|
|
Value *Op = CI->getArgOperand(0);
|
|
if (!isPointerTy(Op->getType()))
|
|
return;
|
|
switch (Opcode) {
|
|
case SPIRV::OpAtomicLoad:
|
|
case SPIRV::OpAtomicCompareExchangeWeak:
|
|
case SPIRV::OpAtomicCompareExchange:
|
|
case SPIRV::OpAtomicExchange:
|
|
case SPIRV::OpAtomicIAdd:
|
|
case SPIRV::OpAtomicISub:
|
|
case SPIRV::OpAtomicOr:
|
|
case SPIRV::OpAtomicXor:
|
|
case SPIRV::OpAtomicAnd:
|
|
case SPIRV::OpAtomicUMin:
|
|
case SPIRV::OpAtomicUMax:
|
|
case SPIRV::OpAtomicSMin:
|
|
case SPIRV::OpAtomicSMax: {
|
|
KnownElemTy = getAtomicElemTy(GR, I, Op);
|
|
if (!KnownElemTy)
|
|
return;
|
|
Ops.push_back(std::make_pair(Op, 0));
|
|
} break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// There is no enough info to deduce types or all is valid.
|
|
if (!KnownElemTy || Ops.size() == 0)
|
|
return;
|
|
|
|
LLVMContext &Ctx = F->getContext();
|
|
IRBuilder<> B(Ctx);
|
|
for (auto &OpIt : Ops) {
|
|
Value *Op = OpIt.first;
|
|
if (Op->use_empty() || (AskOp && Op != AskOp))
|
|
continue;
|
|
Type *Ty = AskOp ? AskTy : GR->findDeducedElementType(Op);
|
|
if (Ty == KnownElemTy)
|
|
continue;
|
|
Value *OpTyVal = PoisonValue::get(KnownElemTy);
|
|
Type *OpTy = Op->getType();
|
|
if (!Ty || AskTy || isUntypedPointerTy(Ty) ||
|
|
UncompleteTypeInfo.contains(Op)) {
|
|
GR->addDeducedElementType(Op, KnownElemTy);
|
|
// check if there is existing Intrinsic::spv_assign_ptr_type instruction
|
|
CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Op);
|
|
if (AssignCI == nullptr) {
|
|
Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get());
|
|
setInsertPointSkippingPhis(B, User ? User->getNextNode() : I);
|
|
CallInst *CI =
|
|
buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op,
|
|
{B.getInt32(getPointerAddressSpace(OpTy))}, B);
|
|
GR->addAssignPtrTypeInstr(Op, CI);
|
|
} else {
|
|
updateAssignType(AssignCI, Op, OpTyVal);
|
|
}
|
|
} else {
|
|
if (auto *OpI = dyn_cast<Instruction>(Op)) {
|
|
// spv_ptrcast's argument Op denotes an instruction that generates
|
|
// a value, and we may use getInsertionPointAfterDef()
|
|
B.SetInsertPoint(*OpI->getInsertionPointAfterDef());
|
|
B.SetCurrentDebugLocation(OpI->getDebugLoc());
|
|
} else if (auto *OpA = dyn_cast<Argument>(Op)) {
|
|
B.SetInsertPointPastAllocas(OpA->getParent());
|
|
B.SetCurrentDebugLocation(DebugLoc());
|
|
} else {
|
|
B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
|
|
}
|
|
SmallVector<Type *, 2> Types = {OpTy, OpTy};
|
|
SmallVector<Value *, 2> Args = {Op, buildMD(OpTyVal),
|
|
B.getInt32(getPointerAddressSpace(OpTy))};
|
|
CallInst *PtrCastI =
|
|
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
|
|
I->setOperand(OpIt.second, PtrCastI);
|
|
}
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
|
|
Instruction *New,
|
|
IRBuilder<> &B) {
|
|
while (!Old->user_empty()) {
|
|
auto *U = Old->user_back();
|
|
if (isAssignTypeInstr(U)) {
|
|
B.SetInsertPoint(U);
|
|
SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
|
|
CallInst *AssignCI =
|
|
B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
|
|
GR->addAssignPtrTypeInstr(New, AssignCI);
|
|
U->eraseFromParent();
|
|
} else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
|
|
isa<CallInst>(U)) {
|
|
U->replaceUsesOfWith(Old, New);
|
|
} else {
|
|
llvm_unreachable("illegal aggregate intrinsic user");
|
|
}
|
|
}
|
|
Old->eraseFromParent();
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
|
|
std::queue<Instruction *> Worklist;
|
|
for (auto &I : instructions(F))
|
|
Worklist.push(&I);
|
|
|
|
while (!Worklist.empty()) {
|
|
Instruction *I = Worklist.front();
|
|
bool BPrepared = false;
|
|
Worklist.pop();
|
|
|
|
for (auto &Op : I->operands()) {
|
|
auto *AggrUndef = dyn_cast<UndefValue>(Op);
|
|
if (!AggrUndef || !Op->getType()->isAggregateType())
|
|
continue;
|
|
|
|
if (!BPrepared) {
|
|
setInsertPointSkippingPhis(B, I);
|
|
BPrepared = true;
|
|
}
|
|
auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {}, {});
|
|
Worklist.push(IntrUndef);
|
|
I->replaceUsesOfWith(Op, IntrUndef);
|
|
AggrConsts[IntrUndef] = AggrUndef;
|
|
AggrConstTypes[IntrUndef] = AggrUndef->getType();
|
|
}
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
|
|
std::queue<Instruction *> Worklist;
|
|
for (auto &I : instructions(F))
|
|
Worklist.push(&I);
|
|
|
|
while (!Worklist.empty()) {
|
|
auto *I = Worklist.front();
|
|
bool IsPhi = isa<PHINode>(I), BPrepared = false;
|
|
assert(I);
|
|
bool KeepInst = false;
|
|
for (const auto &Op : I->operands()) {
|
|
Constant *AggrConst = nullptr;
|
|
Type *ResTy = nullptr;
|
|
if (auto *COp = dyn_cast<ConstantVector>(Op)) {
|
|
AggrConst = cast<Constant>(COp);
|
|
ResTy = COp->getType();
|
|
} else if (auto *COp = dyn_cast<ConstantArray>(Op)) {
|
|
AggrConst = cast<Constant>(COp);
|
|
ResTy = B.getInt32Ty();
|
|
} else if (auto *COp = dyn_cast<ConstantStruct>(Op)) {
|
|
AggrConst = cast<Constant>(COp);
|
|
ResTy = B.getInt32Ty();
|
|
} else if (auto *COp = dyn_cast<ConstantDataArray>(Op)) {
|
|
AggrConst = cast<Constant>(COp);
|
|
ResTy = B.getInt32Ty();
|
|
} else if (auto *COp = dyn_cast<ConstantAggregateZero>(Op)) {
|
|
AggrConst = cast<Constant>(COp);
|
|
ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
|
|
}
|
|
if (AggrConst) {
|
|
SmallVector<Value *> Args;
|
|
if (auto *COp = dyn_cast<ConstantDataSequential>(Op))
|
|
for (unsigned i = 0; i < COp->getNumElements(); ++i)
|
|
Args.push_back(COp->getElementAsConstant(i));
|
|
else
|
|
for (auto &COp : AggrConst->operands())
|
|
Args.push_back(COp);
|
|
if (!BPrepared) {
|
|
IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
|
|
: B.SetInsertPoint(I);
|
|
BPrepared = true;
|
|
}
|
|
auto *CI =
|
|
B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args});
|
|
Worklist.push(CI);
|
|
I->replaceUsesOfWith(Op, CI);
|
|
KeepInst = true;
|
|
AggrConsts[CI] = AggrConst;
|
|
AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst, false);
|
|
}
|
|
}
|
|
if (!KeepInst)
|
|
Worklist.pop();
|
|
}
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
|
|
if (!Call.isInlineAsm())
|
|
return &Call;
|
|
|
|
const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
|
|
LLVMContext &Ctx = F->getContext();
|
|
|
|
Constant *TyC = UndefValue::get(IA->getFunctionType());
|
|
MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString());
|
|
SmallVector<Value *> Args = {
|
|
buildMD(TyC),
|
|
MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))};
|
|
for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
|
|
Args.push_back(Call.getArgOperand(OpIdx));
|
|
|
|
IRBuilder<> B(Call.getParent());
|
|
B.SetInsertPoint(&Call);
|
|
B.CreateIntrinsic(Intrinsic::spv_inline_asm, {}, {Args});
|
|
return &Call;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
|
|
BasicBlock *ParentBB = I.getParent();
|
|
IRBuilder<> B(ParentBB);
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Value *, 4> Args;
|
|
SmallVector<BasicBlock *> BBCases;
|
|
for (auto &Op : I.operands()) {
|
|
if (Op.get()->getType()->isSized()) {
|
|
Args.push_back(Op);
|
|
} else if (BasicBlock *BB = dyn_cast<BasicBlock>(Op.get())) {
|
|
BBCases.push_back(BB);
|
|
Args.push_back(BlockAddress::get(BB->getParent(), BB));
|
|
} else {
|
|
report_fatal_error("Unexpected switch operand");
|
|
}
|
|
}
|
|
CallInst *NewI = B.CreateIntrinsic(Intrinsic::spv_switch,
|
|
{I.getOperand(0)->getType()}, {Args});
|
|
// remove switch to avoid its unneeded and undesirable unwrap into branches
|
|
// and conditions
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
// insert artificial and temporary instruction to preserve valid CFG,
|
|
// it will be removed after IR translation pass
|
|
B.SetInsertPoint(ParentBB);
|
|
IndirectBrInst *BrI = B.CreateIndirectBr(
|
|
Constant::getNullValue(PointerType::getUnqual(ParentBB->getContext())),
|
|
BBCases.size());
|
|
for (BasicBlock *BBCase : BBCases)
|
|
BrI->addDestination(BBCase);
|
|
return BrI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
|
|
SmallVector<Value *, 4> Args;
|
|
Args.push_back(B.getInt1(I.isInBounds()));
|
|
for (auto &Op : I.operands())
|
|
Args.push_back(Op);
|
|
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
Value *Source = I.getOperand(0);
|
|
|
|
// SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
|
|
// varying element types. In case of IR coming from older versions of LLVM
|
|
// such bitcasts do not provide sufficient information, should be just skipped
|
|
// here, and handled in insertPtrCastOrAssignTypeInstr.
|
|
if (isPointerTy(I.getType())) {
|
|
I.replaceAllUsesWith(Source);
|
|
I.eraseFromParent();
|
|
return nullptr;
|
|
}
|
|
|
|
SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
|
|
SmallVector<Value *> Args(I.op_begin(), I.op_end());
|
|
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
|
|
std::string InstName = I.hasName() ? I.getName().str() : "";
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
NewI->setName(InstName);
|
|
return NewI;
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
|
|
TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
|
|
Type *VTy = V->getType();
|
|
|
|
// A couple of sanity checks.
|
|
assert(isPointerTy(VTy) && "Expect a pointer type!");
|
|
if (auto PType = dyn_cast<TypedPointerType>(VTy))
|
|
if (PType->getElementType() != AssignedType)
|
|
report_fatal_error("Unexpected pointer element type!");
|
|
|
|
CallInst *AssignCI = GR->findAssignPtrTypeInstr(V);
|
|
if (!AssignCI) {
|
|
buildAssignType(B, AssignedType, V);
|
|
return;
|
|
}
|
|
|
|
Type *CurrentType =
|
|
dyn_cast<ConstantAsMetadata>(
|
|
cast<MetadataAsValue>(AssignCI->getOperand(1))->getMetadata())
|
|
->getType();
|
|
if (CurrentType == AssignedType)
|
|
return;
|
|
|
|
// Builtin types cannot be redeclared or casted.
|
|
if (CurrentType->isTargetExtTy())
|
|
report_fatal_error("Type mismatch " + CurrentType->getTargetExtName() +
|
|
"/" + AssignedType->getTargetExtName() +
|
|
" for value " + V->getName(),
|
|
false);
|
|
|
|
// Our previous guess about the type seems to be wrong, let's update
|
|
// inferred type according to a new, more precise type information.
|
|
updateAssignType(AssignCI, V, PoisonValue::get(AssignedType));
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
|
|
Instruction *I, Value *Pointer, Type *ExpectedElementType,
|
|
unsigned OperandToReplace, IRBuilder<> &B) {
|
|
// If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source
|
|
// pointer instead. The BitCastInst should be later removed when visited.
|
|
while (BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
|
|
Pointer = BC->getOperand(0);
|
|
|
|
// Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
|
|
Type *PointerElemTy = deduceElementTypeHelper(Pointer, false);
|
|
if (PointerElemTy == ExpectedElementType ||
|
|
isEquivalentTypes(PointerElemTy, ExpectedElementType))
|
|
return;
|
|
|
|
setInsertPointSkippingPhis(B, I);
|
|
MetadataAsValue *VMD = buildMD(PoisonValue::get(ExpectedElementType));
|
|
unsigned AddressSpace = getPointerAddressSpace(Pointer->getType());
|
|
bool FirstPtrCastOrAssignPtrType = true;
|
|
|
|
// Do not emit new spv_ptrcast if equivalent one already exists or when
|
|
// spv_assign_ptr_type already targets this pointer with the same element
|
|
// type.
|
|
for (auto User : Pointer->users()) {
|
|
auto *II = dyn_cast<IntrinsicInst>(User);
|
|
if (!II ||
|
|
(II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
|
|
II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
|
|
II->getOperand(0) != Pointer)
|
|
continue;
|
|
|
|
// There is some spv_ptrcast/spv_assign_ptr_type already targeting this
|
|
// pointer.
|
|
FirstPtrCastOrAssignPtrType = false;
|
|
if (II->getOperand(1) != VMD ||
|
|
dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() !=
|
|
AddressSpace)
|
|
continue;
|
|
|
|
// The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the same
|
|
// element type and address space.
|
|
if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
|
|
return;
|
|
|
|
// This must be a spv_ptrcast, do not emit new if this one has the same BB
|
|
// as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
|
|
if (II->getParent() != I->getParent())
|
|
continue;
|
|
|
|
I->setOperand(OperandToReplace, II);
|
|
return;
|
|
}
|
|
|
|
// // Do not emit spv_ptrcast if it would cast to the default pointer element
|
|
// // type (i8) of the same address space.
|
|
// if (ExpectedElementType->isIntegerTy(8))
|
|
// return;
|
|
|
|
// If this would be the first spv_ptrcast, do not emit spv_ptrcast and emit
|
|
// spv_assign_ptr_type instead.
|
|
if (FirstPtrCastOrAssignPtrType &&
|
|
(isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
|
|
buildAssignPtr(B, ExpectedElementType, Pointer);
|
|
return;
|
|
}
|
|
|
|
// Emit spv_ptrcast
|
|
SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
|
|
SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(AddressSpace)};
|
|
auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
|
|
I->setOperand(OperandToReplace, PtrCastI);
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
|
|
IRBuilder<> &B) {
|
|
// Handle basic instructions:
|
|
StoreInst *SI = dyn_cast<StoreInst>(I);
|
|
if (IsKernelArgInt8(F, SI)) {
|
|
return replacePointerOperandWithPtrCast(
|
|
I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0,
|
|
B);
|
|
} else if (SI) {
|
|
Value *Op = SI->getValueOperand();
|
|
Type *OpTy = Op->getType();
|
|
if (auto *OpI = dyn_cast<Instruction>(Op))
|
|
OpTy = restoreMutatedType(GR, OpI, OpTy);
|
|
if (OpTy == Op->getType())
|
|
OpTy = deduceElementTypeByValueDeep(OpTy, Op, false);
|
|
return replacePointerOperandWithPtrCast(I, SI->getPointerOperand(), OpTy, 1,
|
|
B);
|
|
} else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
|
|
return replacePointerOperandWithPtrCast(I, LI->getPointerOperand(),
|
|
LI->getType(), 0, B);
|
|
} else if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
|
|
return replacePointerOperandWithPtrCast(I, GEPI->getPointerOperand(),
|
|
GEPI->getSourceElementType(), 0, B);
|
|
}
|
|
|
|
// Handle calls to builtins (non-intrinsics):
|
|
CallInst *CI = dyn_cast<CallInst>(I);
|
|
if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
|
|
!CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
|
|
return;
|
|
|
|
// collect information about formal parameter types
|
|
std::string DemangledName =
|
|
getOclOrSpirvBuiltinDemangledName(CI->getCalledFunction()->getName());
|
|
Function *CalledF = CI->getCalledFunction();
|
|
SmallVector<Type *, 4> CalledArgTys;
|
|
bool HaveTypes = false;
|
|
for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
|
|
Argument *CalledArg = CalledF->getArg(OpIdx);
|
|
Type *ArgType = CalledArg->getType();
|
|
if (!isPointerTy(ArgType)) {
|
|
CalledArgTys.push_back(nullptr);
|
|
} else if (isTypedPointerTy(ArgType)) {
|
|
CalledArgTys.push_back(cast<TypedPointerType>(ArgType)->getElementType());
|
|
HaveTypes = true;
|
|
} else {
|
|
Type *ElemTy = GR->findDeducedElementType(CalledArg);
|
|
if (!ElemTy && hasPointeeTypeAttr(CalledArg))
|
|
ElemTy = getPointeeTypeByAttr(CalledArg);
|
|
if (!ElemTy) {
|
|
ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
|
|
if (ElemTy) {
|
|
GR->addDeducedElementType(CalledArg, ElemTy);
|
|
} else {
|
|
for (User *U : CalledArg->users()) {
|
|
if (Instruction *Inst = dyn_cast<Instruction>(U)) {
|
|
if ((ElemTy = deduceElementTypeHelper(Inst, false)) != nullptr)
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
HaveTypes |= ElemTy != nullptr;
|
|
CalledArgTys.push_back(ElemTy);
|
|
}
|
|
}
|
|
|
|
if (DemangledName.empty() && !HaveTypes)
|
|
return;
|
|
|
|
for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
|
|
Value *ArgOperand = CI->getArgOperand(OpIdx);
|
|
if (!isPointerTy(ArgOperand->getType()))
|
|
continue;
|
|
|
|
// Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
|
|
if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) {
|
|
// However, we may have assumptions about the formal argument's type and
|
|
// may have a need to insert a ptr cast for the actual parameter of this
|
|
// call.
|
|
Argument *CalledArg = CalledF->getArg(OpIdx);
|
|
if (!GR->findDeducedElementType(CalledArg))
|
|
continue;
|
|
}
|
|
|
|
Type *ExpectedType =
|
|
OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
|
|
if (!ExpectedType && !DemangledName.empty())
|
|
ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
|
|
DemangledName, OpIdx, I->getContext());
|
|
if (!ExpectedType || ExpectedType->isVoidTy())
|
|
continue;
|
|
|
|
if (ExpectedType->isTargetExtTy())
|
|
insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType),
|
|
ArgOperand, B);
|
|
else
|
|
replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B);
|
|
}
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
|
|
SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
|
|
I.getOperand(1)->getType(),
|
|
I.getOperand(2)->getType()};
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Value *> Args(I.op_begin(), I.op_end());
|
|
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
|
|
std::string InstName = I.hasName() ? I.getName().str() : "";
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
NewI->setName(InstName);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *
|
|
SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
|
|
I.getIndexOperand()->getType()};
|
|
SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
|
|
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
|
|
std::string InstName = I.hasName() ? I.getName().str() : "";
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
NewI->setName(InstName);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
|
|
SmallVector<Value *> Args;
|
|
for (auto &Op : I.operands())
|
|
if (isa<UndefValue>(Op))
|
|
Args.push_back(UndefValue::get(B.getInt32Ty()));
|
|
else
|
|
Args.push_back(Op);
|
|
for (auto &Op : I.indices())
|
|
Args.push_back(B.getInt32(Op));
|
|
Instruction *NewI =
|
|
B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
|
|
replaceMemInstrUses(&I, NewI, B);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Value *> Args;
|
|
for (auto &Op : I.operands())
|
|
Args.push_back(Op);
|
|
for (auto &Op : I.indices())
|
|
Args.push_back(B.getInt32(Op));
|
|
auto *NewI =
|
|
B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
|
|
if (!I.getType()->isAggregateType())
|
|
return &I;
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
TrackConstants = false;
|
|
const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
|
|
MachineMemOperand::Flags Flags =
|
|
TLI->getLoadMemOperandFlags(I, F->getDataLayout());
|
|
auto *NewI =
|
|
B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
|
|
{I.getPointerOperand(), B.getInt16(Flags),
|
|
B.getInt8(I.getAlign().value())});
|
|
replaceMemInstrUses(&I, NewI, B);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
|
|
if (!AggrStores.contains(&I))
|
|
return &I;
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
TrackConstants = false;
|
|
const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
|
|
MachineMemOperand::Flags Flags =
|
|
TLI->getStoreMemOperandFlags(I, F->getDataLayout());
|
|
auto *PtrOp = I.getPointerOperand();
|
|
auto *NewI = B.CreateIntrinsic(
|
|
Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
|
|
{I.getValueOperand(), PtrOp, B.getInt16(Flags),
|
|
B.getInt8(I.getAlign().value())});
|
|
I.eraseFromParent();
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
|
|
Value *ArraySize = nullptr;
|
|
if (I.isArrayAllocation()) {
|
|
const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
|
|
if (!STI->canUseExtension(
|
|
SPIRV::Extension::SPV_INTEL_variable_length_array))
|
|
report_fatal_error(
|
|
"array allocation: this instruction requires the following "
|
|
"SPIR-V extension: SPV_INTEL_variable_length_array",
|
|
false);
|
|
ArraySize = I.getArraySize();
|
|
}
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
TrackConstants = false;
|
|
Type *PtrTy = I.getType();
|
|
auto *NewI =
|
|
ArraySize ? B.CreateIntrinsic(Intrinsic::spv_alloca_array,
|
|
{PtrTy, ArraySize->getType()}, {ArraySize})
|
|
: B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
|
|
std::string InstName = I.hasName() ? I.getName().str() : "";
|
|
I.replaceAllUsesWith(NewI);
|
|
I.eraseFromParent();
|
|
NewI->setName(InstName);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
|
|
assert(I.getType()->isAggregateType() && "Aggregate result is expected");
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
SmallVector<Value *> Args;
|
|
for (auto &Op : I.operands())
|
|
Args.push_back(Op);
|
|
Args.push_back(B.getInt32(I.getSyncScopeID()));
|
|
Args.push_back(B.getInt32(
|
|
static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering()))));
|
|
Args.push_back(B.getInt32(
|
|
static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering()))));
|
|
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_cmpxchg,
|
|
{I.getPointerOperand()->getType()}, {Args});
|
|
replaceMemInstrUses(&I, NewI, B);
|
|
return NewI;
|
|
}
|
|
|
|
Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
|
|
IRBuilder<> B(I.getParent());
|
|
B.SetInsertPoint(&I);
|
|
B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
|
|
return &I;
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
|
|
IRBuilder<> &B) {
|
|
// Skip special artifical variable llvm.global.annotations.
|
|
if (GV.getName() == "llvm.global.annotations")
|
|
return;
|
|
if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
|
|
// Deduce element type and store results in Global Registry.
|
|
// Result is ignored, because TypedPointerType is not supported
|
|
// by llvm IR general logic.
|
|
deduceElementTypeHelper(&GV, false);
|
|
Constant *Init = GV.getInitializer();
|
|
Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType();
|
|
Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init;
|
|
auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global,
|
|
{GV.getType(), Ty}, {&GV, Const});
|
|
InitInst->setArgOperand(1, Init);
|
|
}
|
|
if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
|
|
GV.getNumUses() == 0)
|
|
B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
|
|
}
|
|
|
|
// Return true, if we can't decide what is the pointee type now and will get
|
|
// back to the question later. Return false is spv_assign_ptr_type is not needed
|
|
// or can be inserted immediately.
|
|
bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
|
|
IRBuilder<> &B,
|
|
bool UnknownElemTypeI8) {
|
|
reportFatalOnTokenType(I);
|
|
if (!isPointerTy(I->getType()) || !requireAssignType(I) ||
|
|
isa<BitCastInst>(I))
|
|
return false;
|
|
|
|
setInsertPointAfterDef(B, I);
|
|
if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
|
|
buildAssignPtr(B, ElemTy, I);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
|
|
IRBuilder<> &B) {
|
|
// TODO: extend the list of functions with known result types
|
|
static StringMap<unsigned> ResTypeWellKnown = {
|
|
{"async_work_group_copy", WellKnownTypes::Event},
|
|
{"async_work_group_strided_copy", WellKnownTypes::Event},
|
|
{"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
|
|
|
|
reportFatalOnTokenType(I);
|
|
|
|
bool IsKnown = false;
|
|
if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
|
|
CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
|
|
Function *CalledF = CI->getCalledFunction();
|
|
std::string DemangledName =
|
|
getOclOrSpirvBuiltinDemangledName(CalledF->getName());
|
|
if (DemangledName.length() > 0)
|
|
DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName);
|
|
auto ResIt = ResTypeWellKnown.find(DemangledName);
|
|
if (ResIt != ResTypeWellKnown.end()) {
|
|
IsKnown = true;
|
|
setInsertPointAfterDef(B, I);
|
|
switch (ResIt->second) {
|
|
case WellKnownTypes::Event:
|
|
buildAssignType(B, TargetExtType::get(I->getContext(), "spirv.Event"),
|
|
I);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Type *Ty = I->getType();
|
|
if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(Ty) && requireAssignType(I)) {
|
|
setInsertPointAfterDef(B, I);
|
|
Type *TypeToAssign = Ty;
|
|
if (auto *II = dyn_cast<IntrinsicInst>(I)) {
|
|
if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
|
|
II->getIntrinsicID() == Intrinsic::spv_undef) {
|
|
auto It = AggrConstTypes.find(II);
|
|
if (It == AggrConstTypes.end())
|
|
report_fatal_error("Unknown composite intrinsic type");
|
|
TypeToAssign = It->second;
|
|
}
|
|
}
|
|
TypeToAssign = restoreMutatedType(GR, I, TypeToAssign);
|
|
buildAssignType(B, TypeToAssign, I);
|
|
}
|
|
for (const auto &Op : I->operands()) {
|
|
if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
|
|
// Check GetElementPtrConstantExpr case.
|
|
(isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
|
|
setInsertPointSkippingPhis(B, I);
|
|
Type *OpTy = Op->getType();
|
|
if (isa<UndefValue>(Op) && OpTy->isAggregateType()) {
|
|
CallInst *AssignCI =
|
|
buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op,
|
|
UndefValue::get(B.getInt32Ty()), {}, B);
|
|
GR->addAssignPtrTypeInstr(Op, AssignCI);
|
|
} else if (!isa<Instruction>(Op)) {
|
|
Type *OpTy = Op->getType();
|
|
if (auto PType = dyn_cast<TypedPointerType>(OpTy)) {
|
|
buildAssignPtr(B, PType->getElementType(), Op);
|
|
} else if (isPointerTy(OpTy)) {
|
|
Type *ElemTy = GR->findDeducedElementType(Op);
|
|
buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op);
|
|
} else {
|
|
CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
|
|
{OpTy}, Op, Op, {}, B);
|
|
GR->addAssignPtrTypeInstr(Op, AssignCI);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
|
|
IRBuilder<> &B) {
|
|
if (MDNode *MD = I->getMetadata("spirv.Decorations")) {
|
|
setInsertPointAfterDef(B, I);
|
|
B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
|
|
{I, MetadataAsValue::get(I->getContext(), MD)});
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
|
|
IRBuilder<> &B) {
|
|
auto *II = dyn_cast<IntrinsicInst>(I);
|
|
if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
|
|
TrackConstants) {
|
|
setInsertPointAfterDef(B, I);
|
|
auto t = AggrConsts.find(I);
|
|
assert(t != AggrConsts.end());
|
|
auto *NewOp =
|
|
buildIntrWithMD(Intrinsic::spv_track_constant,
|
|
{II->getType(), II->getType()}, t->second, I, {}, B);
|
|
I->replaceAllUsesWith(NewOp);
|
|
NewOp->setArgOperand(0, I);
|
|
}
|
|
bool IsPhi = isa<PHINode>(I), BPrepared = false;
|
|
for (const auto &Op : I->operands()) {
|
|
if (isa<PHINode>(I) || isa<SwitchInst>(I))
|
|
TrackConstants = false;
|
|
if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) {
|
|
unsigned OpNo = Op.getOperandNo();
|
|
if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
|
|
(II->paramHasAttr(OpNo, Attribute::ImmArg))))
|
|
continue;
|
|
if (!BPrepared) {
|
|
IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
|
|
: B.SetInsertPoint(I);
|
|
BPrepared = true;
|
|
}
|
|
Value *OpTyVal = Op;
|
|
if (Op->getType()->isTargetExtTy())
|
|
OpTyVal = PoisonValue::get(Op->getType());
|
|
auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
|
|
{Op->getType(), OpTyVal->getType()}, Op,
|
|
OpTyVal, {}, B);
|
|
I->setOperand(OpNo, NewOp);
|
|
}
|
|
}
|
|
if (I->hasName()) {
|
|
reportFatalOnTokenType(I);
|
|
setInsertPointAfterDef(B, I);
|
|
std::vector<Value *> Args = {I};
|
|
addStringImm(I->getName(), B, Args);
|
|
B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
|
|
}
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
|
|
unsigned OpIdx) {
|
|
std::unordered_set<Function *> FVisited;
|
|
return deduceFunParamElementType(F, OpIdx, FVisited);
|
|
}
|
|
|
|
Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
|
|
Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
|
|
// maybe a cycle
|
|
if (FVisited.find(F) != FVisited.end())
|
|
return nullptr;
|
|
FVisited.insert(F);
|
|
|
|
std::unordered_set<Value *> Visited;
|
|
SmallVector<std::pair<Function *, unsigned>> Lookup;
|
|
// search in function's call sites
|
|
for (User *U : F->users()) {
|
|
CallInst *CI = dyn_cast<CallInst>(U);
|
|
if (!CI || OpIdx >= CI->arg_size())
|
|
continue;
|
|
Value *OpArg = CI->getArgOperand(OpIdx);
|
|
if (!isPointerTy(OpArg->getType()))
|
|
continue;
|
|
// maybe we already know operand's element type
|
|
if (Type *KnownTy = GR->findDeducedElementType(OpArg))
|
|
return KnownTy;
|
|
// try to deduce from the operand itself
|
|
Visited.clear();
|
|
if (Type *Ty = deduceElementTypeHelper(OpArg, Visited, false))
|
|
return Ty;
|
|
// search in actual parameter's users
|
|
for (User *OpU : OpArg->users()) {
|
|
Instruction *Inst = dyn_cast<Instruction>(OpU);
|
|
if (!Inst || Inst == CI)
|
|
continue;
|
|
Visited.clear();
|
|
if (Type *Ty = deduceElementTypeHelper(Inst, Visited, false))
|
|
return Ty;
|
|
}
|
|
// check if it's a formal parameter of the outer function
|
|
if (!CI->getParent() || !CI->getParent()->getParent())
|
|
continue;
|
|
Function *OuterF = CI->getParent()->getParent();
|
|
if (FVisited.find(OuterF) != FVisited.end())
|
|
continue;
|
|
for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
|
|
if (OuterF->getArg(i) == OpArg) {
|
|
Lookup.push_back(std::make_pair(OuterF, i));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// search in function parameters
|
|
for (auto &Pair : Lookup) {
|
|
if (Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
|
|
return Ty;
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
|
|
IRBuilder<> &B) {
|
|
B.SetInsertPointPastAllocas(F);
|
|
for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
|
|
Argument *Arg = F->getArg(OpIdx);
|
|
if (!isUntypedPointerTy(Arg->getType()))
|
|
continue;
|
|
Type *ElemTy = GR->findDeducedElementType(Arg);
|
|
if (!ElemTy && hasPointeeTypeAttr(Arg) &&
|
|
(ElemTy = getPointeeTypeByAttr(Arg)) != nullptr)
|
|
buildAssignPtr(B, ElemTy, Arg);
|
|
}
|
|
}
|
|
|
|
void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
|
|
B.SetInsertPointPastAllocas(F);
|
|
for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
|
|
Argument *Arg = F->getArg(OpIdx);
|
|
if (!isUntypedPointerTy(Arg->getType()))
|
|
continue;
|
|
Type *ElemTy = GR->findDeducedElementType(Arg);
|
|
if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr)
|
|
buildAssignPtr(B, ElemTy, Arg);
|
|
}
|
|
}
|
|
|
|
bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
|
|
if (Func.isDeclaration())
|
|
return false;
|
|
|
|
const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(Func);
|
|
GR = ST.getSPIRVGlobalRegistry();
|
|
InstrSet = ST.isOpenCLEnv() ? SPIRV::InstructionSet::OpenCL_std
|
|
: SPIRV::InstructionSet::GLSL_std_450;
|
|
|
|
F = &Func;
|
|
IRBuilder<> B(Func.getContext());
|
|
AggrConsts.clear();
|
|
AggrConstTypes.clear();
|
|
AggrStores.clear();
|
|
|
|
processParamTypesByFunHeader(F, B);
|
|
|
|
// StoreInst's operand type can be changed during the next transformations,
|
|
// so we need to store it in the set. Also store already transformed types.
|
|
for (auto &I : instructions(Func)) {
|
|
StoreInst *SI = dyn_cast<StoreInst>(&I);
|
|
if (!SI)
|
|
continue;
|
|
Type *ElTy = SI->getValueOperand()->getType();
|
|
if (ElTy->isAggregateType() || ElTy->isVectorTy())
|
|
AggrStores.insert(&I);
|
|
}
|
|
|
|
B.SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin());
|
|
for (auto &GV : Func.getParent()->globals())
|
|
processGlobalValue(GV, B);
|
|
|
|
preprocessUndefs(B);
|
|
preprocessCompositeConstants(B);
|
|
SmallVector<Instruction *> Worklist;
|
|
for (auto &I : instructions(Func))
|
|
Worklist.push_back(&I);
|
|
|
|
for (auto &I : Worklist) {
|
|
// Don't emit intrinsincs for convergence intrinsics.
|
|
if (isConvergenceIntrinsic(I))
|
|
continue;
|
|
|
|
bool Postpone = insertAssignPtrTypeIntrs(I, B, false);
|
|
// if Postpone is true, we can't decide on pointee type yet
|
|
insertAssignTypeIntrs(I, B);
|
|
insertPtrCastOrAssignTypeInstr(I, B);
|
|
insertSpirvDecorations(I, B);
|
|
// if instruction requires a pointee type set, let's check if we know it
|
|
// already, and force it to be i8 if not
|
|
if (Postpone && !GR->findAssignPtrTypeInstr(I))
|
|
insertAssignPtrTypeIntrs(I, B, true);
|
|
}
|
|
|
|
for (auto &I : instructions(Func))
|
|
deduceOperandElementType(&I);
|
|
|
|
for (auto *I : Worklist) {
|
|
TrackConstants = true;
|
|
if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
|
|
setInsertPointAfterDef(B, I);
|
|
// Visitors return either the original/newly created instruction for further
|
|
// processing, nullptr otherwise.
|
|
I = visit(*I);
|
|
if (!I)
|
|
continue;
|
|
|
|
// Don't emit intrinsics for convergence operations.
|
|
if (isConvergenceIntrinsic(I))
|
|
continue;
|
|
|
|
processInstrAfterVisit(I, B);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Try to deduce a better type for pointers to untyped ptr.
|
|
bool SPIRVEmitIntrinsics::postprocessTypes() {
|
|
bool Changed = false;
|
|
if (!GR)
|
|
return Changed;
|
|
for (auto IB = PostprocessWorklist.rbegin(), IE = PostprocessWorklist.rend();
|
|
IB != IE; ++IB) {
|
|
CallInst *AssignCI = GR->findAssignPtrTypeInstr(*IB);
|
|
Type *KnownTy = GR->findDeducedElementType(*IB);
|
|
if (!KnownTy || !AssignCI || !isa<Instruction>(AssignCI->getArgOperand(0)))
|
|
continue;
|
|
Instruction *I = cast<Instruction>(AssignCI->getArgOperand(0));
|
|
for (User *U : I->users()) {
|
|
Instruction *Inst = dyn_cast<Instruction>(U);
|
|
if (!Inst || isa<IntrinsicInst>(Inst))
|
|
continue;
|
|
deduceOperandElementType(Inst, I, KnownTy, AssignCI);
|
|
if (KnownTy != GR->findDeducedElementType(I)) {
|
|
Changed = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return Changed;
|
|
}
|
|
|
|
bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
|
|
bool Changed = false;
|
|
|
|
UncompleteTypeInfo.clear();
|
|
PostprocessWorklist.clear();
|
|
for (auto &F : M)
|
|
Changed |= runOnFunction(F);
|
|
|
|
for (auto &F : M) {
|
|
// check if function parameter types are set
|
|
if (!F.isDeclaration() && !F.isIntrinsic()) {
|
|
const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
|
|
GR = ST.getSPIRVGlobalRegistry();
|
|
IRBuilder<> B(F.getContext());
|
|
processParamTypes(&F, B);
|
|
}
|
|
}
|
|
|
|
Changed |= postprocessTypes();
|
|
|
|
return Changed;
|
|
}
|
|
|
|
ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
|
|
return new SPIRVEmitIntrinsics(TM);
|
|
}
|