Files
clang-p2996/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
Ilia Diachkov 74c66710a7 [SPIRV] fix several issues in builds with expensive checks
The patch fixes "Virtual register does not match instruction constraint"
and partly "Illegal virtual register for instruction" fails in the SPIRV
backend builds with LLVM_ENABLE_EXPENSIVE_CHECKS enabled. As a result,
the number of passed LIT tests with enabled checks is doubled.

Also, support for ndrange_*D builtins is placed in a separate function.

Differential Revision: https://reviews.llvm.org/D144897
2023-03-17 00:08:23 +03:00

445 lines
18 KiB
C++

//===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the lowering of LLVM calls to machine code calls for
// GlobalISel.
//
//===----------------------------------------------------------------------===//
#include "SPIRVCallLowering.h"
#include "MCTargetDesc/SPIRVBaseInfo.h"
#include "SPIRV.h"
#include "SPIRVBuiltins.h"
#include "SPIRVGlobalRegistry.h"
#include "SPIRVISelLowering.h"
#include "SPIRVRegisterInfo.h"
#include "SPIRVSubtarget.h"
#include "SPIRVUtils.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/Support/ModRef.h"
using namespace llvm;
SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI,
SPIRVGlobalRegistry *GR)
: CallLowering(&TLI), GR(GR) {}
bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
const Value *Val, ArrayRef<Register> VRegs,
FunctionLoweringInfo &FLI,
Register SwiftErrorVReg) const {
// Currently all return types should use a single register.
// TODO: handle the case of multiple registers.
if (VRegs.size() > 1)
return false;
if (Val) {
const auto &STI = MIRBuilder.getMF().getSubtarget();
return MIRBuilder.buildInstr(SPIRV::OpReturnValue)
.addUse(VRegs[0])
.constrainAllUses(MIRBuilder.getTII(), *STI.getRegisterInfo(),
*STI.getRegBankInfo());
}
MIRBuilder.buildInstr(SPIRV::OpReturn);
return true;
}
// Based on the LLVM function attributes, get a SPIR-V FunctionControl.
static uint32_t getFunctionControl(const Function &F) {
MemoryEffects MemEffects = F.getMemoryEffects();
uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None);
if (F.hasFnAttribute(Attribute::AttrKind::NoInline))
FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline);
else if (F.hasFnAttribute(Attribute::AttrKind::AlwaysInline))
FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline);
if (MemEffects.doesNotAccessMemory())
FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure);
else if (MemEffects.onlyReadsMemory())
FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const);
return FuncControl;
}
static ConstantInt *getConstInt(MDNode *MD, unsigned NumOp) {
if (MD->getNumOperands() > NumOp) {
auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(NumOp));
if (CMeta)
return dyn_cast<ConstantInt>(CMeta->getValue());
}
return nullptr;
}
// This code restores function args/retvalue types for composite cases
// because the final types should still be aggregate whereas they're i32
// during the translation to cope with aggregate flattening etc.
static FunctionType *getOriginalFunctionType(const Function &F) {
auto *NamedMD = F.getParent()->getNamedMetadata("spv.cloned_funcs");
if (NamedMD == nullptr)
return F.getFunctionType();
Type *RetTy = F.getFunctionType()->getReturnType();
SmallVector<Type *, 4> ArgTypes;
for (auto &Arg : F.args())
ArgTypes.push_back(Arg.getType());
auto ThisFuncMDIt =
std::find_if(NamedMD->op_begin(), NamedMD->op_end(), [&F](MDNode *N) {
return isa<MDString>(N->getOperand(0)) &&
cast<MDString>(N->getOperand(0))->getString() == F.getName();
});
// TODO: probably one function can have numerous type mutations,
// so we should support this.
if (ThisFuncMDIt != NamedMD->op_end()) {
auto *ThisFuncMD = *ThisFuncMDIt;
MDNode *MD = dyn_cast<MDNode>(ThisFuncMD->getOperand(1));
assert(MD && "MDNode operand is expected");
ConstantInt *Const = getConstInt(MD, 0);
if (Const) {
auto *CMeta = dyn_cast<ConstantAsMetadata>(MD->getOperand(1));
assert(CMeta && "ConstantAsMetadata operand is expected");
assert(Const->getSExtValue() >= -1);
// Currently -1 indicates return value, greater values mean
// argument numbers.
if (Const->getSExtValue() == -1)
RetTy = CMeta->getType();
else
ArgTypes[Const->getSExtValue()] = CMeta->getType();
}
}
return FunctionType::get(RetTy, ArgTypes, F.isVarArg());
}
static MDString *getKernelArgAttribute(const Function &KernelFunction,
unsigned ArgIdx,
const StringRef AttributeName) {
assert(KernelFunction.getCallingConv() == CallingConv::SPIR_KERNEL &&
"Kernel attributes are attached/belong only to kernel functions");
// Lookup the argument attribute in metadata attached to the kernel function.
MDNode *Node = KernelFunction.getMetadata(AttributeName);
if (Node && ArgIdx < Node->getNumOperands())
return cast<MDString>(Node->getOperand(ArgIdx));
// Sometimes metadata containing kernel attributes is not attached to the
// function, but can be found in the named module-level metadata instead.
// For example:
// !opencl.kernels = !{!0}
// !0 = !{void ()* @someKernelFunction, !1, ...}
// !1 = !{!"kernel_arg_addr_space", ...}
// In this case the actual index of searched argument attribute is ArgIdx + 1,
// since the first metadata node operand is occupied by attribute name
// ("kernel_arg_addr_space" in the example above).
unsigned MDArgIdx = ArgIdx + 1;
NamedMDNode *OpenCLKernelsMD =
KernelFunction.getParent()->getNamedMetadata("opencl.kernels");
if (!OpenCLKernelsMD || OpenCLKernelsMD->getNumOperands() == 0)
return nullptr;
// KernelToMDNodeList contains kernel function declarations followed by
// corresponding MDNodes for each attribute. Search only MDNodes "belonging"
// to the currently lowered kernel function.
MDNode *KernelToMDNodeList = OpenCLKernelsMD->getOperand(0);
bool FoundLoweredKernelFunction = false;
for (const MDOperand &Operand : KernelToMDNodeList->operands()) {
ValueAsMetadata *MaybeValue = dyn_cast<ValueAsMetadata>(Operand);
if (MaybeValue && dyn_cast<Function>(MaybeValue->getValue())->getName() ==
KernelFunction.getName()) {
FoundLoweredKernelFunction = true;
continue;
}
if (MaybeValue && FoundLoweredKernelFunction)
return nullptr;
MDNode *MaybeNode = dyn_cast<MDNode>(Operand);
if (FoundLoweredKernelFunction && MaybeNode &&
cast<MDString>(MaybeNode->getOperand(0))->getString() ==
AttributeName &&
MDArgIdx < MaybeNode->getNumOperands())
return cast<MDString>(MaybeNode->getOperand(MDArgIdx));
}
return nullptr;
}
static SPIRV::AccessQualifier::AccessQualifier
getArgAccessQual(const Function &F, unsigned ArgIdx) {
if (F.getCallingConv() != CallingConv::SPIR_KERNEL)
return SPIRV::AccessQualifier::ReadWrite;
MDString *ArgAttribute =
getKernelArgAttribute(F, ArgIdx, "kernel_arg_access_qual");
if (!ArgAttribute)
return SPIRV::AccessQualifier::ReadWrite;
if (ArgAttribute->getString().compare("read_only") == 0)
return SPIRV::AccessQualifier::ReadOnly;
if (ArgAttribute->getString().compare("write_only") == 0)
return SPIRV::AccessQualifier::WriteOnly;
return SPIRV::AccessQualifier::ReadWrite;
}
static std::vector<SPIRV::Decoration::Decoration>
getKernelArgTypeQual(const Function &KernelFunction, unsigned ArgIdx) {
MDString *ArgAttribute =
getKernelArgAttribute(KernelFunction, ArgIdx, "kernel_arg_type_qual");
if (ArgAttribute && ArgAttribute->getString().compare("volatile") == 0)
return {SPIRV::Decoration::Volatile};
return {};
}
static Type *getArgType(const Function &F, unsigned ArgIdx) {
Type *OriginalArgType = getOriginalFunctionType(F)->getParamType(ArgIdx);
if (F.getCallingConv() != CallingConv::SPIR_KERNEL ||
isSpecialOpaqueType(OriginalArgType))
return OriginalArgType;
MDString *MDKernelArgType =
getKernelArgAttribute(F, ArgIdx, "kernel_arg_type");
if (!MDKernelArgType || !MDKernelArgType->getString().endswith("_t"))
return OriginalArgType;
std::string KernelArgTypeStr = "opencl." + MDKernelArgType->getString().str();
Type *ExistingOpaqueType =
StructType::getTypeByName(F.getContext(), KernelArgTypeStr);
return ExistingOpaqueType
? ExistingOpaqueType
: StructType::create(F.getContext(), KernelArgTypeStr);
}
bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
const Function &F,
ArrayRef<ArrayRef<Register>> VRegs,
FunctionLoweringInfo &FLI) const {
assert(GR && "Must initialize the SPIRV type registry before lowering args.");
GR->setCurrentFunc(MIRBuilder.getMF());
// Assign types and names to all args, and store their types for later.
FunctionType *FTy = getOriginalFunctionType(F);
SmallVector<SPIRVType *, 4> ArgTypeVRegs;
if (VRegs.size() > 0) {
unsigned i = 0;
for (const auto &Arg : F.args()) {
// Currently formal args should use single registers.
// TODO: handle the case of multiple registers.
if (VRegs[i].size() > 1)
return false;
SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
getArgAccessQual(F, i);
auto *SpirvTy = GR->assignTypeToVReg(getArgType(F, i), VRegs[i][0],
MIRBuilder, ArgAccessQual);
ArgTypeVRegs.push_back(SpirvTy);
if (Arg.hasName())
buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder);
if (Arg.getType()->isPointerTy()) {
auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes());
if (DerefBytes != 0)
buildOpDecorate(VRegs[i][0], MIRBuilder,
SPIRV::Decoration::MaxByteOffset, {DerefBytes});
}
if (Arg.hasAttribute(Attribute::Alignment)) {
auto Alignment = static_cast<unsigned>(
Arg.getAttribute(Attribute::Alignment).getValueAsInt());
buildOpDecorate(VRegs[i][0], MIRBuilder, SPIRV::Decoration::Alignment,
{Alignment});
}
if (Arg.hasAttribute(Attribute::ReadOnly)) {
auto Attr =
static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite);
buildOpDecorate(VRegs[i][0], MIRBuilder,
SPIRV::Decoration::FuncParamAttr, {Attr});
}
if (Arg.hasAttribute(Attribute::ZExt)) {
auto Attr =
static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext);
buildOpDecorate(VRegs[i][0], MIRBuilder,
SPIRV::Decoration::FuncParamAttr, {Attr});
}
if (Arg.hasAttribute(Attribute::NoAlias)) {
auto Attr =
static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias);
buildOpDecorate(VRegs[i][0], MIRBuilder,
SPIRV::Decoration::FuncParamAttr, {Attr});
}
if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
std::vector<SPIRV::Decoration::Decoration> ArgTypeQualDecs =
getKernelArgTypeQual(F, i);
for (SPIRV::Decoration::Decoration Decoration : ArgTypeQualDecs)
buildOpDecorate(VRegs[i][0], MIRBuilder, Decoration, {});
}
MDNode *Node = F.getMetadata("spirv.ParameterDecorations");
if (Node && i < Node->getNumOperands() &&
isa<MDNode>(Node->getOperand(i))) {
MDNode *MD = cast<MDNode>(Node->getOperand(i));
for (const MDOperand &MDOp : MD->operands()) {
MDNode *MD2 = dyn_cast<MDNode>(MDOp);
assert(MD2 && "Metadata operand is expected");
ConstantInt *Const = getConstInt(MD2, 0);
assert(Const && "MDOperand should be ConstantInt");
auto Dec =
static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue());
std::vector<uint32_t> DecVec;
for (unsigned j = 1; j < MD2->getNumOperands(); j++) {
ConstantInt *Const = getConstInt(MD2, j);
assert(Const && "MDOperand should be ConstantInt");
DecVec.push_back(static_cast<uint32_t>(Const->getZExtValue()));
}
buildOpDecorate(VRegs[i][0], MIRBuilder, Dec, DecVec);
}
}
++i;
}
}
// Generate a SPIR-V type for the function.
auto MRI = MIRBuilder.getMRI();
Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass);
if (F.isDeclaration())
GR->add(&F, &MIRBuilder.getMF(), FuncVReg);
SPIRVType *RetTy = GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder);
SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
FTy, RetTy, ArgTypeVRegs, MIRBuilder);
// Build the OpTypeFunction declaring it.
uint32_t FuncControl = getFunctionControl(F);
MIRBuilder.buildInstr(SPIRV::OpFunction)
.addDef(FuncVReg)
.addUse(GR->getSPIRVTypeID(RetTy))
.addImm(FuncControl)
.addUse(GR->getSPIRVTypeID(FuncTy));
// Add OpFunctionParameters.
int i = 0;
for (const auto &Arg : F.args()) {
assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
MRI->setRegClass(VRegs[i][0], &SPIRV::IDRegClass);
MIRBuilder.buildInstr(SPIRV::OpFunctionParameter)
.addDef(VRegs[i][0])
.addUse(GR->getSPIRVTypeID(ArgTypeVRegs[i]));
if (F.isDeclaration())
GR->add(&Arg, &MIRBuilder.getMF(), VRegs[i][0]);
i++;
}
// Name the function.
if (F.hasName())
buildOpName(FuncVReg, F.getName(), MIRBuilder);
// Handle entry points and function linkage.
if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint)
.addImm(static_cast<uint32_t>(SPIRV::ExecutionModel::Kernel))
.addUse(FuncVReg);
addStringImm(F.getName(), MIB);
} else if (F.getLinkage() == GlobalValue::LinkageTypes::ExternalLinkage ||
F.getLinkage() == GlobalValue::LinkOnceODRLinkage) {
auto LnkTy = F.isDeclaration() ? SPIRV::LinkageType::Import
: SPIRV::LinkageType::Export;
buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::LinkageAttributes,
{static_cast<uint32_t>(LnkTy)}, F.getGlobalIdentifier());
}
return true;
}
bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
CallLoweringInfo &Info) const {
// Currently call returns should have single vregs.
// TODO: handle the case of multiple registers.
if (Info.OrigRet.Regs.size() > 1)
return false;
MachineFunction &MF = MIRBuilder.getMF();
GR->setCurrentFunc(MF);
FunctionType *FTy = nullptr;
const Function *CF = nullptr;
// Emit a regular OpFunctionCall. If it's an externally declared function,
// be sure to emit its type and function declaration here. It will be hoisted
// globally later.
if (Info.Callee.isGlobal()) {
CF = dyn_cast_or_null<const Function>(Info.Callee.getGlobal());
// TODO: support constexpr casts and indirect calls.
if (CF == nullptr)
return false;
FTy = getOriginalFunctionType(*CF);
}
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register ResVReg =
Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0];
std::string FuncName = Info.Callee.getGlobal()->getName().str();
std::string DemangledName = getOclOrSpirvBuiltinDemangledName(FuncName);
const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
// TODO: check that it's OCL builtin, then apply OpenCL_std.
if (!DemangledName.empty() && CF && CF->isDeclaration() &&
ST->canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
const Type *OrigRetTy = Info.OrigRet.Ty;
if (FTy)
OrigRetTy = FTy->getReturnType();
SmallVector<Register, 8> ArgVRegs;
for (auto Arg : Info.OrigArgs) {
assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
ArgVRegs.push_back(Arg.Regs[0]);
SPIRVType *SPIRVTy = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
GR->assignSPIRVTypeToVReg(SPIRVTy, Arg.Regs[0], MIRBuilder.getMF());
}
if (auto Res = SPIRV::lowerBuiltin(
DemangledName, SPIRV::InstructionSet::OpenCL_std, MIRBuilder,
ResVReg, OrigRetTy, ArgVRegs, GR))
return *Res;
}
if (CF && CF->isDeclaration() &&
!GR->find(CF, &MIRBuilder.getMF()).isValid()) {
// Emit the type info and forward function declaration to the first MBB
// to ensure VReg definition dependencies are valid across all MBBs.
MachineIRBuilder FirstBlockBuilder;
FirstBlockBuilder.setMF(MF);
FirstBlockBuilder.setMBB(*MF.getBlockNumbered(0));
SmallVector<ArrayRef<Register>, 8> VRegArgs;
SmallVector<SmallVector<Register, 1>, 8> ToInsert;
for (const Argument &Arg : CF->args()) {
if (MIRBuilder.getDataLayout().getTypeStoreSize(Arg.getType()).isZero())
continue; // Don't handle zero sized types.
Register Reg = MRI->createGenericVirtualRegister(LLT::scalar(32));
MRI->setRegClass(Reg, &SPIRV::IDRegClass);
ToInsert.push_back({Reg});
VRegArgs.push_back(ToInsert.back());
}
// TODO: Reuse FunctionLoweringInfo
FunctionLoweringInfo FuncInfo;
lowerFormalArguments(FirstBlockBuilder, *CF, VRegArgs, FuncInfo);
}
// Make sure there's a valid return reg, even for functions returning void.
if (!ResVReg.isValid())
ResVReg = MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass);
SPIRVType *RetType =
GR->assignTypeToVReg(FTy->getReturnType(), ResVReg, MIRBuilder);
// Emit the OpFunctionCall and its args.
auto MIB = MIRBuilder.buildInstr(SPIRV::OpFunctionCall)
.addDef(ResVReg)
.addUse(GR->getSPIRVTypeID(RetType))
.add(Info.Callee);
for (const auto &Arg : Info.OrigArgs) {
// Currently call args should have single vregs.
if (Arg.Regs.size() > 1)
return false;
MIB.addUse(Arg.Regs[0]);
}
return MIB.constrainAllUses(MIRBuilder.getTII(), *ST->getRegisterInfo(),
*ST->getRegBankInfo());
}