7e54ae24d8
This revision updates the llvm dialect inliner to explicitly disallow the inlining of variadic functions. Already previously the inlining failed if the number of function arguments did not match the number of call arguments. After the change, inlining checks the function is not variadic and it does not contain a va_start intrinsic.
803 lines
33 KiB
C++
803 lines
33 KiB
C++
//===- LLVMInlining.cpp - LLVM inlining interface and logic -----*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Logic for inlining LLVM functions and the definition of the
|
|
// LLVMInliningInterface.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "LLVMInlining.h"
|
|
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
|
|
#include "mlir/IR/Matchers.h"
|
|
#include "mlir/Interfaces/DataLayoutInterfaces.h"
|
|
#include "mlir/Transforms/InliningUtils.h"
|
|
#include "llvm/ADT/ScopeExit.h"
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#define DEBUG_TYPE "llvm-inliner"
|
|
|
|
using namespace mlir;
|
|
|
|
/// Check whether the given alloca is an input to a lifetime intrinsic,
|
|
/// optionally passing through one or more casts on the way. This is not
|
|
/// transitive through block arguments.
|
|
static bool hasLifetimeMarkers(LLVM::AllocaOp allocaOp) {
|
|
SmallVector<Operation *> stack(allocaOp->getUsers().begin(),
|
|
allocaOp->getUsers().end());
|
|
while (!stack.empty()) {
|
|
Operation *op = stack.pop_back_val();
|
|
if (isa<LLVM::LifetimeStartOp, LLVM::LifetimeEndOp>(op))
|
|
return true;
|
|
if (isa<LLVM::BitcastOp>(op))
|
|
stack.append(op->getUsers().begin(), op->getUsers().end());
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// Handles alloca operations in the inlined blocks:
|
|
/// - Moves all alloca operations with a constant size in the former entry block
|
|
/// of the callee into the entry block of the caller, so they become part of
|
|
/// the function prologue/epilogue during code generation.
|
|
/// - Inserts lifetime intrinsics that limit the scope of inlined static allocas
|
|
/// to the inlined blocks.
|
|
/// - Inserts StackSave and StackRestore operations if dynamic allocas were
|
|
/// inlined.
|
|
static void
|
|
handleInlinedAllocas(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
// Locate the entry block of the closest callsite ancestor that has either the
|
|
// IsolatedFromAbove or AutomaticAllocationScope trait. In pure LLVM dialect
|
|
// programs, this is the LLVMFuncOp containing the call site. However, in
|
|
// mixed-dialect programs, the callsite might be nested in another operation
|
|
// that carries one of these traits. In such scenarios, this traversal stops
|
|
// at the closest ancestor with either trait, ensuring visibility post
|
|
// relocation and respecting allocation scopes.
|
|
Block *callerEntryBlock = nullptr;
|
|
Operation *currentOp = call;
|
|
while (Operation *parentOp = currentOp->getParentOp()) {
|
|
if (parentOp->mightHaveTrait<OpTrait::IsIsolatedFromAbove>() ||
|
|
parentOp->mightHaveTrait<OpTrait::AutomaticAllocationScope>()) {
|
|
callerEntryBlock = ¤tOp->getParentRegion()->front();
|
|
break;
|
|
}
|
|
currentOp = parentOp;
|
|
}
|
|
|
|
// Avoid relocating the alloca operations if the call has been inlined into
|
|
// the entry block already, which is typically the encompassing
|
|
// LLVM function, or if the relevant entry block cannot be identified.
|
|
Block *calleeEntryBlock = &(*inlinedBlocks.begin());
|
|
if (!callerEntryBlock || callerEntryBlock == calleeEntryBlock)
|
|
return;
|
|
|
|
SmallVector<std::tuple<LLVM::AllocaOp, IntegerAttr, bool>> allocasToMove;
|
|
bool shouldInsertLifetimes = false;
|
|
bool hasDynamicAlloca = false;
|
|
// Conservatively only move static alloca operations that are part of the
|
|
// entry block and do not inspect nested regions, since they may execute
|
|
// conditionally or have other unknown semantics.
|
|
for (auto allocaOp : calleeEntryBlock->getOps<LLVM::AllocaOp>()) {
|
|
IntegerAttr arraySize;
|
|
if (!matchPattern(allocaOp.getArraySize(), m_Constant(&arraySize))) {
|
|
hasDynamicAlloca = true;
|
|
continue;
|
|
}
|
|
bool shouldInsertLifetime =
|
|
arraySize.getValue() != 0 && !hasLifetimeMarkers(allocaOp);
|
|
shouldInsertLifetimes |= shouldInsertLifetime;
|
|
allocasToMove.emplace_back(allocaOp, arraySize, shouldInsertLifetime);
|
|
}
|
|
// Check the remaining inlined blocks for dynamic allocas as well.
|
|
for (Block &block : llvm::drop_begin(inlinedBlocks)) {
|
|
if (hasDynamicAlloca)
|
|
break;
|
|
hasDynamicAlloca =
|
|
llvm::any_of(block.getOps<LLVM::AllocaOp>(), [](auto allocaOp) {
|
|
return !matchPattern(allocaOp.getArraySize(), m_Constant());
|
|
});
|
|
}
|
|
if (allocasToMove.empty() && !hasDynamicAlloca)
|
|
return;
|
|
OpBuilder builder(calleeEntryBlock, calleeEntryBlock->begin());
|
|
Value stackPtr;
|
|
if (hasDynamicAlloca) {
|
|
// This may result in multiple stacksave/stackrestore intrinsics in the same
|
|
// scope if some are already present in the body of the caller. This is not
|
|
// invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
|
|
// other cases where the stacksave/stackrestore is redundant.
|
|
stackPtr = builder.create<LLVM::StackSaveOp>(
|
|
call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
|
|
}
|
|
builder.setInsertionPoint(callerEntryBlock, callerEntryBlock->begin());
|
|
for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
|
|
auto newConstant = builder.create<LLVM::ConstantOp>(
|
|
allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
|
|
// Insert a lifetime start intrinsic where the alloca was before moving it.
|
|
if (shouldInsertLifetime) {
|
|
OpBuilder::InsertionGuard insertionGuard(builder);
|
|
builder.setInsertionPoint(allocaOp);
|
|
builder.create<LLVM::LifetimeStartOp>(
|
|
allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
|
|
allocaOp.getResult());
|
|
}
|
|
allocaOp->moveAfter(newConstant);
|
|
allocaOp.getArraySizeMutable().assign(newConstant.getResult());
|
|
}
|
|
if (!shouldInsertLifetimes && !hasDynamicAlloca)
|
|
return;
|
|
// Insert a lifetime end intrinsic before each return in the callee function.
|
|
for (Block &block : inlinedBlocks) {
|
|
if (!block.getTerminator()->hasTrait<OpTrait::ReturnLike>())
|
|
continue;
|
|
builder.setInsertionPoint(block.getTerminator());
|
|
if (hasDynamicAlloca)
|
|
builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
|
|
for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
|
|
if (shouldInsertLifetime)
|
|
builder.create<LLVM::LifetimeEndOp>(
|
|
allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
|
|
allocaOp.getResult());
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Maps all alias scopes in the inlined operations to deep clones of the scopes
|
|
/// and domain. This is required for code such as `foo(a, b); foo(a2, b2);` to
|
|
/// not incorrectly return `noalias` for e.g. operations on `a` and `a2`.
|
|
static void
|
|
deepCloneAliasScopes(iterator_range<Region::iterator> inlinedBlocks) {
|
|
DenseMap<Attribute, Attribute> mapping;
|
|
|
|
// Register handles in the walker to create the deep clones.
|
|
// The walker ensures that an attribute is only ever walked once and does a
|
|
// post-order walk, ensuring the domain is visited prior to the scope.
|
|
AttrTypeWalker walker;
|
|
|
|
// Perform the deep clones while visiting. Builders create a distinct
|
|
// attribute to make sure that new instances are always created by the
|
|
// uniquer.
|
|
walker.addWalk([&](LLVM::AliasScopeDomainAttr domainAttr) {
|
|
mapping[domainAttr] = LLVM::AliasScopeDomainAttr::get(
|
|
domainAttr.getContext(), domainAttr.getDescription());
|
|
});
|
|
|
|
walker.addWalk([&](LLVM::AliasScopeAttr scopeAttr) {
|
|
mapping[scopeAttr] = LLVM::AliasScopeAttr::get(
|
|
cast<LLVM::AliasScopeDomainAttr>(mapping.lookup(scopeAttr.getDomain())),
|
|
scopeAttr.getDescription());
|
|
});
|
|
|
|
// Map an array of scopes to an array of deep clones.
|
|
auto convertScopeList = [&](ArrayAttr arrayAttr) -> ArrayAttr {
|
|
if (!arrayAttr)
|
|
return nullptr;
|
|
|
|
// Create the deep clones if necessary.
|
|
walker.walk(arrayAttr);
|
|
|
|
return ArrayAttr::get(arrayAttr.getContext(),
|
|
llvm::map_to_vector(arrayAttr, [&](Attribute attr) {
|
|
return mapping.lookup(attr);
|
|
}));
|
|
};
|
|
|
|
for (Block &block : inlinedBlocks) {
|
|
for (Operation &op : block) {
|
|
if (auto aliasInterface = dyn_cast<LLVM::AliasAnalysisOpInterface>(op)) {
|
|
aliasInterface.setAliasScopes(
|
|
convertScopeList(aliasInterface.getAliasScopesOrNull()));
|
|
aliasInterface.setNoAliasScopes(
|
|
convertScopeList(aliasInterface.getNoAliasScopesOrNull()));
|
|
}
|
|
|
|
if (auto noAliasScope = dyn_cast<LLVM::NoAliasScopeDeclOp>(op)) {
|
|
// Create the deep clones if necessary.
|
|
walker.walk(noAliasScope.getScopeAttr());
|
|
|
|
noAliasScope.setScopeAttr(cast<LLVM::AliasScopeAttr>(
|
|
mapping.lookup(noAliasScope.getScopeAttr())));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Creates a new ArrayAttr by concatenating `lhs` with `rhs`.
|
|
/// Returns null if both parameters are null. If only one attribute is null,
|
|
/// return the other.
|
|
static ArrayAttr concatArrayAttr(ArrayAttr lhs, ArrayAttr rhs) {
|
|
if (!lhs)
|
|
return rhs;
|
|
if (!rhs)
|
|
return lhs;
|
|
|
|
SmallVector<Attribute> result;
|
|
llvm::append_range(result, lhs);
|
|
llvm::append_range(result, rhs);
|
|
return ArrayAttr::get(lhs.getContext(), result);
|
|
}
|
|
|
|
/// Attempts to return the underlying pointer value that `pointerValue` is based
|
|
/// on. This traverses down the chain of operations to the last operation
|
|
/// producing the base pointer and returns it. If it encounters an operation it
|
|
/// cannot further traverse through, returns the operation's result.
|
|
static Value getUnderlyingObject(Value pointerValue) {
|
|
while (true) {
|
|
if (auto gepOp = pointerValue.getDefiningOp<LLVM::GEPOp>()) {
|
|
pointerValue = gepOp.getBase();
|
|
continue;
|
|
}
|
|
|
|
if (auto addrCast = pointerValue.getDefiningOp<LLVM::AddrSpaceCastOp>()) {
|
|
pointerValue = addrCast.getOperand();
|
|
continue;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
return pointerValue;
|
|
}
|
|
|
|
/// Attempts to return the set of all underlying pointer values that
|
|
/// `pointerValue` is based on. This function traverses through select
|
|
/// operations and block arguments unlike getUnderlyingObject.
|
|
static SmallVector<Value> getUnderlyingObjectSet(Value pointerValue) {
|
|
SmallVector<Value> result;
|
|
|
|
SmallVector<Value> workList{pointerValue};
|
|
// Avoid dataflow loops.
|
|
SmallPtrSet<Value, 4> seen;
|
|
do {
|
|
Value current = workList.pop_back_val();
|
|
current = getUnderlyingObject(current);
|
|
|
|
if (!seen.insert(current).second)
|
|
continue;
|
|
|
|
if (auto selectOp = current.getDefiningOp<LLVM::SelectOp>()) {
|
|
workList.push_back(selectOp.getTrueValue());
|
|
workList.push_back(selectOp.getFalseValue());
|
|
continue;
|
|
}
|
|
|
|
if (auto blockArg = dyn_cast<BlockArgument>(current)) {
|
|
Block *parentBlock = blockArg.getParentBlock();
|
|
|
|
// Attempt to find all block argument operands for every predecessor.
|
|
// If any operand to the block argument wasn't found in a predecessor,
|
|
// conservatively add the block argument to the result set.
|
|
SmallVector<Value> operands;
|
|
bool anyUnknown = false;
|
|
for (auto iter = parentBlock->pred_begin();
|
|
iter != parentBlock->pred_end(); iter++) {
|
|
auto branch = dyn_cast<BranchOpInterface>((*iter)->getTerminator());
|
|
if (!branch) {
|
|
result.push_back(blockArg);
|
|
anyUnknown = true;
|
|
break;
|
|
}
|
|
|
|
Value operand = branch.getSuccessorOperands(
|
|
iter.getSuccessorIndex())[blockArg.getArgNumber()];
|
|
if (!operand) {
|
|
result.push_back(blockArg);
|
|
anyUnknown = true;
|
|
break;
|
|
}
|
|
|
|
operands.push_back(operand);
|
|
}
|
|
|
|
if (!anyUnknown)
|
|
llvm::append_range(workList, operands);
|
|
|
|
continue;
|
|
}
|
|
|
|
result.push_back(current);
|
|
} while (!workList.empty());
|
|
|
|
return result;
|
|
}
|
|
|
|
/// Creates a new AliasScopeAttr for every noalias parameter and attaches it to
|
|
/// the appropriate inlined memory operations in an attempt to preserve the
|
|
/// original semantics of the parameter attribute.
|
|
static void createNewAliasScopesFromNoAliasParameter(
|
|
Operation *call, iterator_range<Region::iterator> inlinedBlocks) {
|
|
|
|
// First collect all noalias parameters. These have been specially marked by
|
|
// the `handleArgument` implementation by using the `ssa.copy` intrinsic and
|
|
// attaching a `noalias` attribute to it.
|
|
// These are only meant to be temporary and should therefore be deleted after
|
|
// we're done using them here.
|
|
SetVector<LLVM::SSACopyOp> noAliasParams;
|
|
for (Value argument : cast<LLVM::CallOp>(call).getArgOperands()) {
|
|
for (Operation *user : argument.getUsers()) {
|
|
auto ssaCopy = llvm::dyn_cast<LLVM::SSACopyOp>(user);
|
|
if (!ssaCopy)
|
|
continue;
|
|
if (!ssaCopy->hasAttr(LLVM::LLVMDialect::getNoAliasAttrName()))
|
|
continue;
|
|
|
|
noAliasParams.insert(ssaCopy);
|
|
}
|
|
}
|
|
|
|
// If there were none, we have nothing to do here.
|
|
if (noAliasParams.empty())
|
|
return;
|
|
|
|
// Scope exit block to make it impossible to forget to get rid of the
|
|
// intrinsics.
|
|
auto exit = llvm::make_scope_exit([&] {
|
|
for (LLVM::SSACopyOp ssaCopyOp : noAliasParams) {
|
|
ssaCopyOp.replaceAllUsesWith(ssaCopyOp.getOperand());
|
|
ssaCopyOp->erase();
|
|
}
|
|
});
|
|
|
|
// Create a new domain for this specific inlining and a new scope for every
|
|
// noalias parameter.
|
|
auto functionDomain = LLVM::AliasScopeDomainAttr::get(
|
|
call->getContext(), cast<LLVM::CallOp>(call).getCalleeAttr().getAttr());
|
|
DenseMap<Value, LLVM::AliasScopeAttr> pointerScopes;
|
|
for (LLVM::SSACopyOp copyOp : noAliasParams) {
|
|
auto scope = LLVM::AliasScopeAttr::get(functionDomain);
|
|
pointerScopes[copyOp] = scope;
|
|
|
|
OpBuilder(call).create<LLVM::NoAliasScopeDeclOp>(call->getLoc(), scope);
|
|
}
|
|
|
|
// Go through every instruction and attempt to find which noalias parameters
|
|
// it is definitely based on and definitely not based on.
|
|
for (Block &inlinedBlock : inlinedBlocks) {
|
|
for (auto aliasInterface :
|
|
inlinedBlock.getOps<LLVM::AliasAnalysisOpInterface>()) {
|
|
|
|
// Collect the pointer arguments affected by the alias scopes.
|
|
SmallVector<Value> pointerArgs = aliasInterface.getAccessedOperands();
|
|
|
|
// Find the set of underlying pointers that this pointer is based on.
|
|
SmallPtrSet<Value, 4> basedOnPointers;
|
|
for (Value pointer : pointerArgs)
|
|
llvm::copy(getUnderlyingObjectSet(pointer),
|
|
std::inserter(basedOnPointers, basedOnPointers.begin()));
|
|
|
|
bool aliasesOtherKnownObject = false;
|
|
// Go through the based on pointers and check that they are either:
|
|
// * Constants that can be ignored (undef, poison, null pointer).
|
|
// * Based on a noalias parameter.
|
|
// * Other pointers that we know can't alias with our noalias parameter.
|
|
//
|
|
// Any other value might be a pointer based on any noalias parameter that
|
|
// hasn't been identified. In that case conservatively don't add any
|
|
// scopes to this operation indicating either aliasing or not aliasing
|
|
// with any parameter.
|
|
if (llvm::any_of(basedOnPointers, [&](Value object) {
|
|
if (matchPattern(object, m_Constant()))
|
|
return false;
|
|
|
|
if (noAliasParams.contains(object.getDefiningOp<LLVM::SSACopyOp>()))
|
|
return false;
|
|
|
|
// TODO: This should include other arguments from the inlined
|
|
// callable.
|
|
if (isa_and_nonnull<LLVM::AllocaOp, LLVM::AddressOfOp>(
|
|
object.getDefiningOp())) {
|
|
aliasesOtherKnownObject = true;
|
|
return false;
|
|
}
|
|
return true;
|
|
}))
|
|
continue;
|
|
|
|
// Add all noalias parameter scopes to the noalias scope list that we are
|
|
// not based on.
|
|
SmallVector<Attribute> noAliasScopes;
|
|
for (LLVM::SSACopyOp noAlias : noAliasParams) {
|
|
if (basedOnPointers.contains(noAlias))
|
|
continue;
|
|
|
|
noAliasScopes.push_back(pointerScopes[noAlias]);
|
|
}
|
|
|
|
if (!noAliasScopes.empty())
|
|
aliasInterface.setNoAliasScopes(
|
|
concatArrayAttr(aliasInterface.getNoAliasScopesOrNull(),
|
|
ArrayAttr::get(call->getContext(), noAliasScopes)));
|
|
|
|
// Don't add alias scopes to call operations or operations that might
|
|
// operate on pointers not based on any noalias parameter.
|
|
// Since we add all scopes to an operation's noalias list that it
|
|
// definitely doesn't alias, we mustn't do the same for the alias.scope
|
|
// list if other objects are involved.
|
|
//
|
|
// Consider the following case:
|
|
// %0 = llvm.alloca
|
|
// %1 = select %magic, %0, %noalias_param
|
|
// store 5, %1 (1) noalias=[scope(...)]
|
|
// ...
|
|
// store 3, %0 (2) noalias=[scope(noalias_param), scope(...)]
|
|
//
|
|
// We can add the scopes of any noalias parameters that aren't
|
|
// noalias_param's scope to (1) and add all of them to (2). We mustn't add
|
|
// the scope of noalias_param to the alias.scope list of (1) since
|
|
// that would mean (2) cannot alias with (1) which is wrong since both may
|
|
// store to %0.
|
|
//
|
|
// In conclusion, only add scopes to the alias.scope list if all pointers
|
|
// have a corresponding scope.
|
|
// Call operations are included in this list since we do not know whether
|
|
// the callee accesses any memory besides the ones passed as its
|
|
// arguments.
|
|
if (aliasesOtherKnownObject ||
|
|
isa<LLVM::CallOp>(aliasInterface.getOperation()))
|
|
continue;
|
|
|
|
SmallVector<Attribute> aliasScopes;
|
|
for (LLVM::SSACopyOp noAlias : noAliasParams)
|
|
if (basedOnPointers.contains(noAlias))
|
|
aliasScopes.push_back(pointerScopes[noAlias]);
|
|
|
|
if (!aliasScopes.empty())
|
|
aliasInterface.setAliasScopes(
|
|
concatArrayAttr(aliasInterface.getAliasScopesOrNull(),
|
|
ArrayAttr::get(call->getContext(), aliasScopes)));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Appends any alias scopes of the call operation to any inlined memory
|
|
/// operation.
|
|
static void
|
|
appendCallOpAliasScopes(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
auto callAliasInterface = dyn_cast<LLVM::AliasAnalysisOpInterface>(call);
|
|
if (!callAliasInterface)
|
|
return;
|
|
|
|
ArrayAttr aliasScopes = callAliasInterface.getAliasScopesOrNull();
|
|
ArrayAttr noAliasScopes = callAliasInterface.getNoAliasScopesOrNull();
|
|
// If the call has neither alias scopes or noalias scopes we have nothing to
|
|
// do here.
|
|
if (!aliasScopes && !noAliasScopes)
|
|
return;
|
|
|
|
// Simply append the call op's alias and noalias scopes to any operation
|
|
// implementing AliasAnalysisOpInterface.
|
|
for (Block &block : inlinedBlocks) {
|
|
for (auto aliasInterface : block.getOps<LLVM::AliasAnalysisOpInterface>()) {
|
|
if (aliasScopes)
|
|
aliasInterface.setAliasScopes(concatArrayAttr(
|
|
aliasInterface.getAliasScopesOrNull(), aliasScopes));
|
|
|
|
if (noAliasScopes)
|
|
aliasInterface.setNoAliasScopes(concatArrayAttr(
|
|
aliasInterface.getNoAliasScopesOrNull(), noAliasScopes));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Handles all interactions with alias scopes during inlining.
|
|
static void handleAliasScopes(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
deepCloneAliasScopes(inlinedBlocks);
|
|
createNewAliasScopesFromNoAliasParameter(call, inlinedBlocks);
|
|
appendCallOpAliasScopes(call, inlinedBlocks);
|
|
}
|
|
|
|
/// Appends any access groups of the call operation to any inlined memory
|
|
/// operation.
|
|
static void handleAccessGroups(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
auto callAccessGroupInterface = dyn_cast<LLVM::AccessGroupOpInterface>(call);
|
|
if (!callAccessGroupInterface)
|
|
return;
|
|
|
|
auto accessGroups = callAccessGroupInterface.getAccessGroupsOrNull();
|
|
if (!accessGroups)
|
|
return;
|
|
|
|
// Simply append the call op's access groups to any operation implementing
|
|
// AccessGroupOpInterface.
|
|
for (Block &block : inlinedBlocks)
|
|
for (auto accessGroupOpInterface :
|
|
block.getOps<LLVM::AccessGroupOpInterface>())
|
|
accessGroupOpInterface.setAccessGroups(concatArrayAttr(
|
|
accessGroupOpInterface.getAccessGroupsOrNull(), accessGroups));
|
|
}
|
|
|
|
/// If `requestedAlignment` is higher than the alignment specified on `alloca`,
|
|
/// realigns `alloca` if this does not exceed the natural stack alignment.
|
|
/// Returns the post-alignment of `alloca`, whether it was realigned or not.
|
|
static uint64_t tryToEnforceAllocaAlignment(LLVM::AllocaOp alloca,
|
|
uint64_t requestedAlignment,
|
|
DataLayout const &dataLayout) {
|
|
uint64_t allocaAlignment = alloca.getAlignment().value_or(1);
|
|
if (requestedAlignment <= allocaAlignment)
|
|
// No realignment necessary.
|
|
return allocaAlignment;
|
|
uint64_t naturalStackAlignmentBits = dataLayout.getStackAlignment();
|
|
// If the natural stack alignment is not specified, the data layout returns
|
|
// zero. Optimistically allow realignment in this case.
|
|
if (naturalStackAlignmentBits == 0 ||
|
|
// If the requested alignment exceeds the natural stack alignment, this
|
|
// will trigger a dynamic stack realignment, so we prefer to copy...
|
|
8 * requestedAlignment <= naturalStackAlignmentBits ||
|
|
// ...unless the alloca already triggers dynamic stack realignment. Then
|
|
// we might as well further increase the alignment to avoid a copy.
|
|
8 * allocaAlignment > naturalStackAlignmentBits) {
|
|
alloca.setAlignment(requestedAlignment);
|
|
allocaAlignment = requestedAlignment;
|
|
}
|
|
return allocaAlignment;
|
|
}
|
|
|
|
/// Tries to find and return the alignment of the pointer `value` by looking for
|
|
/// an alignment attribute on the defining allocation op or function argument.
|
|
/// If the found alignment is lower than `requestedAlignment`, tries to realign
|
|
/// the pointer, then returns the resulting post-alignment, regardless of
|
|
/// whether it was realigned or not. If no existing alignment attribute is
|
|
/// found, returns 1 (i.e., assume that no alignment is guaranteed).
|
|
static uint64_t tryToEnforceAlignment(Value value, uint64_t requestedAlignment,
|
|
DataLayout const &dataLayout) {
|
|
if (Operation *definingOp = value.getDefiningOp()) {
|
|
if (auto alloca = dyn_cast<LLVM::AllocaOp>(definingOp))
|
|
return tryToEnforceAllocaAlignment(alloca, requestedAlignment,
|
|
dataLayout);
|
|
if (auto addressOf = dyn_cast<LLVM::AddressOfOp>(definingOp))
|
|
if (auto global = SymbolTable::lookupNearestSymbolFrom<LLVM::GlobalOp>(
|
|
definingOp, addressOf.getGlobalNameAttr()))
|
|
return global.getAlignment().value_or(1);
|
|
// We don't currently handle this operation; assume no alignment.
|
|
return 1;
|
|
}
|
|
// Since there is no defining op, this is a block argument. Probably this
|
|
// comes directly from a function argument, so check that this is the case.
|
|
Operation *parentOp = value.getParentBlock()->getParentOp();
|
|
if (auto func = dyn_cast<LLVM::LLVMFuncOp>(parentOp)) {
|
|
// Use the alignment attribute set for this argument in the parent function
|
|
// if it has been set.
|
|
auto blockArg = llvm::cast<BlockArgument>(value);
|
|
if (Attribute alignAttr = func.getArgAttr(
|
|
blockArg.getArgNumber(), LLVM::LLVMDialect::getAlignAttrName()))
|
|
return cast<IntegerAttr>(alignAttr).getValue().getLimitedValue();
|
|
}
|
|
// We didn't find anything useful; assume no alignment.
|
|
return 1;
|
|
}
|
|
|
|
/// Introduces a new alloca and copies the memory pointed to by `argument` to
|
|
/// the address of the new alloca, then returns the value of the new alloca.
|
|
static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
|
|
Value argument, Type elementType,
|
|
uint64_t elementTypeSize,
|
|
uint64_t targetAlignment) {
|
|
// Allocate the new value on the stack.
|
|
Value allocaOp;
|
|
{
|
|
// Since this is a static alloca, we can put it directly in the entry block,
|
|
// so they can be absorbed into the prologue/epilogue at code generation.
|
|
OpBuilder::InsertionGuard insertionGuard(builder);
|
|
Block *entryBlock = &(*argument.getParentRegion()->begin());
|
|
builder.setInsertionPointToStart(entryBlock);
|
|
Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
|
|
builder.getI64IntegerAttr(1));
|
|
allocaOp = builder.create<LLVM::AllocaOp>(
|
|
loc, argument.getType(), elementType, one, targetAlignment);
|
|
}
|
|
// Copy the pointee to the newly allocated value.
|
|
Value copySize = builder.create<LLVM::ConstantOp>(
|
|
loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
|
|
builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
|
|
/*isVolatile=*/false);
|
|
return allocaOp;
|
|
}
|
|
|
|
/// Handles a function argument marked with the byval attribute by introducing a
|
|
/// memcpy or realigning the defining operation, if required either due to the
|
|
/// pointee being writeable in the callee, and/or due to an alignment mismatch.
|
|
/// `requestedAlignment` specifies the alignment set in the "align" argument
|
|
/// attribute (or 1 if no align attribute was set).
|
|
static Value handleByValArgument(OpBuilder &builder, Operation *callable,
|
|
Value argument, Type elementType,
|
|
uint64_t requestedAlignment) {
|
|
auto func = cast<LLVM::LLVMFuncOp>(callable);
|
|
LLVM::MemoryEffectsAttr memoryEffects = func.getMemoryAttr();
|
|
// If there is no memory effects attribute, assume that the function is
|
|
// not read-only.
|
|
bool isReadOnly = memoryEffects &&
|
|
memoryEffects.getArgMem() != LLVM::ModRefInfo::ModRef &&
|
|
memoryEffects.getArgMem() != LLVM::ModRefInfo::Mod;
|
|
// Check if there's an alignment mismatch requiring us to copy.
|
|
DataLayout dataLayout = DataLayout::closest(callable);
|
|
uint64_t minimumAlignment = dataLayout.getTypeABIAlignment(elementType);
|
|
if (isReadOnly) {
|
|
if (requestedAlignment <= minimumAlignment)
|
|
return argument;
|
|
uint64_t currentAlignment =
|
|
tryToEnforceAlignment(argument, requestedAlignment, dataLayout);
|
|
if (currentAlignment >= requestedAlignment)
|
|
return argument;
|
|
}
|
|
uint64_t targetAlignment = std::max(requestedAlignment, minimumAlignment);
|
|
return handleByValArgumentInit(builder, func.getLoc(), argument, elementType,
|
|
dataLayout.getTypeSize(elementType),
|
|
targetAlignment);
|
|
}
|
|
|
|
namespace {
|
|
struct LLVMInlinerInterface : public DialectInlinerInterface {
|
|
using DialectInlinerInterface::DialectInlinerInterface;
|
|
|
|
LLVMInlinerInterface(Dialect *dialect)
|
|
: DialectInlinerInterface(dialect),
|
|
// Cache set of StringAttrs for fast lookup in `isLegalToInline`.
|
|
disallowedFunctionAttrs({
|
|
StringAttr::get(dialect->getContext(), "noduplicate"),
|
|
StringAttr::get(dialect->getContext(), "noinline"),
|
|
StringAttr::get(dialect->getContext(), "optnone"),
|
|
StringAttr::get(dialect->getContext(), "presplitcoroutine"),
|
|
StringAttr::get(dialect->getContext(), "returns_twice"),
|
|
StringAttr::get(dialect->getContext(), "strictfp"),
|
|
}) {}
|
|
|
|
bool isLegalToInline(Operation *call, Operation *callable,
|
|
bool wouldBeCloned) const final {
|
|
if (!wouldBeCloned)
|
|
return false;
|
|
if (!isa<LLVM::CallOp>(call)) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline: call is not an LLVM::CallOp\n");
|
|
return false;
|
|
}
|
|
auto funcOp = dyn_cast<LLVM::LLVMFuncOp>(callable);
|
|
if (!funcOp) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline: callable is not an LLVM::LLVMFuncOp\n");
|
|
return false;
|
|
}
|
|
if (funcOp.isVarArg()) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline: callable is variadic\n");
|
|
return false;
|
|
}
|
|
// TODO: Generate aliasing metadata from noalias argument/result attributes.
|
|
if (auto attrs = funcOp.getArgAttrs()) {
|
|
for (DictionaryAttr attrDict : attrs->getAsRange<DictionaryAttr>()) {
|
|
if (attrDict.contains(LLVM::LLVMDialect::getInAllocaAttrName())) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline " << funcOp.getSymName()
|
|
<< ": inalloca arguments not supported\n");
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
// TODO: Handle exceptions.
|
|
if (funcOp.getPersonality()) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline " << funcOp.getSymName()
|
|
<< ": unhandled function personality\n");
|
|
return false;
|
|
}
|
|
if (funcOp.getPassthrough()) {
|
|
// TODO: Used attributes should not be passthrough.
|
|
if (llvm::any_of(*funcOp.getPassthrough(), [&](Attribute attr) {
|
|
auto stringAttr = dyn_cast<StringAttr>(attr);
|
|
if (!stringAttr)
|
|
return false;
|
|
if (disallowedFunctionAttrs.contains(stringAttr)) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline " << funcOp.getSymName()
|
|
<< ": found disallowed function attribute "
|
|
<< stringAttr << "\n");
|
|
return true;
|
|
}
|
|
return false;
|
|
}))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool isLegalToInline(Region *, Region *, bool, IRMapping &) const final {
|
|
return true;
|
|
}
|
|
|
|
bool isLegalToInline(Operation *op, Region *, bool, IRMapping &) const final {
|
|
// The inliner cannot handle variadic function arguments.
|
|
return !isa<LLVM::VaStartOp>(op);
|
|
}
|
|
|
|
/// Handle the given inlined return by replacing it with a branch. This
|
|
/// overload is called when the inlined region has more than one block.
|
|
void handleTerminator(Operation *op, Block *newDest) const final {
|
|
// Only return needs to be handled here.
|
|
auto returnOp = dyn_cast<LLVM::ReturnOp>(op);
|
|
if (!returnOp)
|
|
return;
|
|
|
|
// Replace the return with a branch to the dest.
|
|
OpBuilder builder(op);
|
|
builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
|
|
op->erase();
|
|
}
|
|
|
|
/// Handle the given inlined return by replacing the uses of the call with the
|
|
/// operands of the return. This overload is called when the inlined region
|
|
/// only contains one block.
|
|
void handleTerminator(Operation *op, ValueRange valuesToRepl) const final {
|
|
// Return will be the only terminator present.
|
|
auto returnOp = cast<LLVM::ReturnOp>(op);
|
|
|
|
// Replace the values directly with the return operands.
|
|
assert(returnOp.getNumOperands() == valuesToRepl.size());
|
|
for (auto [dst, src] : llvm::zip(valuesToRepl, returnOp.getOperands()))
|
|
dst.replaceAllUsesWith(src);
|
|
}
|
|
|
|
Value handleArgument(OpBuilder &builder, Operation *call, Operation *callable,
|
|
Value argument,
|
|
DictionaryAttr argumentAttrs) const final {
|
|
if (std::optional<NamedAttribute> attr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getByValAttrName())) {
|
|
Type elementType = cast<TypeAttr>(attr->getValue()).getValue();
|
|
uint64_t requestedAlignment = 1;
|
|
if (std::optional<NamedAttribute> alignAttr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getAlignAttrName())) {
|
|
requestedAlignment = cast<IntegerAttr>(alignAttr->getValue())
|
|
.getValue()
|
|
.getLimitedValue();
|
|
}
|
|
return handleByValArgument(builder, callable, argument, elementType,
|
|
requestedAlignment);
|
|
}
|
|
if ([[maybe_unused]] std::optional<NamedAttribute> attr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getNoAliasAttrName())) {
|
|
if (argument.use_empty())
|
|
return argument;
|
|
|
|
// This code is essentially a workaround for deficiencies in the
|
|
// inliner interface: We need to transform operations *after* inlined
|
|
// based on the argument attributes of the parameters *before* inlining.
|
|
// This method runs prior to actual inlining and thus cannot transform the
|
|
// post-inlining code, while `processInlinedCallBlocks` does not have
|
|
// access to pre-inlining function arguments. Additionally, it is required
|
|
// to distinguish which parameter an SSA value originally came from.
|
|
// As a workaround until this is changed: Create an ssa.copy intrinsic
|
|
// with the noalias attribute that can easily be found, and is extremely
|
|
// unlikely to exist in the code prior to inlining, using this to
|
|
// communicate between this method and `processInlinedCallBlocks`.
|
|
// TODO: Fix this by refactoring the inliner interface.
|
|
auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
|
|
copyOp->setDiscardableAttr(
|
|
builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
|
|
builder.getUnitAttr());
|
|
return copyOp;
|
|
}
|
|
return argument;
|
|
}
|
|
|
|
void processInlinedCallBlocks(
|
|
Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) const override {
|
|
handleInlinedAllocas(call, inlinedBlocks);
|
|
handleAliasScopes(call, inlinedBlocks);
|
|
handleAccessGroups(call, inlinedBlocks);
|
|
}
|
|
|
|
// Keeping this (immutable) state on the interface allows us to look up
|
|
// StringAttrs instead of looking up strings, since StringAttrs are bound to
|
|
// the current context and thus cannot be initialized as static fields.
|
|
const DenseSet<StringAttr> disallowedFunctionAttrs;
|
|
};
|
|
|
|
} // end anonymous namespace
|
|
|
|
void LLVM::detail::addLLVMInlinerInterface(LLVM::LLVMDialect *dialect) {
|
|
dialect->addInterfaces<LLVMInlinerInterface>();
|
|
}
|