caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1bf05fbc987dddebe94de7b33d810d8221eda1a5
clang-p2996/polly/lib/CodeGen/IslNodeBuilder.cpp
Michael Kruse d5c87162db [Polly] Use VirtualUse to determine references. 
VirtualUse ensures consistency over different source of values with
Polly. In particular, this enables its use of instructions moved between
Statement. Before the patch, the code wrongly assumed that the BB's
instructions are also the ScopStmt's instructions. Reference are
determined for OpenMP outlining and GPGPU kernel extraction.

GPGPU CodeGen had some problems. For one, it generated GPU kernel
parameters for constants. Second, it emitted GPU-side invariant loads
which have already been loaded by the host. This has been partially
fixed, it still generates a store for the invariant load result, but
using the value that the host has already written.

WARNING: I did not test the generated PollyACC code on an actual GPU.

The improved consistency will be made use of in the next patch.
2021-09-26 03:26:43 -05:00

1662 lines
60 KiB
C++
Raw Blame History

//===- IslNodeBuilder.cpp - Translate an isl AST into a LLVM-IR AST -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the IslNodeBuilder, a class to translate an isl AST into
// a LLVM-IR AST.
//
//===----------------------------------------------------------------------===//
#include "polly/CodeGen/IslNodeBuilder.h"
#include "polly/CodeGen/BlockGenerators.h"
#include "polly/CodeGen/CodeGeneration.h"
#include "polly/CodeGen/IslAst.h"
#include "polly/CodeGen/IslExprBuilder.h"
#include "polly/CodeGen/LoopGeneratorsGOMP.h"
#include "polly/CodeGen/LoopGeneratorsKMP.h"
#include "polly/CodeGen/RuntimeDebugBuilder.h"
#include "polly/Options.h"
#include "polly/ScopInfo.h"
#include "polly/Support/ISLTools.h"
#include "polly/Support/SCEVValidator.h"
#include "polly/Support/ScopHelper.h"
#include "polly/Support/VirtualInstruction.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/RegionInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "isl/aff.h"
#include "isl/aff_type.h"
#include "isl/ast.h"
#include "isl/ast_build.h"
#include "isl/isl-noexceptions.h"
#include "isl/map.h"
#include "isl/set.h"
#include "isl/union_map.h"
#include "isl/union_set.h"
#include "isl/val.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <string>
#include <utility>
#include <vector>
using namespace llvm;
using namespace polly;
#define DEBUG_TYPE "polly-codegen"
STATISTIC(VersionedScops, "Number of SCoPs that required versioning.");
STATISTIC(SequentialLoops, "Number of generated sequential for-loops");
STATISTIC(ParallelLoops, "Number of generated parallel for-loops");
STATISTIC(VectorLoops, "Number of generated vector for-loops");
STATISTIC(IfConditions, "Number of generated if-conditions");
/// OpenMP backend options
enum class OpenMPBackend { GNU, LLVM };
static cl::opt<bool> PollyGenerateRTCPrint(
"polly-codegen-emit-rtc-print",
cl::desc("Emit code that prints the runtime check result dynamically."),
cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
// If this option is set we always use the isl AST generator to regenerate
// memory accesses. Without this option set we regenerate expressions using the
// original SCEV expressions and only generate new expressions in case the
// access relation has been changed and consequently must be regenerated.
static cl::opt<bool> PollyGenerateExpressions(
"polly-codegen-generate-expressions",
cl::desc("Generate AST expressions for unmodified and modified accesses"),
cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
static cl::opt<int> PollyTargetFirstLevelCacheLineSize(
"polly-target-first-level-cache-line-size",
cl::desc("The size of the first level cache line size specified in bytes."),
cl::Hidden, cl::init(64), cl::ZeroOrMore, cl::cat(PollyCategory));
static cl::opt<OpenMPBackend> PollyOmpBackend(
"polly-omp-backend", cl::desc("Choose the OpenMP library to use:"),
cl::values(clEnumValN(OpenMPBackend::GNU, "GNU", "GNU OpenMP"),
clEnumValN(OpenMPBackend::LLVM, "LLVM", "LLVM OpenMP")),
cl::Hidden, cl::init(OpenMPBackend::GNU), cl::cat(PollyCategory));
isl::ast_expr IslNodeBuilder::getUpperBound(isl::ast_node_for For,
ICmpInst::Predicate &Predicate) {
isl::ast_expr Cond = For.cond();
isl::ast_expr Iterator = For.iterator();
assert(isl_ast_expr_get_type(Cond.get()) == isl_ast_expr_op &&
"conditional expression is not an atomic upper bound");
isl_ast_op_type OpType = isl_ast_expr_get_op_type(Cond.get());
switch (OpType) {
case isl_ast_op_le:
Predicate = ICmpInst::ICMP_SLE;
break;
case isl_ast_op_lt:
Predicate = ICmpInst::ICMP_SLT;
break;
default:
llvm_unreachable("Unexpected comparison type in loop condition");
}
isl::ast_expr Arg0 = Cond.get_op_arg(0);
assert(isl_ast_expr_get_type(Arg0.get()) == isl_ast_expr_id &&
"conditional expression is not an atomic upper bound");
isl::id UBID = Arg0.get_id();
assert(isl_ast_expr_get_type(Iterator.get()) == isl_ast_expr_id &&
"Could not get the iterator");
isl::id IteratorID = Iterator.get_id();
assert(UBID.get() == IteratorID.get() &&
"conditional expression is not an atomic upper bound");
return Cond.get_op_arg(1);
}
/// Return true if a return value of Predicate is true for the value represented
/// by passed isl_ast_expr_int.
static bool checkIslAstExprInt(__isl_take isl_ast_expr *Expr,
isl_bool (*Predicate)(__isl_keep isl_val *)) {
if (isl_ast_expr_get_type(Expr) != isl_ast_expr_int) {
isl_ast_expr_free(Expr);
return false;
}
auto ExprVal = isl_ast_expr_get_val(Expr);
isl_ast_expr_free(Expr);
if (Predicate(ExprVal) != isl_bool_true) {
isl_val_free(ExprVal);
return false;
}
isl_val_free(ExprVal);
return true;
}
int IslNodeBuilder::getNumberOfIterations(isl::ast_node_for For) {
assert(isl_ast_node_get_type(For.get()) == isl_ast_node_for);
isl::ast_node Body = For.body();
// First, check if we can actually handle this code.
switch (isl_ast_node_get_type(Body.get())) {
case isl_ast_node_user:
break;
case isl_ast_node_block: {
isl::ast_node_block BodyBlock = Body.as<isl::ast_node_block>();
isl::ast_node_list List = BodyBlock.children();
for (isl::ast_node Node : List) {
isl_ast_node_type NodeType = isl_ast_node_get_type(Node.get());
if (NodeType != isl_ast_node_user)
return -1;
}
break;
}
default:
return -1;
}
isl::ast_expr Init = For.init();
if (!checkIslAstExprInt(Init.release(), isl_val_is_zero))
return -1;
isl::ast_expr Inc = For.inc();
if (!checkIslAstExprInt(Inc.release(), isl_val_is_one))
return -1;
CmpInst::Predicate Predicate;
isl::ast_expr UB = getUpperBound(For, Predicate);
if (isl_ast_expr_get_type(UB.get()) != isl_ast_expr_int)
return -1;
isl::val UpVal = UB.get_val();
int NumberIterations = UpVal.get_num_si();
if (NumberIterations < 0)
return -1;
if (Predicate == CmpInst::ICMP_SLT)
return NumberIterations;
else
return NumberIterations + 1;
}
static void findReferencesByUse(Value *SrcVal, ScopStmt *UserStmt,
Loop *UserScope, const ValueMapT &GlobalMap,
SetVector<Value *> &Values,
SetVector<const SCEV *> &SCEVs) {
VirtualUse VUse = VirtualUse::create(UserStmt, UserScope, SrcVal, true);
switch (VUse.getKind()) {
case VirtualUse::Constant:
// When accelerator-offloading, GlobalValue is a host address whose content
// must still be transferred to the GPU.
if (isa<GlobalValue>(SrcVal))
Values.insert(SrcVal);
break;
case VirtualUse::Synthesizable:
SCEVs.insert(VUse.getScevExpr());
return;
case VirtualUse::Block:
case VirtualUse::ReadOnly:
case VirtualUse::Hoisted:
case VirtualUse::Intra:
case VirtualUse::Inter:
break;
}
if (Value *NewVal = GlobalMap.lookup(SrcVal))
Values.insert(NewVal);
}
static void findReferencesInInst(Instruction *Inst, ScopStmt *UserStmt,
Loop *UserScope, const ValueMapT &GlobalMap,
SetVector<Value *> &Values,
SetVector<const SCEV *> &SCEVs) {
for (Use &U : Inst->operands())
findReferencesByUse(U.get(), UserStmt, UserScope, GlobalMap, Values, SCEVs);
}
static void findReferencesInStmt(ScopStmt *Stmt, SetVector<Value *> &Values,
ValueMapT &GlobalMap,
SetVector<const SCEV *> &SCEVs) {
LoopInfo *LI = Stmt->getParent()->getLI();
BasicBlock *BB = Stmt->getBasicBlock();
Loop *Scope = LI->getLoopFor(BB);
for (Instruction *Inst : Stmt->getInstructions())
findReferencesInInst(Inst, Stmt, Scope, GlobalMap, Values, SCEVs);
if (Stmt->isRegionStmt()) {
for (BasicBlock *BB : Stmt->getRegion()->blocks()) {
Loop *Scope = LI->getLoopFor(BB);
for (Instruction &Inst : *BB)
findReferencesInInst(&Inst, Stmt, Scope, GlobalMap, Values, SCEVs);
}
}
}
void polly::addReferencesFromStmt(ScopStmt *Stmt, void *UserPtr,
bool CreateScalarRefs) {
auto &References = *static_cast<struct SubtreeReferences *>(UserPtr);
findReferencesInStmt(Stmt, References.Values, References.GlobalMap,
References.SCEVs);
for (auto &Access : *Stmt) {
if (References.ParamSpace) {
isl::space ParamSpace = Access->getLatestAccessRelation().get_space();
(*References.ParamSpace) =
References.ParamSpace->align_params(ParamSpace);
}
if (Access->isLatestArrayKind()) {
auto *BasePtr = Access->getLatestScopArrayInfo()->getBasePtr();
if (Instruction *OpInst = dyn_cast<Instruction>(BasePtr))
if (Stmt->getParent()->contains(OpInst))
continue;
References.Values.insert(BasePtr);
continue;
}
if (CreateScalarRefs)
References.Values.insert(References.BlockGen.getOrCreateAlloca(*Access));
}
}
/// Extract the out-of-scop values and SCEVs referenced from a set describing
/// a ScopStmt.
///
/// This includes the SCEVUnknowns referenced by the SCEVs used in the
/// statement and the base pointers of the memory accesses. For scalar
/// statements we force the generation of alloca memory locations and list
/// these locations in the set of out-of-scop values as well.
///
/// @param Set A set which references the ScopStmt we are interested in.
/// @param UserPtr A void pointer that can be casted to a SubtreeReferences
/// structure.
static void addReferencesFromStmtSet(isl::set Set,
struct SubtreeReferences *UserPtr) {
isl::id Id = Set.get_tuple_id();
auto *Stmt = static_cast<ScopStmt *>(Id.get_user());
addReferencesFromStmt(Stmt, UserPtr);
}
/// Extract the out-of-scop values and SCEVs referenced from a union set
/// referencing multiple ScopStmts.
///
/// This includes the SCEVUnknowns referenced by the SCEVs used in the
/// statement and the base pointers of the memory accesses. For scalar
/// statements we force the generation of alloca memory locations and list
/// these locations in the set of out-of-scop values as well.
///
/// @param USet A union set referencing the ScopStmts we are interested
/// in.
/// @param References The SubtreeReferences data structure through which
/// results are returned and further information is
/// provided.
static void
addReferencesFromStmtUnionSet(isl::union_set USet,
struct SubtreeReferences &References) {
for (isl::set Set : USet.get_set_list())
addReferencesFromStmtSet(Set, &References);
}
isl::union_map
IslNodeBuilder::getScheduleForAstNode(const isl::ast_node &Node) {
return IslAstInfo::getSchedule(Node);
}
void IslNodeBuilder::getReferencesInSubtree(const isl::ast_node &For,
SetVector<Value *> &Values,
SetVector<const Loop *> &Loops) {
SetVector<const SCEV *> SCEVs;
struct SubtreeReferences References = {
LI, SE, S, ValueMap, Values, SCEVs, getBlockGenerator(), nullptr};
for (const auto &I : IDToValue)
Values.insert(I.second);
// NOTE: this is populated in IslNodeBuilder::addParameters
for (const auto &I : OutsideLoopIterations)
Values.insert(cast<SCEVUnknown>(I.second)->getValue());
isl::union_set Schedule = getScheduleForAstNode(For).domain();
addReferencesFromStmtUnionSet(Schedule, References);
for (const SCEV *Expr : SCEVs) {
findValues(Expr, SE, Values);
findLoops(Expr, Loops);
}
Values.remove_if([](const Value *V) { return isa<GlobalValue>(V); });
/// Note: Code generation of induction variables of loops outside Scops
///
/// Remove loops that contain the scop or that are part of the scop, as they
/// are considered local. This leaves only loops that are before the scop, but
/// do not contain the scop itself.
/// We ignore loops perfectly contained in the Scop because these are already
/// generated at `IslNodeBuilder::addParameters`. These `Loops` are loops
/// whose induction variables are referred to by the Scop, but the Scop is not
/// fully contained in these Loops. Since there can be many of these,
/// we choose to codegen these on-demand.
/// @see IslNodeBuilder::materializeNonScopLoopInductionVariable.
Loops.remove_if([this](const Loop *L) {
return S.contains(L) || L->contains(S.getEntry());
});
// Contains Values that may need to be replaced with other values
// due to replacements from the ValueMap. We should make sure
// that we return correctly remapped values.
// NOTE: this code path is tested by:
// 1. test/Isl/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
// 2. test/Isl/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
SetVector<Value *> ReplacedValues;
for (Value *V : Values) {
ReplacedValues.insert(getLatestValue(V));
}
Values = ReplacedValues;
}
void IslNodeBuilder::updateValues(ValueMapT &NewValues) {
SmallPtrSet<Value *, 5> Inserted;
for (const auto &I : IDToValue) {
IDToValue[I.first] = NewValues[I.second];
Inserted.insert(I.second);
}
for (const auto &I : NewValues) {
if (Inserted.count(I.first))
continue;
ValueMap[I.first] = I.second;
}
}
Value *IslNodeBuilder::getLatestValue(Value *Original) const {
auto It = ValueMap.find(Original);
if (It == ValueMap.end())
return Original;
return It->second;
}
void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User,
std::vector<Value *> &IVS,
__isl_take isl_id *IteratorID,
__isl_take isl_union_map *Schedule) {
isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
isl_id *Id = isl_ast_expr_get_id(StmtExpr);
isl_ast_expr_free(StmtExpr);
ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id);
std::vector<LoopToScevMapT> VLTS(IVS.size());
isl_union_set *Domain = isl_union_set_from_set(Stmt->getDomain().release());
Schedule = isl_union_map_intersect_domain(Schedule, Domain);
isl_map *S = isl_map_from_union_map(Schedule);
auto *NewAccesses = createNewAccesses(Stmt, User);
createSubstitutionsVector(Expr, Stmt, VLTS, IVS, IteratorID);
VectorBlockGenerator::generate(BlockGen, *Stmt, VLTS, S, NewAccesses);
isl_id_to_ast_expr_free(NewAccesses);
isl_map_free(S);
isl_id_free(Id);
isl_ast_node_free(User);
}
void IslNodeBuilder::createMark(__isl_take isl_ast_node *Node) {
auto *Id = isl_ast_node_mark_get_id(Node);
auto Child = isl_ast_node_mark_get_node(Node);
isl_ast_node_free(Node);
// If a child node of a 'SIMD mark' is a loop that has a single iteration,
// it will be optimized away and we should skip it.
if (strcmp(isl_id_get_name(Id), "SIMD") == 0 &&
isl_ast_node_get_type(Child) == isl_ast_node_for) {
bool Vector = PollyVectorizerChoice == VECTORIZER_POLLY;
int VectorWidth =
getNumberOfIterations(isl::manage_copy(Child).as<isl::ast_node_for>());
if (Vector && 1 < VectorWidth && VectorWidth <= 16)
createForVector(Child, VectorWidth);
else
createForSequential(isl::manage(Child).as<isl::ast_node_for>(), true);
isl_id_free(Id);
return;
}
BandAttr *ChildLoopAttr = getLoopAttr(isl::manage_copy(Id));
BandAttr *AncestorLoopAttr;
if (ChildLoopAttr) {
// Save current LoopAttr environment to restore again when leaving this
// subtree. This means there was no loop between the ancestor LoopAttr and
// this mark, i.e. the ancestor LoopAttr did not directly mark a loop. This
// can happen e.g. if the AST build peeled or unrolled the loop.
AncestorLoopAttr = Annotator.getStagingAttrEnv();
Annotator.getStagingAttrEnv() = ChildLoopAttr;
}
create(Child);
if (ChildLoopAttr) {
assert(Annotator.getStagingAttrEnv() == ChildLoopAttr &&
"Nest must not overwrite loop attr environment");
Annotator.getStagingAttrEnv() = AncestorLoopAttr;
}
isl_id_free(Id);
}
void IslNodeBuilder::createForVector(__isl_take isl_ast_node *For,
int VectorWidth) {
isl_ast_node *Body = isl_ast_node_for_get_body(For);
isl_ast_expr *Init = isl_ast_node_for_get_init(For);
isl_ast_expr *Inc = isl_ast_node_for_get_inc(For);
isl_ast_expr *Iterator = isl_ast_node_for_get_iterator(For);
isl_id *IteratorID = isl_ast_expr_get_id(Iterator);
Value *ValueLB = ExprBuilder.create(Init);
Value *ValueInc = ExprBuilder.create(Inc);
Type *MaxType = ExprBuilder.getType(Iterator);
MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
if (MaxType != ValueLB->getType())
ValueLB = Builder.CreateSExt(ValueLB, MaxType);
if (MaxType != ValueInc->getType())
ValueInc = Builder.CreateSExt(ValueInc, MaxType);
std::vector<Value *> IVS(VectorWidth);
IVS[0] = ValueLB;
for (int i = 1; i < VectorWidth; i++)
IVS[i] = Builder.CreateAdd(IVS[i - 1], ValueInc, "p_vector_iv");
isl::union_map Schedule = getScheduleForAstNode(isl::manage_copy(For));
assert(!Schedule.is_null() &&
"For statement annotation does not contain its schedule");
IDToValue[IteratorID] = ValueLB;
switch (isl_ast_node_get_type(Body)) {
case isl_ast_node_user:
createUserVector(Body, IVS, isl_id_copy(IteratorID), Schedule.copy());
break;
case isl_ast_node_block: {
isl_ast_node_list *List = isl_ast_node_block_get_children(Body);
for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
createUserVector(isl_ast_node_list_get_ast_node(List, i), IVS,
isl_id_copy(IteratorID), Schedule.copy());
isl_ast_node_free(Body);
isl_ast_node_list_free(List);
break;
}
default:
isl_ast_node_dump(Body);
llvm_unreachable("Unhandled isl_ast_node in vectorizer");
}
IDToValue.erase(IDToValue.find(IteratorID));
isl_id_free(IteratorID);
isl_ast_node_free(For);
isl_ast_expr_free(Iterator);
VectorLoops++;
}
/// Restore the initial ordering of dimensions of the band node
///
/// In case the band node represents all the dimensions of the iteration
/// domain, recreate the band node to restore the initial ordering of the
/// dimensions.
///
/// @param Node The band node to be modified.
/// @return The modified schedule node.
static bool IsLoopVectorizerDisabled(isl::ast_node_for Node) {
assert(isl_ast_node_get_type(Node.get()) == isl_ast_node_for);
isl::ast_node Body = Node.body();
if (isl_ast_node_get_type(Body.get()) != isl_ast_node_mark)
return false;
isl::ast_node_mark BodyMark = Body.as<isl::ast_node_mark>();
auto Id = BodyMark.id();
if (strcmp(Id.get_name().c_str(), "Loop Vectorizer Disabled") == 0)
return true;
return false;
}
void IslNodeBuilder::createForSequential(isl::ast_node_for For,
bool MarkParallel) {
Value *ValueLB, *ValueUB, *ValueInc;
Type *MaxType;
BasicBlock *ExitBlock;
Value *IV;
CmpInst::Predicate Predicate;
bool LoopVectorizerDisabled = IsLoopVectorizerDisabled(For);
isl::ast_node Body = For.body();
// isl_ast_node_for_is_degenerate(For)
//
// TODO: For degenerated loops we could generate a plain assignment.
// However, for now we just reuse the logic for normal loops, which will
// create a loop with a single iteration.
isl::ast_expr Init = For.init();
isl::ast_expr Inc = For.inc();
isl::ast_expr Iterator = For.iterator();
isl::id IteratorID = Iterator.get_id();
isl::ast_expr UB = getUpperBound(For, Predicate);
ValueLB = ExprBuilder.create(Init.release());
ValueUB = ExprBuilder.create(UB.release());
ValueInc = ExprBuilder.create(Inc.release());
MaxType = ExprBuilder.getType(Iterator.get());
MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
MaxType = ExprBuilder.getWidestType(MaxType, ValueUB->getType());
MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
if (MaxType != ValueLB->getType())
ValueLB = Builder.CreateSExt(ValueLB, MaxType);
if (MaxType != ValueUB->getType())
ValueUB = Builder.CreateSExt(ValueUB, MaxType);
if (MaxType != ValueInc->getType())
ValueInc = Builder.CreateSExt(ValueInc, MaxType);
// If we can show that LB <Predicate> UB holds at least once, we can
// omit the GuardBB in front of the loop.
bool UseGuardBB =
!SE.isKnownPredicate(Predicate, SE.getSCEV(ValueLB), SE.getSCEV(ValueUB));
IV = createLoop(ValueLB, ValueUB, ValueInc, Builder, LI, DT, ExitBlock,
Predicate, &Annotator, MarkParallel, UseGuardBB,
LoopVectorizerDisabled);
IDToValue[IteratorID.get()] = IV;
create(Body.release());
Annotator.popLoop(MarkParallel);
IDToValue.erase(IDToValue.find(IteratorID.get()));
Builder.SetInsertPoint(&ExitBlock->front());
SequentialLoops++;
}
/// Remove the BBs contained in a (sub)function from the dominator tree.
///
/// This function removes the basic blocks that are part of a subfunction from
/// the dominator tree. Specifically, when generating code it may happen that at
/// some point the code generation continues in a new sub-function (e.g., when
/// generating OpenMP code). The basic blocks that are created in this
/// sub-function are then still part of the dominator tree of the original
/// function, such that the dominator tree reaches over function boundaries.
/// This is not only incorrect, but also causes crashes. This function now
/// removes from the dominator tree all basic blocks that are dominated (and
/// consequently reachable) from the entry block of this (sub)function.
///
/// FIXME: A LLVM (function or region) pass should not touch anything outside of
/// the function/region it runs on. Hence, the pure need for this function shows
/// that we do not comply to this rule. At the moment, this does not cause any
/// issues, but we should be aware that such issues may appear. Unfortunately
/// the current LLVM pass infrastructure does not allow to make Polly a module
/// or call-graph pass to solve this issue, as such a pass would not have access
/// to the per-function analyses passes needed by Polly. A future pass manager
/// infrastructure is supposed to enable such kind of access possibly allowing
/// us to create a cleaner solution here.
///
/// FIXME: Instead of adding the dominance information and then dropping it
/// later on, we should try to just not add it in the first place. This requires
/// some careful testing to make sure this does not break in interaction with
/// the SCEVBuilder and SplitBlock which may rely on the dominator tree or
/// which may try to update it.
///
/// @param F The function which contains the BBs to removed.
/// @param DT The dominator tree from which to remove the BBs.
static void removeSubFuncFromDomTree(Function *F, DominatorTree &DT) {
DomTreeNode *N = DT.getNode(&F->getEntryBlock());
std::vector<BasicBlock *> Nodes;
// We can only remove an element from the dominator tree, if all its children
// have been removed. To ensure this we obtain the list of nodes to remove
// using a post-order tree traversal.
for (po_iterator<DomTreeNode *> I = po_begin(N), E = po_end(N); I != E; ++I)
Nodes.push_back(I->getBlock());
for (BasicBlock *BB : Nodes)
DT.eraseNode(BB);
}
void IslNodeBuilder::createForParallel(__isl_take isl_ast_node *For) {
isl_ast_node *Body;
isl_ast_expr *Init, *Inc, *Iterator, *UB;
isl_id *IteratorID;
Value *ValueLB, *ValueUB, *ValueInc;
Type *MaxType;
Value *IV;
CmpInst::Predicate Predicate;
// The preamble of parallel code interacts different than normal code with
// e.g., scalar initialization. Therefore, we ensure the parallel code is
// separated from the last basic block.
BasicBlock *ParBB = SplitBlock(Builder.GetInsertBlock(),
&*Builder.GetInsertPoint(), &DT, &LI);
ParBB->setName("polly.parallel.for");
Builder.SetInsertPoint(&ParBB->front());
Body = isl_ast_node_for_get_body(For);
Init = isl_ast_node_for_get_init(For);
Inc = isl_ast_node_for_get_inc(For);
Iterator = isl_ast_node_for_get_iterator(For);
IteratorID = isl_ast_expr_get_id(Iterator);
UB = getUpperBound(isl::manage_copy(For).as<isl::ast_node_for>(), Predicate)
.release();
ValueLB = ExprBuilder.create(Init);
ValueUB = ExprBuilder.create(UB);
ValueInc = ExprBuilder.create(Inc);
// OpenMP always uses SLE. In case the isl generated AST uses a SLT
// expression, we need to adjust the loop bound by one.
if (Predicate == CmpInst::ICMP_SLT)
ValueUB = Builder.CreateAdd(
ValueUB, Builder.CreateSExt(Builder.getTrue(), ValueUB->getType()));
MaxType = ExprBuilder.getType(Iterator);
MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
MaxType = ExprBuilder.getWidestType(MaxType, ValueUB->getType());
MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
if (MaxType != ValueLB->getType())
ValueLB = Builder.CreateSExt(ValueLB, MaxType);
if (MaxType != ValueUB->getType())
ValueUB = Builder.CreateSExt(ValueUB, MaxType);
if (MaxType != ValueInc->getType())
ValueInc = Builder.CreateSExt(ValueInc, MaxType);
BasicBlock::iterator LoopBody;
SetVector<Value *> SubtreeValues;
SetVector<const Loop *> Loops;
getReferencesInSubtree(isl::manage_copy(For), SubtreeValues, Loops);
// Create for all loops we depend on values that contain the current loop
// iteration. These values are necessary to generate code for SCEVs that
// depend on such loops. As a result we need to pass them to the subfunction.
// See [Code generation of induction variables of loops outside Scops]
for (const Loop *L : Loops) {
Value *LoopInductionVar = materializeNonScopLoopInductionVariable(L);
SubtreeValues.insert(LoopInductionVar);
}
ValueMapT NewValues;
std::unique_ptr<ParallelLoopGenerator> ParallelLoopGenPtr;
switch (PollyOmpBackend) {
case OpenMPBackend::GNU:
ParallelLoopGenPtr.reset(
new ParallelLoopGeneratorGOMP(Builder, LI, DT, DL));
break;
case OpenMPBackend::LLVM:
ParallelLoopGenPtr.reset(new ParallelLoopGeneratorKMP(Builder, LI, DT, DL));
break;
}
IV = ParallelLoopGenPtr->createParallelLoop(
ValueLB, ValueUB, ValueInc, SubtreeValues, NewValues, &LoopBody);
BasicBlock::iterator AfterLoop = Builder.GetInsertPoint();
Builder.SetInsertPoint(&*LoopBody);
// Remember the parallel subfunction
ParallelSubfunctions.push_back(LoopBody->getFunction());
// Save the current values.
auto ValueMapCopy = ValueMap;
IslExprBuilder::IDToValueTy IDToValueCopy = IDToValue;
updateValues(NewValues);
IDToValue[IteratorID] = IV;
ValueMapT NewValuesReverse;
for (auto P : NewValues)
NewValuesReverse[P.second] = P.first;
Annotator.addAlternativeAliasBases(NewValuesReverse);
create(Body);
Annotator.resetAlternativeAliasBases();
// Restore the original values.
ValueMap = ValueMapCopy;
IDToValue = IDToValueCopy;
Builder.SetInsertPoint(&*AfterLoop);
removeSubFuncFromDomTree((*LoopBody).getParent()->getParent(), DT);
for (const Loop *L : Loops)
OutsideLoopIterations.erase(L);
isl_ast_node_free(For);
isl_ast_expr_free(Iterator);
isl_id_free(IteratorID);
ParallelLoops++;
}
/// Return whether any of @p Node's statements contain partial accesses.
///
/// Partial accesses are not supported by Polly's vector code generator.
static bool hasPartialAccesses(__isl_take isl_ast_node *Node) {
return isl_ast_node_foreach_descendant_top_down(
Node,
[](isl_ast_node *Node, void *User) -> isl_bool {
if (isl_ast_node_get_type(Node) != isl_ast_node_user)
return isl_bool_true;
isl::ast_expr Expr =
isl::manage(isl_ast_node_user_get_expr(Node));
isl::ast_expr StmtExpr = Expr.get_op_arg(0);
isl::id Id = StmtExpr.get_id();
ScopStmt *Stmt =
static_cast<ScopStmt *>(isl_id_get_user(Id.get()));
isl::set StmtDom = Stmt->getDomain();
for (auto *MA : *Stmt) {
if (MA->isLatestPartialAccess())
return isl_bool_error;
}
return isl_bool_true;
},
nullptr) == isl_stat_error;
}
void IslNodeBuilder::createFor(__isl_take isl_ast_node *For) {
bool Vector = PollyVectorizerChoice == VECTORIZER_POLLY;
if (Vector && IslAstInfo::isInnermostParallel(isl::manage_copy(For)) &&
!IslAstInfo::isReductionParallel(isl::manage_copy(For))) {
int VectorWidth =
getNumberOfIterations(isl::manage_copy(For).as<isl::ast_node_for>());
if (1 < VectorWidth && VectorWidth <= 16 && !hasPartialAccesses(For)) {
createForVector(For, VectorWidth);
return;
}
}
if (IslAstInfo::isExecutedInParallel(isl::manage_copy(For))) {
createForParallel(For);
return;
}
bool Parallel = (IslAstInfo::isParallel(isl::manage_copy(For)) &&
!IslAstInfo::isReductionParallel(isl::manage_copy(For)));
createForSequential(isl::manage(For).as<isl::ast_node_for>(), Parallel);
}
void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) {
isl_ast_expr *Cond = isl_ast_node_if_get_cond(If);
Function *F = Builder.GetInsertBlock()->getParent();
LLVMContext &Context = F->getContext();
BasicBlock *CondBB = SplitBlock(Builder.GetInsertBlock(),
&*Builder.GetInsertPoint(), &DT, &LI);
CondBB->setName("polly.cond");
BasicBlock *MergeBB = SplitBlock(CondBB, &CondBB->front(), &DT, &LI);
MergeBB->setName("polly.merge");
BasicBlock *ThenBB = BasicBlock::Create(Context, "polly.then", F);
BasicBlock *ElseBB = BasicBlock::Create(Context, "polly.else", F);
DT.addNewBlock(ThenBB, CondBB);
DT.addNewBlock(ElseBB, CondBB);
DT.changeImmediateDominator(MergeBB, CondBB);
Loop *L = LI.getLoopFor(CondBB);
if (L) {
L->addBasicBlockToLoop(ThenBB, LI);
L->addBasicBlockToLoop(ElseBB, LI);
}
CondBB->getTerminator()->eraseFromParent();
Builder.SetInsertPoint(CondBB);
Value *Predicate = ExprBuilder.create(Cond);
Builder.CreateCondBr(Predicate, ThenBB, ElseBB);
Builder.SetInsertPoint(ThenBB);
Builder.CreateBr(MergeBB);
Builder.SetInsertPoint(ElseBB);
Builder.CreateBr(MergeBB);
Builder.SetInsertPoint(&ThenBB->front());
create(isl_ast_node_if_get_then(If));
Builder.SetInsertPoint(&ElseBB->front());
if (isl_ast_node_if_has_else(If))
create(isl_ast_node_if_get_else(If));
Builder.SetInsertPoint(&MergeBB->front());
isl_ast_node_free(If);
IfConditions++;
}
__isl_give isl_id_to_ast_expr *
IslNodeBuilder::createNewAccesses(ScopStmt *Stmt,
__isl_keep isl_ast_node *Node) {
isl::id_to_ast_expr NewAccesses =
isl::id_to_ast_expr::alloc(Stmt->getParent()->getIslCtx(), 0);
isl::ast_build Build = IslAstInfo::getBuild(isl::manage_copy(Node));
assert(!Build.is_null() && "Could not obtain isl_ast_build from user node");
Stmt->setAstBuild(Build);
for (auto *MA : *Stmt) {
if (!MA->hasNewAccessRelation()) {
if (PollyGenerateExpressions) {
if (!MA->isAffine())
continue;
if (MA->getLatestScopArrayInfo()->getBasePtrOriginSAI())
continue;
auto *BasePtr =
dyn_cast<Instruction>(MA->getLatestScopArrayInfo()->getBasePtr());
if (BasePtr && Stmt->getParent()->getRegion().contains(BasePtr))
continue;
} else {
continue;
}
}
assert(MA->isAffine() &&
"Only affine memory accesses can be code generated");
isl::union_map Schedule = Build.get_schedule();
#ifndef NDEBUG
if (MA->isRead()) {
auto Dom = Stmt->getDomain().release();
auto SchedDom = isl_set_from_union_set(Schedule.domain().release());
auto AccDom = isl_map_domain(MA->getAccessRelation().release());
Dom = isl_set_intersect_params(Dom,
Stmt->getParent()->getContext().release());
SchedDom = isl_set_intersect_params(
SchedDom, Stmt->getParent()->getContext().release());
assert(isl_set_is_subset(SchedDom, AccDom) &&
"Access relation not defined on full schedule domain");
assert(isl_set_is_subset(Dom, AccDom) &&
"Access relation not defined on full domain");
isl_set_free(AccDom);
isl_set_free(SchedDom);
isl_set_free(Dom);
}
#endif
isl::pw_multi_aff PWAccRel = MA->applyScheduleToAccessRelation(Schedule);
// isl cannot generate an index expression for access-nothing accesses.
isl::set AccDomain = PWAccRel.domain();
isl::set Context = S.getContext();
AccDomain = AccDomain.intersect_params(Context);
if (AccDomain.is_empty())
continue;
isl::ast_expr AccessExpr = Build.access_from(PWAccRel);
NewAccesses = NewAccesses.set(MA->getId(), AccessExpr);
}
return NewAccesses.release();
}
void IslNodeBuilder::createSubstitutions(__isl_take isl_ast_expr *Expr,
ScopStmt *Stmt, LoopToScevMapT &LTS) {
assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_op &&
"Expression of type 'op' expected");
assert(isl_ast_expr_get_op_type(Expr) == isl_ast_op_call &&
"Operation of type 'call' expected");
for (int i = 0; i < isl_ast_expr_get_op_n_arg(Expr) - 1; ++i) {
isl_ast_expr *SubExpr;
Value *V;
SubExpr = isl_ast_expr_get_op_arg(Expr, i + 1);
V = ExprBuilder.create(SubExpr);
ScalarEvolution *SE = Stmt->getParent()->getSE();
LTS[Stmt->getLoopForDimension(i)] = SE->getUnknown(V);
}
isl_ast_expr_free(Expr);
}
void IslNodeBuilder::createSubstitutionsVector(
__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
std::vector<LoopToScevMapT> &VLTS, std::vector<Value *> &IVS,
__isl_take isl_id *IteratorID) {
int i = 0;
Value *OldValue = IDToValue[IteratorID];
for (Value *IV : IVS) {
IDToValue[IteratorID] = IV;
createSubstitutions(isl_ast_expr_copy(Expr), Stmt, VLTS[i]);
i++;
}
IDToValue[IteratorID] = OldValue;
isl_id_free(IteratorID);
isl_ast_expr_free(Expr);
}
void IslNodeBuilder::generateCopyStmt(
ScopStmt *Stmt, __isl_keep isl_id_to_ast_expr *NewAccesses) {
assert(Stmt->size() == 2);
auto ReadAccess = Stmt->begin();
auto WriteAccess = ReadAccess++;
assert((*ReadAccess)->isRead() && (*WriteAccess)->isMustWrite());
assert((*ReadAccess)->getElementType() == (*WriteAccess)->getElementType() &&
"Accesses use the same data type");
assert((*ReadAccess)->isArrayKind() && (*WriteAccess)->isArrayKind());
auto *AccessExpr =
isl_id_to_ast_expr_get(NewAccesses, (*ReadAccess)->getId().release());
auto *LoadValue = ExprBuilder.create(AccessExpr);
AccessExpr =
isl_id_to_ast_expr_get(NewAccesses, (*WriteAccess)->getId().release());
auto *StoreAddr = ExprBuilder.createAccessAddress(AccessExpr).first;
Builder.CreateStore(LoadValue, StoreAddr);
}
Value *IslNodeBuilder::materializeNonScopLoopInductionVariable(const Loop *L) {
assert(OutsideLoopIterations.find(L) == OutsideLoopIterations.end() &&
"trying to materialize loop induction variable twice");
const SCEV *OuterLIV = SE.getAddRecExpr(SE.getUnknown(Builder.getInt64(0)),
SE.getUnknown(Builder.getInt64(1)), L,
SCEV::FlagAnyWrap);
Value *V = generateSCEV(OuterLIV);
OutsideLoopIterations[L] = SE.getUnknown(V);
return V;
}
void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) {
LoopToScevMapT LTS;
isl_id *Id;
ScopStmt *Stmt;
isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
Id = isl_ast_expr_get_id(StmtExpr);
isl_ast_expr_free(StmtExpr);
LTS.insert(OutsideLoopIterations.begin(), OutsideLoopIterations.end());
Stmt = (ScopStmt *)isl_id_get_user(Id);
auto *NewAccesses = createNewAccesses(Stmt, User);
if (Stmt->isCopyStmt()) {
generateCopyStmt(Stmt, NewAccesses);
isl_ast_expr_free(Expr);
} else {
createSubstitutions(Expr, Stmt, LTS);
if (Stmt->isBlockStmt())
BlockGen.copyStmt(*Stmt, LTS, NewAccesses);
else
RegionGen.copyStmt(*Stmt, LTS, NewAccesses);
}
isl_id_to_ast_expr_free(NewAccesses);
isl_ast_node_free(User);
isl_id_free(Id);
}
void IslNodeBuilder::createBlock(__isl_take isl_ast_node *Block) {
isl_ast_node_list *List = isl_ast_node_block_get_children(Block);
for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
create(isl_ast_node_list_get_ast_node(List, i));
isl_ast_node_free(Block);
isl_ast_node_list_free(List);
}
void IslNodeBuilder::create(__isl_take isl_ast_node *Node) {
switch (isl_ast_node_get_type(Node)) {
case isl_ast_node_error:
llvm_unreachable("code generation error");
case isl_ast_node_mark:
createMark(Node);
return;
case isl_ast_node_for:
createFor(Node);
return;
case isl_ast_node_if:
createIf(Node);
return;
case isl_ast_node_user:
createUser(Node);
return;
case isl_ast_node_block:
createBlock(Node);
return;
}
llvm_unreachable("Unknown isl_ast_node type");
}
bool IslNodeBuilder::materializeValue(isl_id *Id) {
// If the Id is already mapped, skip it.
if (!IDToValue.count(Id)) {
auto *ParamSCEV = (const SCEV *)isl_id_get_user(Id);
Value *V = nullptr;
// Parameters could refer to invariant loads that need to be
// preloaded before we can generate code for the parameter. Thus,
// check if any value referred to in ParamSCEV is an invariant load
// and if so make sure its equivalence class is preloaded.
SetVector<Value *> Values;
findValues(ParamSCEV, SE, Values);
for (auto *Val : Values) {
// Check if the value is an instruction in a dead block within the SCoP
// and if so do not code generate it.
if (auto *Inst = dyn_cast<Instruction>(Val)) {
if (S.contains(Inst)) {
bool IsDead = true;
// Check for "undef" loads first, then if there is a statement for
// the parent of Inst and lastly if the parent of Inst has an empty
// domain. In the first and last case the instruction is dead but if
// there is a statement or the domain is not empty Inst is not dead.
auto MemInst = MemAccInst::dyn_cast(Inst);
auto Address = MemInst ? MemInst.getPointerOperand() : nullptr;
if (Address && SE.getUnknown(UndefValue::get(Address->getType())) ==
SE.getPointerBase(SE.getSCEV(Address))) {
} else if (S.getStmtFor(Inst)) {
IsDead = false;
} else {
auto *Domain = S.getDomainConditions(Inst->getParent()).release();
IsDead = isl_set_is_empty(Domain);
isl_set_free(Domain);
}
if (IsDead) {
V = UndefValue::get(ParamSCEV->getType());
break;
}
}
}
if (auto *IAClass = S.lookupInvariantEquivClass(Val)) {
// Check if this invariant access class is empty, hence if we never
// actually added a loads instruction to it. In that case it has no
// (meaningful) users and we should not try to code generate it.
if (IAClass->InvariantAccesses.empty())
V = UndefValue::get(ParamSCEV->getType());
if (!preloadInvariantEquivClass(*IAClass)) {
isl_id_free(Id);
return false;
}
}
}
V = V ? V : generateSCEV(ParamSCEV);
IDToValue[Id] = V;
}
isl_id_free(Id);
return true;
}
bool IslNodeBuilder::materializeParameters(isl_set *Set) {
for (unsigned i = 0, e = isl_set_dim(Set, isl_dim_param); i < e; ++i) {
if (!isl_set_involves_dims(Set, isl_dim_param, i, 1))
continue;
isl_id *Id = isl_set_get_dim_id(Set, isl_dim_param, i);
if (!materializeValue(Id))
return false;
}
return true;
}
bool IslNodeBuilder::materializeParameters() {
for (const SCEV *Param : S.parameters()) {
isl_id *Id = S.getIdForParam(Param).release();
if (!materializeValue(Id))
return false;
}
return true;
}
/// Generate the computation of the size of the outermost dimension from the
/// Fortran array descriptor (in this case, `@g_arr`). The final `%size`
/// contains the size of the array.
///
/// %arrty = type { i8*, i64, i64, [3 x %desc.dimensionty] }
/// %desc.dimensionty = type { i64, i64, i64 }
/// @g_arr = global %arrty zeroinitializer, align 32
/// ...
/// %0 = load i64, i64* getelementptr inbounds
/// (%arrty, %arrty* @g_arr, i64 0, i32 3, i64 0, i32 2)
/// %1 = load i64, i64* getelementptr inbounds
/// (%arrty, %arrty* @g_arr, i64 0, i32 3, i64 0, i32 1)
/// %2 = sub nsw i64 %0, %1
/// %size = add nsw i64 %2, 1
static Value *buildFADOutermostDimensionLoad(Value *GlobalDescriptor,
PollyIRBuilder &Builder,
std::string ArrayName) {
assert(GlobalDescriptor && "invalid global descriptor given");
Type *Ty = GlobalDescriptor->getType()->getPointerElementType();
Value *endIdx[4] = {Builder.getInt64(0), Builder.getInt32(3),
Builder.getInt64(0), Builder.getInt32(2)};
Value *endPtr = Builder.CreateInBoundsGEP(Ty, GlobalDescriptor, endIdx,
ArrayName + "_end_ptr");
Type *type = cast<GEPOperator>(endPtr)->getResultElementType();
assert(isa<IntegerType>(type) && "expected type of end to be integral");
Value *end = Builder.CreateLoad(type, endPtr, ArrayName + "_end");
Value *beginIdx[4] = {Builder.getInt64(0), Builder.getInt32(3),
Builder.getInt64(0), Builder.getInt32(1)};
Value *beginPtr = Builder.CreateInBoundsGEP(Ty, GlobalDescriptor, beginIdx,
ArrayName + "_begin_ptr");
Value *begin = Builder.CreateLoad(type, beginPtr, ArrayName + "_begin");
Value *size =
Builder.CreateNSWSub(end, begin, ArrayName + "_end_begin_delta");
size = Builder.CreateNSWAdd(
end, ConstantInt::get(type, 1, /* signed = */ true), ArrayName + "_size");
return size;
}
bool IslNodeBuilder::materializeFortranArrayOutermostDimension() {
for (ScopArrayInfo *Array : S.arrays()) {
if (Array->getNumberOfDimensions() == 0)
continue;
Value *FAD = Array->getFortranArrayDescriptor();
if (!FAD)
continue;
isl_pw_aff *ParametricPwAff = Array->getDimensionSizePw(0).release();
assert(ParametricPwAff && "parametric pw_aff corresponding "
"to outermost dimension does not "
"exist");
isl_id *Id = isl_pw_aff_get_dim_id(ParametricPwAff, isl_dim_param, 0);
isl_pw_aff_free(ParametricPwAff);
assert(Id && "pw_aff is not parametric");
if (IDToValue.count(Id)) {
isl_id_free(Id);
continue;
}
Value *FinalValue =
buildFADOutermostDimensionLoad(FAD, Builder, Array->getName());
assert(FinalValue && "unable to build Fortran array "
"descriptor load of outermost dimension");
IDToValue[Id] = FinalValue;
isl_id_free(Id);
}
return true;
}
Value *IslNodeBuilder::preloadUnconditionally(isl_set *AccessRange,
isl_ast_build *Build,
Instruction *AccInst) {
isl_pw_multi_aff *PWAccRel = isl_pw_multi_aff_from_set(AccessRange);
isl_ast_expr *Access =
isl_ast_build_access_from_pw_multi_aff(Build, PWAccRel);
auto *Address = isl_ast_expr_address_of(Access);
auto *AddressValue = ExprBuilder.create(Address);
Value *PreloadVal;
// Correct the type as the SAI might have a different type than the user
// expects, especially if the base pointer is a struct.
Type *Ty = AccInst->getType();
auto *Ptr = AddressValue;
auto Name = Ptr->getName();
auto AS = Ptr->getType()->getPointerAddressSpace();
Ptr = Builder.CreatePointerCast(Ptr, Ty->getPointerTo(AS), Name + ".cast");
PreloadVal = Builder.CreateLoad(Ty, Ptr, Name + ".load");
if (LoadInst *PreloadInst = dyn_cast<LoadInst>(PreloadVal))
PreloadInst->setAlignment(cast<LoadInst>(AccInst)->getAlign());
// TODO: This is only a hot fix for SCoP sequences that use the same load
// instruction contained and hoisted by one of the SCoPs.
if (SE.isSCEVable(Ty))
SE.forgetValue(AccInst);
return PreloadVal;
}
Value *IslNodeBuilder::preloadInvariantLoad(const MemoryAccess &MA,
isl_set *Domain) {
isl_set *AccessRange = isl_map_range(MA.getAddressFunction().release());
AccessRange = isl_set_gist_params(AccessRange, S.getContext().release());
if (!materializeParameters(AccessRange)) {
isl_set_free(AccessRange);
isl_set_free(Domain);
return nullptr;
}
auto *Build =
isl_ast_build_from_context(isl_set_universe(S.getParamSpace().release()));
isl_set *Universe = isl_set_universe(isl_set_get_space(Domain));
bool AlwaysExecuted = isl_set_is_equal(Domain, Universe);
isl_set_free(Universe);
Instruction *AccInst = MA.getAccessInstruction();
Type *AccInstTy = AccInst->getType();
Value *PreloadVal = nullptr;
if (AlwaysExecuted) {
PreloadVal = preloadUnconditionally(AccessRange, Build, AccInst);
isl_ast_build_free(Build);
isl_set_free(Domain);
return PreloadVal;
}
if (!materializeParameters(Domain)) {
isl_ast_build_free(Build);
isl_set_free(AccessRange);
isl_set_free(Domain);
return nullptr;
}
isl_ast_expr *DomainCond = isl_ast_build_expr_from_set(Build, Domain);
Domain = nullptr;
ExprBuilder.setTrackOverflow(true);
Value *Cond = ExprBuilder.create(DomainCond);
Value *OverflowHappened = Builder.CreateNot(ExprBuilder.getOverflowState(),
"polly.preload.cond.overflown");
Cond = Builder.CreateAnd(Cond, OverflowHappened, "polly.preload.cond.result");
ExprBuilder.setTrackOverflow(false);
if (!Cond->getType()->isIntegerTy(1))
Cond = Builder.CreateIsNotNull(Cond);
BasicBlock *CondBB = SplitBlock(Builder.GetInsertBlock(),
&*Builder.GetInsertPoint(), &DT, &LI);
CondBB->setName("polly.preload.cond");
BasicBlock *MergeBB = SplitBlock(CondBB, &CondBB->front(), &DT, &LI);
MergeBB->setName("polly.preload.merge");
Function *F = Builder.GetInsertBlock()->getParent();
LLVMContext &Context = F->getContext();
BasicBlock *ExecBB = BasicBlock::Create(Context, "polly.preload.exec", F);
DT.addNewBlock(ExecBB, CondBB);
if (Loop *L = LI.getLoopFor(CondBB))
L->addBasicBlockToLoop(ExecBB, LI);
auto *CondBBTerminator = CondBB->getTerminator();
Builder.SetInsertPoint(CondBBTerminator);
Builder.CreateCondBr(Cond, ExecBB, MergeBB);
CondBBTerminator->eraseFromParent();
Builder.SetInsertPoint(ExecBB);
Builder.CreateBr(MergeBB);
Builder.SetInsertPoint(ExecBB->getTerminator());
Value *PreAccInst = preloadUnconditionally(AccessRange, Build, AccInst);
Builder.SetInsertPoint(MergeBB->getTerminator());
auto *MergePHI = Builder.CreatePHI(
AccInstTy, 2, "polly.preload." + AccInst->getName() + ".merge");
PreloadVal = MergePHI;
if (!PreAccInst) {
PreloadVal = nullptr;
PreAccInst = UndefValue::get(AccInstTy);
}
MergePHI->addIncoming(PreAccInst, ExecBB);
MergePHI->addIncoming(Constant::getNullValue(AccInstTy), CondBB);
isl_ast_build_free(Build);
return PreloadVal;
}
bool IslNodeBuilder::preloadInvariantEquivClass(
InvariantEquivClassTy &IAClass) {
// For an equivalence class of invariant loads we pre-load the representing
// element with the unified execution context. However, we have to map all
// elements of the class to the one preloaded load as they are referenced
// during the code generation and therefor need to be mapped.
const MemoryAccessList &MAs = IAClass.InvariantAccesses;
if (MAs.empty())
return true;
MemoryAccess *MA = MAs.front();
assert(MA->isArrayKind() && MA->isRead());
// If the access function was already mapped, the preload of this equivalence
// class was triggered earlier already and doesn't need to be done again.
if (ValueMap.count(MA->getAccessInstruction()))
return true;
// Check for recursion which can be caused by additional constraints, e.g.,
// non-finite loop constraints. In such a case we have to bail out and insert
// a "false" runtime check that will cause the original code to be executed.
auto PtrId = std::make_pair(IAClass.IdentifyingPointer, IAClass.AccessType);
if (!PreloadedPtrs.insert(PtrId).second)
return false;
// The execution context of the IAClass.
isl::set &ExecutionCtx = IAClass.ExecutionContext;
// If the base pointer of this class is dependent on another one we have to
// make sure it was preloaded already.
auto *SAI = MA->getScopArrayInfo();
if (auto *BaseIAClass = S.lookupInvariantEquivClass(SAI->getBasePtr())) {
if (!preloadInvariantEquivClass(*BaseIAClass))
return false;
// After we preloaded the BaseIAClass we adjusted the BaseExecutionCtx and
// we need to refine the ExecutionCtx.
isl::set BaseExecutionCtx = BaseIAClass->ExecutionContext;
ExecutionCtx = ExecutionCtx.intersect(BaseExecutionCtx);
}
// If the size of a dimension is dependent on another class, make sure it is
// preloaded.
for (unsigned i = 1, e = SAI->getNumberOfDimensions(); i < e; ++i) {
const SCEV *Dim = SAI->getDimensionSize(i);
SetVector<Value *> Values;
findValues(Dim, SE, Values);
for (auto *Val : Values) {
if (auto *BaseIAClass = S.lookupInvariantEquivClass(Val)) {
if (!preloadInvariantEquivClass(*BaseIAClass))
return false;
// After we preloaded the BaseIAClass we adjusted the BaseExecutionCtx
// and we need to refine the ExecutionCtx.
isl::set BaseExecutionCtx = BaseIAClass->ExecutionContext;
ExecutionCtx = ExecutionCtx.intersect(BaseExecutionCtx);
}
}
}
Instruction *AccInst = MA->getAccessInstruction();
Type *AccInstTy = AccInst->getType();
Value *PreloadVal = preloadInvariantLoad(*MA, ExecutionCtx.copy());
if (!PreloadVal)
return false;
for (const MemoryAccess *MA : MAs) {
Instruction *MAAccInst = MA->getAccessInstruction();
assert(PreloadVal->getType() == MAAccInst->getType());
ValueMap[MAAccInst] = PreloadVal;
}
if (SE.isSCEVable(AccInstTy)) {
isl_id *ParamId = S.getIdForParam(SE.getSCEV(AccInst)).release();
if (ParamId)
IDToValue[ParamId] = PreloadVal;
isl_id_free(ParamId);
}
BasicBlock *EntryBB = &Builder.GetInsertBlock()->getParent()->getEntryBlock();
auto *Alloca = new AllocaInst(AccInstTy, DL.getAllocaAddrSpace(),
AccInst->getName() + ".preload.s2a",
&*EntryBB->getFirstInsertionPt());
Builder.CreateStore(PreloadVal, Alloca);
ValueMapT PreloadedPointer;
PreloadedPointer[PreloadVal] = AccInst;
Annotator.addAlternativeAliasBases(PreloadedPointer);
for (auto *DerivedSAI : SAI->getDerivedSAIs()) {
Value *BasePtr = DerivedSAI->getBasePtr();
for (const MemoryAccess *MA : MAs) {
// As the derived SAI information is quite coarse, any load from the
// current SAI could be the base pointer of the derived SAI, however we
// should only change the base pointer of the derived SAI if we actually
// preloaded it.
if (BasePtr == MA->getOriginalBaseAddr()) {
assert(BasePtr->getType() == PreloadVal->getType());
DerivedSAI->setBasePtr(PreloadVal);
}
// For scalar derived SAIs we remap the alloca used for the derived value.
if (BasePtr == MA->getAccessInstruction())
ScalarMap[DerivedSAI] = Alloca;
}
}
for (const MemoryAccess *MA : MAs) {
Instruction *MAAccInst = MA->getAccessInstruction();
// Use the escape system to get the correct value to users outside the SCoP.
BlockGenerator::EscapeUserVectorTy EscapeUsers;
for (auto *U : MAAccInst->users())
if (Instruction *UI = dyn_cast<Instruction>(U))
if (!S.contains(UI))
EscapeUsers.push_back(UI);
if (EscapeUsers.empty())
continue;
EscapeMap[MA->getAccessInstruction()] =
std::make_pair(Alloca, std::move(EscapeUsers));
}
return true;
}
void IslNodeBuilder::allocateNewArrays(BBPair StartExitBlocks) {
for (auto &SAI : S.arrays()) {
if (SAI->getBasePtr())
continue;
assert(SAI->getNumberOfDimensions() > 0 && SAI->getDimensionSize(0) &&
"The size of the outermost dimension is used to declare newly "
"created arrays that require memory allocation.");
Type *NewArrayType = nullptr;
// Get the size of the array = size(dim_1)*...*size(dim_n)
uint64_t ArraySizeInt = 1;
for (int i = SAI->getNumberOfDimensions() - 1; i >= 0; i--) {
auto *DimSize = SAI->getDimensionSize(i);
unsigned UnsignedDimSize = static_cast<const SCEVConstant *>(DimSize)
->getAPInt()
.getLimitedValue();
if (!NewArrayType)
NewArrayType = SAI->getElementType();
NewArrayType = ArrayType::get(NewArrayType, UnsignedDimSize);
ArraySizeInt *= UnsignedDimSize;
}
if (SAI->isOnHeap()) {
LLVMContext &Ctx = NewArrayType->getContext();
// Get the IntPtrTy from the Datalayout
auto IntPtrTy = DL.getIntPtrType(Ctx);
// Get the size of the element type in bits
unsigned Size = SAI->getElemSizeInBytes();
// Insert the malloc call at polly.start
auto InstIt = std::get<0>(StartExitBlocks)->getTerminator();
auto *CreatedArray = CallInst::CreateMalloc(
&*InstIt, IntPtrTy, SAI->getElementType(),
ConstantInt::get(Type::getInt64Ty(Ctx), Size),
ConstantInt::get(Type::getInt64Ty(Ctx), ArraySizeInt), nullptr,
SAI->getName());
SAI->setBasePtr(CreatedArray);
// Insert the free call at polly.exiting
CallInst::CreateFree(CreatedArray,
std::get<1>(StartExitBlocks)->getTerminator());
} else {
auto InstIt = Builder.GetInsertBlock()
->getParent()
->getEntryBlock()
.getTerminator();
auto *CreatedArray = new AllocaInst(NewArrayType, DL.getAllocaAddrSpace(),
SAI->getName(), &*InstIt);
if (PollyTargetFirstLevelCacheLineSize)
CreatedArray->setAlignment(Align(PollyTargetFirstLevelCacheLineSize));
SAI->setBasePtr(CreatedArray);
}
}
}
bool IslNodeBuilder::preloadInvariantLoads() {
auto &InvariantEquivClasses = S.getInvariantAccesses();
if (InvariantEquivClasses.empty())
return true;
BasicBlock *PreLoadBB = SplitBlock(Builder.GetInsertBlock(),
&*Builder.GetInsertPoint(), &DT, &LI);
PreLoadBB->setName("polly.preload.begin");
Builder.SetInsertPoint(&PreLoadBB->front());
for (auto &IAClass : InvariantEquivClasses)
if (!preloadInvariantEquivClass(IAClass))
return false;
return true;
}
void IslNodeBuilder::addParameters(__isl_take isl_set *Context) {
// Materialize values for the parameters of the SCoP.
materializeParameters();
// materialize the outermost dimension parameters for a Fortran array.
// NOTE: materializeParameters() does not work since it looks through
// the SCEVs. We don't have a corresponding SCEV for the array size
// parameter
materializeFortranArrayOutermostDimension();
// Generate values for the current loop iteration for all surrounding loops.
//
// We may also reference loops outside of the scop which do not contain the
// scop itself, but as the number of such scops may be arbitrarily large we do
// not generate code for them here, but only at the point of code generation
// where these values are needed.
Loop *L = LI.getLoopFor(S.getEntry());
while (L != nullptr && S.contains(L))
L = L->getParentLoop();
while (L != nullptr) {
materializeNonScopLoopInductionVariable(L);
L = L->getParentLoop();
}
isl_set_free(Context);
}
Value *IslNodeBuilder::generateSCEV(const SCEV *Expr) {
/// We pass the insert location of our Builder, as Polly ensures during IR
/// generation that there is always a valid CFG into which instructions are
/// inserted. As a result, the insertpoint is known to be always followed by a
/// terminator instruction. This means the insert point may be specified by a
/// terminator instruction, but it can never point to an ->end() iterator
/// which does not have a corresponding instruction. Hence, dereferencing
/// the insertpoint to obtain an instruction is known to be save.
///
/// We also do not need to update the Builder here, as new instructions are
/// always inserted _before_ the given InsertLocation. As a result, the
/// insert location remains valid.
assert(Builder.GetInsertBlock()->end() != Builder.GetInsertPoint() &&
"Insert location points after last valid instruction");
Instruction *InsertLocation = &*Builder.GetInsertPoint();
return expandCodeFor(S, SE, DL, "polly", Expr, Expr->getType(),
InsertLocation, &ValueMap,
StartBlock->getSinglePredecessor());
}
/// The AST expression we generate to perform the run-time check assumes
/// computations on integer types of infinite size. As we only use 64-bit
/// arithmetic we check for overflows, in case of which we set the result
/// of this run-time check to false to be conservatively correct,
Value *IslNodeBuilder::createRTC(isl_ast_expr *Condition) {
auto ExprBuilder = getExprBuilder();
// In case the AST expression has integers larger than 64 bit, bail out. The
// resulting LLVM-IR will contain operations on types that use more than 64
// bits. These are -- in case wrapping intrinsics are used -- translated to
// runtime library calls that are not available on all systems (e.g., Android)
// and consequently will result in linker errors.
if (ExprBuilder.hasLargeInts(isl::manage_copy(Condition))) {
isl_ast_expr_free(Condition);
return Builder.getFalse();
}
ExprBuilder.setTrackOverflow(true);
Value *RTC = ExprBuilder.create(Condition);
if (!RTC->getType()->isIntegerTy(1))
RTC = Builder.CreateIsNotNull(RTC);
Value *OverflowHappened =
Builder.CreateNot(ExprBuilder.getOverflowState(), "polly.rtc.overflown");
if (PollyGenerateRTCPrint) {
auto *F = Builder.GetInsertBlock()->getParent();
RuntimeDebugBuilder::createCPUPrinter(
Builder,
"F: " + F->getName().str() + " R: " + S.getRegion().getNameStr() +
"RTC: ",
RTC, " Overflow: ", OverflowHappened,
"\n"
" (0 failed, -1 succeeded)\n"
" (if one or both are 0 falling back to original code, if both are -1 "
"executing Polly code)\n");
}
RTC = Builder.CreateAnd(RTC, OverflowHappened, "polly.rtc.result");
ExprBuilder.setTrackOverflow(false);
if (!isa<ConstantInt>(RTC))
VersionedScops++;
return RTC;
}
Reference in New Issue View Git Blame Copy Permalink