caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[VPlan] Create edge masks all cases up front needed.(NFC)

Browse Source 
Similarly to how block masks are created up front and later only
retrieved also make sure masks are created in cases where edge masks are
needed, i.e. blend recipes.

Creating block-in masks for all blocks in the loop also ensures edge
masks for all relevant edges have been created. Later, the new
getEdgeMask can be used to look up cached edge masks.

This makes sure edge masks are available in all cases for
https://github.com/llvm/llvm-project/pull/76090.
This commit is contained in:
Florian Hahn
2024-01-28 21:05:16 +00:00
parent a7cfff8dc6
commit 2d0d65b3ba
2 changed files with 23 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@@ -8038,6 +8038,17 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
return EdgeMaskCache[Edge] = EdgeMask;
}
VPValue *VPRecipeBuilder::getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const {
assert(is_contained(predecessors(Dst), Src) && "Invalid edge");
// Look for cached value.
std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
EdgeMaskCacheTy::const_iterator ECEntryIt = EdgeMaskCache.find(Edge);
assert(ECEntryIt != EdgeMaskCache.end() &&
"looking up mask for edge which has not been created");
return ECEntryIt->second;
}
void VPRecipeBuilder::createHeaderMask(VPlan &Plan) {
BasicBlock *Header = OrigLoop->getHeader();
@@ -8728,10 +8739,13 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
DFS.perform(LI);
VPBasicBlock *VPBB = HeaderVPBB;
bool NeedsMasks = CM.foldTailByMasking() ||
any_of(OrigLoop->blocks(), [this](BasicBlock *BB) {
return Legal->blockNeedsPredication(BB);
});
BasicBlock *HeaderBB = OrigLoop->getHeader();
bool NeedsMasks =
CM.foldTailByMasking() ||
any_of(OrigLoop->blocks(), [this, HeaderBB](BasicBlock *BB) {
bool NeedsBlends = BB != HeaderBB && !BB->phis().empty();
return Legal->blockNeedsPredication(BB) || NeedsBlends;
});
for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
// Relevant instructions from basic block BB will be grouped into VPRecipe
// ingredients and fill a new VPBasicBlock.
@@ -8750,7 +8764,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Instruction *Instr = &I;
SmallVector<VPValue *, 4> Operands;
auto *Phi = dyn_cast<PHINode>(Instr);
if (Phi && Phi->getParent() == OrigLoop->getHeader()) {
if (Phi && Phi->getParent() == HeaderBB) {
Operands.push_back(Plan->getVPValueOrAddLiveIn(
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
} else {

4
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
View File

@@ -148,6 +148,10 @@ public:
/// and DST.
VPValue *createEdgeMask(BasicBlock *Src, BasicBlock *Dst, VPlan &Plan);
/// A helper that returns the previously computed predicate of the edge
/// between SRC and DST.
VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const;
/// Mark given ingredient for recording its recipe once one is created for
/// it.
void recordRecipeOf(Instruction *I) {