caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[VPlan] Only store single vector per VPValue in VPTransformState. (NFC)

Browse Source 
After 8ec406757c (https://github.com/llvm/llvm-project/pull/95842),
VPTransformState only stores a single vector value per VPValue.

Simplify the code by replacing the SmallVector in PerPartOutput with a
single Value * and rename to VPV2Vector for clarity.

Also remove the redundant Part argument from various accessors.
This commit is contained in:
Florian Hahn
2024-09-23 11:28:23 +01:00
parent 31ac3d092b
commit 57f5d8f2fe
4 changed files with 145 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@@ -3075,13 +3075,13 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan,
VPWidenPHIRecipe *VPPhi = dyn_cast<VPWidenPHIRecipe>(&P);
if (!VPPhi)
continue;
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) {
VPValue *Inc = VPPhi->getIncomingValue(Idx);
VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
}
}
}
@@ -9445,7 +9445,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
Value *Poison = PoisonValue::get(
VectorType::get(UI->getType(), State.VF));
State.set(this, Poison, State.Instance->Part);
State.set(this, Poison);
}
State.packScalarIntoVectorValue(this, *State.Instance);
}

54
llvm/lib/Transforms/Vectorize/VPlan.cpp
View File

@@ -242,8 +242,8 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
return Data.PerPartScalars[Def][Instance.Part][0];
}
assert(hasVectorValue(Def, Instance.Part));
auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
assert(hasVectorValue(Def));
auto *VecPart = Data.VPV2Vector[Def];
if (!VecPart->getType()->isVectorTy()) {
assert(Instance.Lane.isFirstLane() && "cannot get lane > 0 for scalar");
return VecPart;
@@ -255,20 +255,20 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
return Extract;
}
Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
if (NeedsScalar) {
assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def, Part) ||
assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def) ||
!vputils::onlyFirstLaneUsed(Def) ||
(hasScalarValue(Def, VPIteration(Part, 0)) &&
Data.PerPartScalars[Def][Part].size() == 1)) &&
(hasScalarValue(Def, VPIteration(0, 0)) &&
Data.PerPartScalars[Def][0].size() == 1)) &&
"Trying to access a single scalar per part but has multiple scalars "
"per part.");
return get(Def, VPIteration(Part, 0));
return get(Def, VPIteration(0, 0));
}
// If Values have been set for this Def return the one relevant for \p Part.
if (hasVectorValue(Def, Part))
return Data.PerPartOutput[Def][Part];
if (hasVectorValue(Def))
return Data.VPV2Vector[Def];
auto GetBroadcastInstrs = [this, Def](Value *V) {
bool SafeToHoist = Def->isDefinedOutsideLoopRegions();
@@ -290,21 +290,19 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
return Shuf;
};
if (!hasScalarValue(Def, {Part, 0})) {
if (!hasScalarValue(Def, {0, 0})) {
assert(Def->isLiveIn() && "expected a live-in");
if (Part != 0)
return get(Def, 0);
Value *IRV = Def->getLiveInIRValue();
Value *B = GetBroadcastInstrs(IRV);
set(Def, B, Part);
set(Def, B);
return B;
}
Value *ScalarValue = get(Def, {Part, 0});
Value *ScalarValue = get(Def, {0, 0});
// If we aren't vectorizing, we can just copy the scalar map values over
// to the vector map.
if (VF.isScalar()) {
set(Def, ScalarValue, Part);
set(Def, ScalarValue);
return ScalarValue;
}
@@ -312,7 +310,7 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
unsigned LastLane = IsUniform ? 0 : VF.getKnownMinValue() - 1;
// Check if there is a scalar value for the selected lane.
if (!hasScalarValue(Def, {Part, LastLane})) {
if (!hasScalarValue(Def, {0, LastLane})) {
// At the moment, VPWidenIntOrFpInductionRecipes, VPScalarIVStepsRecipes and
// VPExpandSCEVRecipes can also be uniform.
assert((isa<VPWidenIntOrFpInductionRecipe>(Def->getDefiningRecipe()) ||
@@ -323,7 +321,7 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
LastLane = 0;
}
auto *LastInst = cast<Instruction>(get(Def, {Part, LastLane}));
auto *LastInst = cast<Instruction>(get(Def, {0, LastLane}));
// Set the insert point after the last scalarized instruction or after the
// last PHI, if LastInst is a PHI. This ensures the insertelement sequence
// will directly follow the scalar definitions.
@@ -343,15 +341,15 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
Value *VectorValue = nullptr;
if (IsUniform) {
VectorValue = GetBroadcastInstrs(ScalarValue);
set(Def, VectorValue, Part);
set(Def, VectorValue);
} else {
// Initialize packing with insertelements to start from undef.
assert(!VF.isScalable() && "VF is assumed to be non scalable.");
Value *Undef = PoisonValue::get(VectorType::get(LastInst->getType(), VF));
set(Def, Undef, Part);
set(Def, Undef);
for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
packScalarIntoVectorValue(Def, {Part, Lane});
VectorValue = get(Def, Part);
packScalarIntoVectorValue(Def, {0, Lane});
VectorValue = get(Def);
}
Builder.restoreIP(OldIP);
return VectorValue;
@@ -406,10 +404,10 @@ void VPTransformState::setDebugLocFrom(DebugLoc DL) {
void VPTransformState::packScalarIntoVectorValue(VPValue *Def,
const VPIteration &Instance) {
Value *ScalarInst = get(Def, Instance);
Value *VectorValue = get(Def, Instance.Part);
Value *VectorValue = get(Def);
VectorValue = Builder.CreateInsertElement(
VectorValue, ScalarInst, Instance.Lane.getAsRuntimeExpr(Builder, VF));
set(Def, VectorValue, Instance.Part);
set(Def, VectorValue);
}
BasicBlock *
@@ -1074,12 +1072,12 @@ void VPlan::execute(VPTransformState *State) {
isa<VPWidenIntOrFpInductionRecipe>(&R)) {
PHINode *Phi = nullptr;
if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
} else {
auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
"recipe generating only scalars should have been replaced");
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
Phi = cast<PHINode>(GEP->getPointerOperand());
}
@@ -1092,7 +1090,7 @@ void VPlan::execute(VPTransformState *State) {
// Use the steps for the last part as backedge value for the induction.
if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand(), 0));
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
continue;
}
@@ -1101,8 +1099,8 @@ void VPlan::execute(VPTransformState *State) {
isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
Value *Phi = State->get(PhiR, 0, NeedsScalar);
Value *Val = State->get(PhiR->getBackedgeValue(), 0, NeedsScalar);
Value *Phi = State->get(PhiR, NeedsScalar);
Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}

46
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@@ -263,30 +263,22 @@ struct VPTransformState {
std::optional<VPIteration> Instance;
struct DataState {
/// A type for vectorized values in the new loop. Each value from the
/// original loop, when vectorized, is represented by UF vector values in
/// the new unrolled loop, where UF is the unroll factor.
typedef SmallVector<Value *, 2> PerPartValuesTy;
DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
// Each value from the original loop, when vectorized, is represented by a
// vector value in the map.
DenseMap<VPValue *, Value *> VPV2Vector;
using ScalarsPerPartValuesTy = SmallVector<SmallVector<Value *, 4>, 2>;
DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
} Data;
/// Get the generated vector Value for a given VPValue \p Def and a given \p
/// Part if \p IsScalar is false, otherwise return the generated scalar
/// for \p Part. \See set.
Value *get(VPValue *Def, unsigned Part, bool IsScalar = false);
/// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
/// is false, otherwise return the generated scalar. \See set.
Value *get(VPValue *Def, bool IsScalar = false);
/// Get the generated Value for a given VPValue and given Part and Lane.
Value *get(VPValue *Def, const VPIteration &Instance);
bool hasVectorValue(VPValue *Def, unsigned Part) {
auto I = Data.PerPartOutput.find(Def);
return I != Data.PerPartOutput.end() && Part < I->second.size() &&
I->second[Part];
}
bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
bool hasScalarValue(VPValue *Def, VPIteration Instance) {
auto I = Data.PerPartScalars.find(Def);
@@ -298,28 +290,22 @@ struct VPTransformState {
I->second[Instance.Part][CacheIdx];
}
/// Set the generated vector Value for a given VPValue and a given Part, if \p
/// IsScalar is false. If \p IsScalar is true, set the scalar in (Part, 0).
void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar = false) {
/// Set the generated vector Value for a given VPValue, if \p
/// IsScalar is false. If \p IsScalar is true, set the scalar in lane 0.
void set(VPValue *Def, Value *V, bool IsScalar = false) {
if (IsScalar) {
set(Def, V, VPIteration(Part, 0));
set(Def, V, VPIteration(0, 0));
return;
}
assert((VF.isScalar() || V->getType()->isVectorTy()) &&
"scalar values must be stored as (Part, 0)");
if (!Data.PerPartOutput.count(Def)) {
DataState::PerPartValuesTy Entry(1);
Data.PerPartOutput[Def] = Entry;
}
Data.PerPartOutput[Def][Part] = V;
"scalar values must be stored as (0, 0)");
Data.VPV2Vector[Def] = V;
}
/// Reset an existing vector value for \p Def and a given \p Part.
void reset(VPValue *Def, Value *V, unsigned Part) {
auto Iter = Data.PerPartOutput.find(Def);
assert(Iter != Data.PerPartOutput.end() &&
"need to overwrite existing value");
Iter->second[Part] = V;
void reset(VPValue *Def, Value *V) {
assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");
Data.VPV2Vector[Def] = V;
}
/// Set the generated scalar \p V for \p Def and the given \p Instance.

201
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View File

@@ -404,8 +404,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
if (Instruction::isBinaryOp(getOpcode())) {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *A = State.get(getOperand(0), 0, OnlyFirstLaneUsed);
Value *B = State.get(getOperand(1), 0, OnlyFirstLaneUsed);
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
auto *Res =
Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
if (auto *I = dyn_cast<Instruction>(Res))
@@ -415,19 +415,19 @@ Value *VPInstruction::generate(VPTransformState &State) {
switch (getOpcode()) {
case VPInstruction::Not: {
Value *A = State.get(getOperand(0), 0);
Value *A = State.get(getOperand(0));
return Builder.CreateNot(A, Name);
}
case Instruction::ICmp: {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *A = State.get(getOperand(0), 0, OnlyFirstLaneUsed);
Value *B = State.get(getOperand(1), 0, OnlyFirstLaneUsed);
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
return Builder.CreateCmp(getPredicate(), A, B, Name);
}
case Instruction::Select: {
Value *Cond = State.get(getOperand(0), 0);
Value *Op1 = State.get(getOperand(1), 0);
Value *Op2 = State.get(getOperand(2), 0);
Value *Cond = State.get(getOperand(0));
Value *Op1 = State.get(getOperand(1));
Value *Op2 = State.get(getOperand(2));
return Builder.CreateSelect(Cond, Op1, Op2, Name);
}
case VPInstruction::ActiveLaneMask: {
@@ -461,10 +461,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
// v2 = a[i, i+1, i+2, i+3];
// v3 = vector(v1(3), v2(0, 1, 2))
auto *V1 = State.get(getOperand(0), 0);
auto *V1 = State.get(getOperand(0));
if (!V1->getType()->isVectorTy())
return V1;
Value *V2 = State.get(getOperand(1), 0);
Value *V2 = State.get(getOperand(1));
return Builder.CreateVectorSplice(V1, V2, -1, Name);
}
case VPInstruction::CalculateTripCountMinusVF: {
@@ -530,8 +530,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case VPInstruction::BranchOnCount: {
// First create the compare.
Value *IV = State.get(getOperand(0), 0, /*IsScalar*/ true);
Value *TC = State.get(getOperand(1), 0, /*IsScalar*/ true);
Value *IV = State.get(getOperand(0), /*IsScalar*/ true);
Value *TC = State.get(getOperand(1), /*IsScalar*/ true);
Value *Cond = Builder.CreateICmpEQ(IV, TC);
// Now create the branch.
@@ -566,7 +566,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
unsigned UF = getNumOperands() - 1;
VectorParts RdxParts(UF);
for (unsigned Part = 0; Part < UF; ++Part)
RdxParts[Part] = State.get(getOperand(1 + Part), 0, PhiR->isInLoop());
RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());
// If the vector reduction can be performed in a smaller type, we truncate
// then extend the loop exit value to enable InstCombine to evaluate the
@@ -637,30 +637,29 @@ Value *VPInstruction::generate(VPTransformState &State) {
VPIteration(0, VPLane::getLaneFromEnd(State.VF, Offset)));
} else {
assert(Offset <= 1 && "invalid offset to extract from");
// When loop is unrolled without vectorizing, retrieve UF - Offset.
Res = State.get(getOperand(0), 1 - Offset);
Res = State.get(getOperand(0));
}
if (isa<ExtractElementInst>(Res))
Res->setName(Name);
return Res;
}
case VPInstruction::LogicalAnd: {
Value *A = State.get(getOperand(0), 0);
Value *B = State.get(getOperand(1), 0);
Value *A = State.get(getOperand(0));
Value *B = State.get(getOperand(1));
return Builder.CreateLogicalAnd(A, B, Name);
}
case VPInstruction::PtrAdd: {
assert(vputils::onlyFirstLaneUsed(this) &&
"can only generate first lane for PtrAdd");
Value *Ptr = State.get(getOperand(0), 0, /* IsScalar */ true);
Value *Addend = State.get(getOperand(1), 0, /* IsScalar */ true);
Value *Ptr = State.get(getOperand(0), /* IsScalar */ true);
Value *Addend = State.get(getOperand(1), /* IsScalar */ true);
return Builder.CreatePtrAdd(Ptr, Addend, Name);
}
case VPInstruction::ResumePhi: {
Value *IncomingFromVPlanPred =
State.get(getOperand(0), 0, /* IsScalar */ true);
State.get(getOperand(0), /* IsScalar */ true);
Value *IncomingFromOtherPreds =
State.get(getOperand(1), 0, /* IsScalar */ true);
State.get(getOperand(1), /* IsScalar */ true);
auto *NewPhi =
Builder.CreatePHI(IncomingFromOtherPreds->getType(), 2, Name);
BasicBlock *VPlanPred =
@@ -731,7 +730,7 @@ void VPInstruction::execute(VPTransformState &State) {
(GeneratedValue->getType()->isVectorTy() == !GeneratesPerFirstLaneOnly ||
State.VF.isScalar()) &&
"scalar value but not only first lane defined");
State.set(this, GeneratedValue, 0,
State.set(this, GeneratedValue,
/*IsScalar*/ GeneratesPerFirstLaneOnly);
}
@@ -921,7 +920,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
else if (VFTy && !VFTy->getParamType(I.index())->isVectorTy())
Arg = State.get(I.value(), VPIteration(0, 0));
else
Arg = State.get(I.value(), 0);
Arg = State.get(I.value());
if (UseIntrinsic &&
isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
TysForDecl.push_back(Arg->getType());
@@ -952,7 +951,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
V->copyFastMathFlags(CI);
if (!V->getType()->isVoidTy())
State.set(this, V, 0);
State.set(this, V);
State.addMetadata(V, CI);
}
@@ -1056,11 +1055,11 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
auto *InvarCond =
isInvariantCond() ? State.get(getCond(), VPIteration(0, 0)) : nullptr;
Value *Cond = InvarCond ? InvarCond : State.get(getCond(), 0);
Value *Op0 = State.get(getOperand(1), 0);
Value *Op1 = State.get(getOperand(2), 0);
Value *Cond = InvarCond ? InvarCond : State.get(getCond());
Value *Op0 = State.get(getOperand(1));
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
State.set(this, Sel, 0);
State.set(this, Sel);
State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
}
@@ -1146,7 +1145,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
// Just widen unops and binops.
SmallVector<Value *, 2> Ops;
for (VPValue *VPOp : operands())
Ops.push_back(State.get(VPOp, 0));
Ops.push_back(State.get(VPOp));
Value *V = Builder.CreateNAryOp(Opcode, Ops);
@@ -1154,23 +1153,23 @@ void VPWidenRecipe::execute(VPTransformState &State) {
setFlags(VecOp);
// Use this vector value for all users of the original instruction.
State.set(this, V, 0);
State.set(this, V);
State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
break;
}
case Instruction::Freeze: {
Value *Op = State.get(getOperand(0), 0);
Value *Op = State.get(getOperand(0));
Value *Freeze = Builder.CreateFreeze(Op);
State.set(this, Freeze, 0);
State.set(this, Freeze);
break;
}
case Instruction::ICmp:
case Instruction::FCmp: {
// Widen compares. Generate vector compares.
bool FCmp = Opcode == Instruction::FCmp;
Value *A = State.get(getOperand(0), 0);
Value *B = State.get(getOperand(1), 0);
Value *A = State.get(getOperand(0));
Value *B = State.get(getOperand(1));
Value *C = nullptr;
if (FCmp) {
// Propagate fast math flags.
@@ -1181,7 +1180,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
} else {
C = Builder.CreateICmp(getPredicate(), A, B);
}
State.set(this, C, 0);
State.set(this, C);
State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
break;
}
@@ -1196,7 +1195,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
// Verify that VPlan type inference results agree with the type of the
// generated values.
assert(VectorType::get(State.TypeAnalysis.inferScalarType(this), State.VF) ==
State.get(this, 0)->getType() &&
State.get(this)->getType() &&
"inferred type and type from generated instructions do not match");
#endif
}
@@ -1283,11 +1282,11 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
State.setDebugLocFrom(getDebugLoc());
assert(State.get(getOperand(0), 0)->getType()->isVectorTy() &&
assert(State.get(getOperand(0))->getType()->isVectorTy() &&
"VPWidenEVLRecipe should not be used for scalars");
VPValue *EVL = getEVL();
Value *EVLArg = State.get(EVL, 0, /*NeedsScalar=*/true);
Value *EVLArg = State.get(EVL, /*NeedsScalar=*/true);
IRBuilderBase &BuilderIR = State.Builder;
VectorBuilder Builder(BuilderIR);
Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue());
@@ -1295,7 +1294,7 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
SmallVector<Value *, 4> Ops;
for (unsigned I = 0, E = getNumOperands() - 1; I < E; ++I) {
VPValue *VPOp = getOperand(I);
Ops.push_back(State.get(VPOp, 0));
Ops.push_back(State.get(VPOp));
}
Builder.setMask(Mask).setEVL(EVLArg);
@@ -1306,7 +1305,7 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
if (isa<FPMathOperator>(VPInst))
setFlags(cast<Instruction>(VPInst));
State.set(this, VPInst, 0);
State.set(this, VPInst);
State.addMetadata(VPInst,
dyn_cast_or_null<Instruction>(getUnderlyingValue()));
}
@@ -1338,9 +1337,9 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "Not vectorizing?");
Type *DestTy = VectorType::get(getResultType(), State.VF);
VPValue *Op = getOperand(0);
Value *A = State.get(Op, 0);
Value *A = State.get(Op);
Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
State.set(this, Cast, 0);
State.set(this, Cast);
State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
}
@@ -1474,7 +1473,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
if (VPValue *SplatVFOperand = getSplatVFValue()) {
// The recipe has been unrolled. In that case, fetch the splat value for the
// induction increment.
SplatVF = State.get(SplatVFOperand, 0);
SplatVF = State.get(SplatVFOperand);
} else {
// Multiply the vectorization factor by the step using integer or
// floating-point arithmetic as appropriate.
@@ -1503,7 +1502,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind");
VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
VecInd->setDebugLoc(EntryVal->getDebugLoc());
State.set(this, VecInd, 0);
State.set(this, VecInd);
Instruction *LastInduction = cast<Instruction>(Builder.CreateBinOp(
AddOp, VecInd, SplatVF, "vec.ind.next", EntryVal->getDebugLoc()));
@@ -1628,7 +1627,7 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
State.set(this, Add, 0);
State.set(this, Add);
// It's useful to record the lane values too for the known minimum number
// of elements so we do those below. This improves the code quality when
// trying to extract the first element, for example.
@@ -1691,7 +1690,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],
ArrayRef(Ops).drop_front(), "", isInBounds());
Value *Splat = State.Builder.CreateVectorSplat(State.VF, NewGEP);
State.set(this, Splat, 0);
State.set(this, Splat);
State.addMetadata(Splat, GEP);
} else {
// If the GEP has at least one loop-varying operand, we are sure to
@@ -1700,7 +1699,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
// won't broadcast it.
auto *Ptr = isPointerLoopInvariant()
? State.get(getOperand(0), VPIteration(0, 0))
: State.get(getOperand(0), 0);
: State.get(getOperand(0));
// Collect all the indices for the new GEP. If any index is
// loop-invariant, we won't broadcast it.
@@ -1710,7 +1709,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
if (isIndexLoopInvariant(I - 1))
Indices.push_back(State.get(Operand, VPIteration(0, 0)));
else
Indices.push_back(State.get(Operand, 0));
Indices.push_back(State.get(Operand));
}
// Create the new GEP. Note that this GEP may be a scalar if VF == 1,
@@ -1719,7 +1718,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
Indices, "", isInBounds());
assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector");
State.set(this, NewGEP, 0);
State.set(this, NewGEP);
State.addMetadata(NewGEP, GEP);
}
}
@@ -1773,7 +1772,7 @@ void VPVectorPointerRecipe ::execute(VPTransformState &State) {
ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, Increment, "", InBounds);
}
State.set(this, ResultPtr, 0, /*IsScalar*/ true);
State.set(this, ResultPtr, /*IsScalar*/ true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1813,17 +1812,17 @@ void VPBlendRecipe::execute(VPTransformState &State) {
for (unsigned In = 0; In < NumIncoming; ++In) {
// We might have single edge PHIs (blocks) - use an identity
// 'select' for the first PHI operand.
Value *In0 = State.get(getIncomingValue(In), 0, OnlyFirstLaneUsed);
Value *In0 = State.get(getIncomingValue(In), OnlyFirstLaneUsed);
if (In == 0)
Result = In0; // Initialize with the first incoming value.
else {
// Select between the current value and the previous incoming edge
// based on the incoming mask.
Value *Cond = State.get(getMask(In), 0, OnlyFirstLaneUsed);
Value *Cond = State.get(getMask(In), OnlyFirstLaneUsed);
Result = State.Builder.CreateSelect(Cond, In0, Result, "predphi");
}
}
State.set(this, Result, 0, OnlyFirstLaneUsed);
State.set(this, Result, OnlyFirstLaneUsed);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1852,14 +1851,14 @@ void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
void VPReductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Reduction being replicated.");
Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
Value *PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
RecurKind Kind = RdxDesc.getRecurrenceKind();
// Propagate the fast-math flags carried by the underlying instruction.
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
Value *NewVecOp = State.get(getVecOp(), 0);
Value *NewVecOp = State.get(getVecOp());
if (VPValue *Cond = getCondOp()) {
Value *NewCond = State.get(Cond, 0, State.VF.isScalar());
Value *NewCond = State.get(Cond, State.VF.isScalar());
VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
@@ -1888,7 +1887,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
PrevInChain = NewRed;
NextInChain = NewRed;
} else {
PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
NewRed = createReduction(State.Builder, RdxDesc, NewVecOp);
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
@@ -1897,7 +1896,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
NextInChain = State.Builder.CreateBinOp(
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
}
State.set(this, NextInChain, 0, /*IsScalar*/ true);
State.set(this, NextInChain, /*IsScalar*/ true);
}
void VPReductionEVLRecipe::execute(VPTransformState &State) {
@@ -1910,8 +1909,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
RecurKind Kind = RdxDesc.getRecurrenceKind();
Value *Prev = State.get(getChainOp(), 0, /*IsScalar*/ true);
Value *VecOp = State.get(getVecOp(), 0);
Value *Prev = State.get(getChainOp(), /*IsScalar*/ true);
Value *VecOp = State.get(getVecOp());
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
VectorBuilder VBuilder(Builder);
@@ -1919,7 +1918,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
Value *Mask;
// TODO: move the all-true mask generation into VectorBuilder.
if (VPValue *CondOp = getCondOp())
Mask = State.get(CondOp, 0);
Mask = State.get(CondOp);
else
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
VBuilder.setMask(Mask);
@@ -1935,7 +1934,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
NewRed = Builder.CreateBinOp(
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, Prev);
}
State.set(this, NewRed, 0, /*IsScalar*/ true);
State.set(this, NewRed, /*IsScalar*/ true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2065,7 +2064,7 @@ void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
Value *ConditionBit = nullptr;
VPValue *BlockInMask = getMask();
if (BlockInMask) {
ConditionBit = State.get(BlockInMask, 0);
ConditionBit = State.get(BlockInMask);
if (ConditionBit->getType()->isVectorTy())
ConditionBit = State.Builder.CreateExtractElement(
ConditionBit, State.Builder.getInt32(Lane));
@@ -2098,19 +2097,19 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
// needed. In this case the recipe of the predicated instruction is marked to
// also do that packing, thereby "hoisting" the insert-element sequence.
// Otherwise, a phi node for the scalar value is needed.
if (State.hasVectorValue(getOperand(0), 0)) {
Value *VectorValue = State.get(getOperand(0), 0);
if (State.hasVectorValue(getOperand(0))) {
Value *VectorValue = State.get(getOperand(0));
InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
if (State.hasVectorValue(this, 0))
State.reset(this, VPhi, 0);
if (State.hasVectorValue(this))
State.reset(this, VPhi);
else
State.set(this, VPhi, 0);
State.set(this, VPhi);
// NOTE: Currently we need to update the value of the operand, so the next
// predicated iteration inserts its generated value in the correct vector.
State.reset(getOperand(0), VPhi, 0);
State.reset(getOperand(0), VPhi);
} else {
Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
@@ -2190,12 +2189,12 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
if (auto *VPMask = getMask()) {
// Mask reversal is only needed for non-all-one (null) masks, as reverse
// of a null all-one mask is a null mask.
Mask = State.get(VPMask, 0);
Mask = State.get(VPMask);
if (isReverse())
Mask = Builder.CreateVectorReverse(Mask, "reverse");
}
Value *Addr = State.get(getAddr(), 0, /*IsScalar*/ !CreateGather);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateGather);
Value *NewLI;
if (CreateGather) {
NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,
@@ -2211,7 +2210,7 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
State.addMetadata(NewLI, LI);
if (Reverse)
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
State.set(this, NewLI, 0);
State.set(this, NewLI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2247,10 +2246,10 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
State.setDebugLocFrom(getDebugLoc());
CallInst *NewLI;
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
Value *Addr = State.get(getAddr(), 0, !CreateGather);
Value *Addr = State.get(getAddr(), !CreateGather);
Value *Mask = nullptr;
if (VPValue *VPMask = getMask()) {
Mask = State.get(VPMask, 0);
Mask = State.get(VPMask);
if (isReverse())
Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
} else {
@@ -2273,7 +2272,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
Instruction *Res = NewLI;
if (isReverse())
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
State.set(this, Res, 0);
State.set(this, Res);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2300,12 +2299,12 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
if (auto *VPMask = getMask()) {
// Mask reversal is only needed for non-all-one (null) masks, as reverse
// of a null all-one mask is a null mask.
Mask = State.get(VPMask, 0);
Mask = State.get(VPMask);
if (isReverse())
Mask = Builder.CreateVectorReverse(Mask, "reverse");
}
Value *StoredVal = State.get(StoredVPValue, 0);
Value *StoredVal = State.get(StoredVPValue);
if (isReverse()) {
// If we store to reverse consecutive memory locations, then we need
// to reverse the order of elements in the stored value.
@@ -2313,7 +2312,7 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
// We don't want to update the value in the map as it might be used in
// another expression. So don't call resetVectorValue(StoredVal).
}
Value *Addr = State.get(getAddr(), 0, /*IsScalar*/ !CreateScatter);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
Instruction *NewSI = nullptr;
if (CreateScatter)
NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
@@ -2343,19 +2342,19 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
State.setDebugLocFrom(getDebugLoc());
CallInst *NewSI = nullptr;
Value *StoredVal = State.get(StoredValue, 0);
Value *StoredVal = State.get(StoredValue);
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
if (isReverse())
StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");
Value *Mask = nullptr;
if (VPValue *VPMask = getMask()) {
Mask = State.get(VPMask, 0);
Mask = State.get(VPMask);
if (isReverse())
Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
} else {
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
}
Value *Addr = State.get(getAddr(), 0, !CreateScatter);
Value *Addr = State.get(getAddr(), !CreateScatter);
if (CreateScatter) {
NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
Intrinsic::vp_scatter,
@@ -2536,7 +2535,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
assert(InterleaveFactor == 2 &&
"Unsupported deinterleave factor for scalable vectors");
auto *ResBlockInMask = State.get(BlockInMask, 0);
auto *ResBlockInMask = State.get(BlockInMask);
SmallVector<Value *, 2> Ops = {ResBlockInMask, ResBlockInMask};
auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(),
State.VF.getKnownMinValue() * 2, true);
@@ -2548,7 +2547,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
if (!BlockInMask)
return MaskForGaps;
Value *ResBlockInMask = State.get(BlockInMask, 0);
Value *ResBlockInMask = State.get(BlockInMask);
Value *ShuffledMask = State.Builder.CreateShuffleVector(
ResBlockInMask,
createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
@@ -2608,7 +2607,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
if (Group->isReverse())
StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
State.set(VPDefs[J], StridedVec, 0);
State.set(VPDefs[J], StridedVec);
++J;
}
@@ -2641,7 +2640,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
if (Group->isReverse())
StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
State.set(VPDefs[J], StridedVec, 0);
State.set(VPDefs[J], StridedVec);
++J;
}
return;
@@ -2671,7 +2670,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
continue;
}
Value *StoredVec = State.get(StoredValues[StoredIdx], 0);
Value *StoredVec = State.get(StoredValues[StoredIdx]);
++StoredIdx;
if (Group->isReverse())
@@ -2738,7 +2737,7 @@ void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Phi->addIncoming(Start, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, 0, /*IsScalar*/ true);
State.set(this, Phi, /*IsScalar*/ true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2783,7 +2782,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
"Recipe should have been replaced");
auto *IVR = getParent()->getPlan()->getCanonicalIV();
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
unsigned CurrentPart = getUnrollPart(*this);
// Build a pointer phi
@@ -2800,7 +2799,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
// The recipe has been unrolled. In that case, fetch the single pointer phi
// shared among all unrolled parts of the recipe.
auto *GEP =
cast<GetElementPtrInst>(State.get(getFirstUnrolledPartOperand(), 0));
cast<GetElementPtrInst>(State.get(getFirstUnrolledPartOperand()));
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
}
@@ -2847,7 +2846,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
State.Builder.CreateMul(StartOffset, State.Builder.CreateVectorSplat(
State.VF, ScalarStepValue)),
"vector.gep");
State.set(this, GEP, 0);
State.set(this, GEP);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2892,7 +2891,7 @@ void VPExpandSCEVRecipe::print(raw_ostream &O, const Twine &Indent,
#endif
void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
Value *CanonicalIV = State.get(getOperand(0), 0, /*IsScalar*/ true);
Value *CanonicalIV = State.get(getOperand(0), /*IsScalar*/ true);
Type *STy = CanonicalIV->getType();
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
ElementCount VF = State.VF;
@@ -2906,7 +2905,7 @@ void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
}
Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
State.set(this, CanonicalVectorIV, 0);
State.set(this, CanonicalVectorIV);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2944,7 +2943,7 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
PHINode *Phi = PHINode::Create(VecTy, 2, "vector.recur");
Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
Phi->addIncoming(VectorInit, VectorPH);
State.set(this, Phi, 0);
State.set(this, Phi);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2978,7 +2977,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
"recipe must be in the vector loop header");
auto *Phi = PHINode::Create(VecTy, 2, "vec.phi");
Phi->insertBefore(HeaderBB->getFirstInsertionPt());
State.set(this, Phi, 0, IsInLoop);
State.set(this, Phi, IsInLoop);
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
@@ -2994,7 +2993,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
} else {
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(VectorPH->getTerminator());
StartV = Iden = State.get(StartVPV, 0);
StartV = Iden = State.get(StartVPV);
}
} else {
Iden = llvm::getRecurrenceIdentity(RK, VecTy->getScalarType(),
@@ -3016,7 +3015,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
}
}
Phi = cast<PHINode>(State.get(this, 0, IsInLoop));
Phi = cast<PHINode>(State.get(this, IsInLoop));
Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
Phi->addIncoming(StartVal, VectorPH);
}
@@ -3036,10 +3035,10 @@ void VPWidenPHIRecipe::execute(VPTransformState &State) {
assert(EnableVPlanNativePath &&
"Non-native vplans are not expected to have VPWidenPHIRecipes.");
Value *Op0 = State.get(getOperand(0), 0);
Value *Op0 = State.get(getOperand(0));
Type *VecTy = Op0->getType();
Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
State.set(this, VecPhi, 0);
State.set(this, VecPhi);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -3067,12 +3066,12 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
// remove VPActiveLaneMaskPHIRecipe.
void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Value *StartMask = State.get(getOperand(0), 0);
Value *StartMask = State.get(getOperand(0));
PHINode *Phi =
State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
Phi->addIncoming(StartMask, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, 0);
State.set(this, Phi);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -3092,7 +3091,7 @@ void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) {
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv");
Phi->addIncoming(Start, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, 0, /*IsScalar=*/true);
State.set(this, Phi, /*IsScalar=*/true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)