3bc2dade36
This work feeds part of PR https://github.com/llvm/llvm-project/pull/88385, and adds support for vectorising loops with uncountable early exits and outside users of loop-defined variables. When calculating the final value from an uncountable early exit we need to calculate the vector lane that triggered the exit, and hence determine the value at the point we exited. All code for calculating the last value when exiting the loop early now lives in a new vector.early.exit block, which sits between the middle.split block and the original exit block. Doing this required two fixes: 1. The vplan verifier incorrectly assumed that the block containing a definition always dominates the block of the user. That's not true if you can arrive at the use block from multiple incoming blocks. This is possible for early exit loops where both the early exit and the latch jump to the same block. 2. We were adding the new vector.early.exit to the wrong parent loop. It needs to have the same parent as the actual early exit block from the original loop. I've added a new ExtractFirstActive VPInstruction that extracts the first active lane of a vector, i.e. the lane of the vector predicate that triggered the exit. NOTE: The IR generated for dealing with live-outs from early exit loops is unoptimised, as opposed to normal loops. This inevitably leads to poor quality code, but this can be fixed up later.
4352 lines
156 KiB
C++
4352 lines
156 KiB
C++
//===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// This file contains the declarations of the Vectorization Plan base classes:
|
|
/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
|
|
/// VPBlockBase, together implementing a Hierarchical CFG;
|
|
/// 2. Pure virtual VPRecipeBase serving as the base class for recipes contained
|
|
/// within VPBasicBlocks;
|
|
/// 3. Pure virtual VPSingleDefRecipe serving as a base class for recipes that
|
|
/// also inherit from VPValue.
|
|
/// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
|
|
/// instruction;
|
|
/// 5. The VPlan class holding a candidate for vectorization;
|
|
/// 6. The VPlanPrinter class providing a way to print a plan in dot format;
|
|
/// These are documented in docs/VectorizationPlan.rst.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
|
#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
|
|
|
#include "VPlanAnalysis.h"
|
|
#include "VPlanValue.h"
|
|
#include "llvm/ADT/DenseMap.h"
|
|
#include "llvm/ADT/SmallBitVector.h"
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/Twine.h"
|
|
#include "llvm/ADT/ilist.h"
|
|
#include "llvm/ADT/ilist_node.h"
|
|
#include "llvm/Analysis/DomTreeUpdater.h"
|
|
#include "llvm/Analysis/IVDescriptors.h"
|
|
#include "llvm/Analysis/LoopInfo.h"
|
|
#include "llvm/Analysis/TargetTransformInfo.h"
|
|
#include "llvm/Analysis/VectorUtils.h"
|
|
#include "llvm/IR/DebugLoc.h"
|
|
#include "llvm/IR/FMF.h"
|
|
#include "llvm/IR/Operator.h"
|
|
#include "llvm/Support/InstructionCost.h"
|
|
#include <algorithm>
|
|
#include <cassert>
|
|
#include <cstddef>
|
|
#include <string>
|
|
|
|
namespace llvm {
|
|
|
|
class BasicBlock;
|
|
class DominatorTree;
|
|
class InnerLoopVectorizer;
|
|
class IRBuilderBase;
|
|
class LoopInfo;
|
|
class raw_ostream;
|
|
class RecurrenceDescriptor;
|
|
class SCEV;
|
|
class Type;
|
|
class VPBasicBlock;
|
|
class VPBuilder;
|
|
class VPRegionBlock;
|
|
class VPlan;
|
|
class VPReplicateRecipe;
|
|
class VPlanSlp;
|
|
class Value;
|
|
class LoopVectorizationCostModel;
|
|
class LoopVersioning;
|
|
|
|
struct VPCostContext;
|
|
|
|
namespace Intrinsic {
|
|
typedef unsigned ID;
|
|
}
|
|
|
|
/// Returns a calculation for the total number of elements for a given \p VF.
|
|
/// For fixed width vectors this value is a constant, whereas for scalable
|
|
/// vectors it is an expression determined at runtime.
|
|
Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
|
|
|
|
/// Return a value for Step multiplied by VF.
|
|
Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
|
|
int64_t Step);
|
|
|
|
/// A helper function that returns the reciprocal of the block probability of
|
|
/// predicated blocks. If we return X, we are assuming the predicated block
|
|
/// will execute once for every X iterations of the loop header.
|
|
///
|
|
/// TODO: We should use actual block probability here, if available. Currently,
|
|
/// we always assume predicated blocks have a 50% chance of executing.
|
|
inline unsigned getReciprocalPredBlockProb() { return 2; }
|
|
|
|
/// A range of powers-of-2 vectorization factors with fixed start and
|
|
/// adjustable end. The range includes start and excludes end, e.g.,:
|
|
/// [1, 16) = {1, 2, 4, 8}
|
|
struct VFRange {
|
|
// A power of 2.
|
|
const ElementCount Start;
|
|
|
|
// A power of 2. If End <= Start range is empty.
|
|
ElementCount End;
|
|
|
|
bool isEmpty() const {
|
|
return End.getKnownMinValue() <= Start.getKnownMinValue();
|
|
}
|
|
|
|
VFRange(const ElementCount &Start, const ElementCount &End)
|
|
: Start(Start), End(End) {
|
|
assert(Start.isScalable() == End.isScalable() &&
|
|
"Both Start and End should have the same scalable flag");
|
|
assert(isPowerOf2_32(Start.getKnownMinValue()) &&
|
|
"Expected Start to be a power of 2");
|
|
assert(isPowerOf2_32(End.getKnownMinValue()) &&
|
|
"Expected End to be a power of 2");
|
|
}
|
|
|
|
/// Iterator to iterate over vectorization factors in a VFRange.
|
|
class iterator
|
|
: public iterator_facade_base<iterator, std::forward_iterator_tag,
|
|
ElementCount> {
|
|
ElementCount VF;
|
|
|
|
public:
|
|
iterator(ElementCount VF) : VF(VF) {}
|
|
|
|
bool operator==(const iterator &Other) const { return VF == Other.VF; }
|
|
|
|
ElementCount operator*() const { return VF; }
|
|
|
|
iterator &operator++() {
|
|
VF *= 2;
|
|
return *this;
|
|
}
|
|
};
|
|
|
|
iterator begin() { return iterator(Start); }
|
|
iterator end() {
|
|
assert(isPowerOf2_32(End.getKnownMinValue()));
|
|
return iterator(End);
|
|
}
|
|
};
|
|
|
|
using VPlanPtr = std::unique_ptr<VPlan>;
|
|
|
|
/// In what follows, the term "input IR" refers to code that is fed into the
|
|
/// vectorizer whereas the term "output IR" refers to code that is generated by
|
|
/// the vectorizer.
|
|
|
|
/// VPLane provides a way to access lanes in both fixed width and scalable
|
|
/// vectors, where for the latter the lane index sometimes needs calculating
|
|
/// as a runtime expression.
|
|
class VPLane {
|
|
public:
|
|
/// Kind describes how to interpret Lane.
|
|
enum class Kind : uint8_t {
|
|
/// For First, Lane is the index into the first N elements of a
|
|
/// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
|
|
First,
|
|
/// For ScalableLast, Lane is the offset from the start of the last
|
|
/// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
|
|
/// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
|
|
/// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
|
|
ScalableLast
|
|
};
|
|
|
|
private:
|
|
/// in [0..VF)
|
|
unsigned Lane;
|
|
|
|
/// Indicates how the Lane should be interpreted, as described above.
|
|
Kind LaneKind;
|
|
|
|
public:
|
|
VPLane(unsigned Lane) : Lane(Lane), LaneKind(VPLane::Kind::First) {}
|
|
VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
|
|
|
|
static VPLane getFirstLane() { return VPLane(0, VPLane::Kind::First); }
|
|
|
|
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) {
|
|
assert(Offset > 0 && Offset <= VF.getKnownMinValue() &&
|
|
"trying to extract with invalid offset");
|
|
unsigned LaneOffset = VF.getKnownMinValue() - Offset;
|
|
Kind LaneKind;
|
|
if (VF.isScalable())
|
|
// In this case 'LaneOffset' refers to the offset from the start of the
|
|
// last subvector with VF.getKnownMinValue() elements.
|
|
LaneKind = VPLane::Kind::ScalableLast;
|
|
else
|
|
LaneKind = VPLane::Kind::First;
|
|
return VPLane(LaneOffset, LaneKind);
|
|
}
|
|
|
|
static VPLane getLastLaneForVF(const ElementCount &VF) {
|
|
return getLaneFromEnd(VF, 1);
|
|
}
|
|
|
|
/// Returns a compile-time known value for the lane index and asserts if the
|
|
/// lane can only be calculated at runtime.
|
|
unsigned getKnownLane() const {
|
|
assert(LaneKind == Kind::First);
|
|
return Lane;
|
|
}
|
|
|
|
/// Returns an expression describing the lane index that can be used at
|
|
/// runtime.
|
|
Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
|
|
|
|
/// Returns the Kind of lane offset.
|
|
Kind getKind() const { return LaneKind; }
|
|
|
|
/// Returns true if this is the first lane of the whole vector.
|
|
bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
|
|
|
|
/// Maps the lane to a cache index based on \p VF.
|
|
unsigned mapToCacheIndex(const ElementCount &VF) const {
|
|
switch (LaneKind) {
|
|
case VPLane::Kind::ScalableLast:
|
|
assert(VF.isScalable() && Lane < VF.getKnownMinValue());
|
|
return VF.getKnownMinValue() + Lane;
|
|
default:
|
|
assert(Lane < VF.getKnownMinValue());
|
|
return Lane;
|
|
}
|
|
}
|
|
};
|
|
|
|
/// VPTransformState holds information passed down when "executing" a VPlan,
|
|
/// needed for generating the output IR.
|
|
struct VPTransformState {
|
|
VPTransformState(const TargetTransformInfo *TTI, ElementCount VF, unsigned UF,
|
|
LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder,
|
|
InnerLoopVectorizer *ILV, VPlan *Plan,
|
|
Loop *CurrentParentLoop, Type *CanonicalIVTy);
|
|
/// Target Transform Info.
|
|
const TargetTransformInfo *TTI;
|
|
|
|
/// The chosen Vectorization Factor of the loop being vectorized.
|
|
ElementCount VF;
|
|
|
|
/// Hold the index to generate specific scalar instructions. Null indicates
|
|
/// that all instances are to be generated, using either scalar or vector
|
|
/// instructions.
|
|
std::optional<VPLane> Lane;
|
|
|
|
struct DataState {
|
|
// Each value from the original loop, when vectorized, is represented by a
|
|
// vector value in the map.
|
|
DenseMap<VPValue *, Value *> VPV2Vector;
|
|
|
|
DenseMap<VPValue *, SmallVector<Value *, 4>> VPV2Scalars;
|
|
} Data;
|
|
|
|
/// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
|
|
/// is false, otherwise return the generated scalar. \See set.
|
|
Value *get(VPValue *Def, bool IsScalar = false);
|
|
|
|
/// Get the generated Value for a given VPValue and given Part and Lane.
|
|
Value *get(VPValue *Def, const VPLane &Lane);
|
|
|
|
bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
|
|
|
|
bool hasScalarValue(VPValue *Def, VPLane Lane) {
|
|
auto I = Data.VPV2Scalars.find(Def);
|
|
if (I == Data.VPV2Scalars.end())
|
|
return false;
|
|
unsigned CacheIdx = Lane.mapToCacheIndex(VF);
|
|
return CacheIdx < I->second.size() && I->second[CacheIdx];
|
|
}
|
|
|
|
/// Set the generated vector Value for a given VPValue, if \p
|
|
/// IsScalar is false. If \p IsScalar is true, set the scalar in lane 0.
|
|
void set(VPValue *Def, Value *V, bool IsScalar = false) {
|
|
if (IsScalar) {
|
|
set(Def, V, VPLane(0));
|
|
return;
|
|
}
|
|
assert((VF.isScalar() || V->getType()->isVectorTy()) &&
|
|
"scalar values must be stored as (0, 0)");
|
|
Data.VPV2Vector[Def] = V;
|
|
}
|
|
|
|
/// Reset an existing vector value for \p Def and a given \p Part.
|
|
void reset(VPValue *Def, Value *V) {
|
|
assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");
|
|
Data.VPV2Vector[Def] = V;
|
|
}
|
|
|
|
/// Set the generated scalar \p V for \p Def and the given \p Lane.
|
|
void set(VPValue *Def, Value *V, const VPLane &Lane) {
|
|
auto &Scalars = Data.VPV2Scalars[Def];
|
|
unsigned CacheIdx = Lane.mapToCacheIndex(VF);
|
|
if (Scalars.size() <= CacheIdx)
|
|
Scalars.resize(CacheIdx + 1);
|
|
assert(!Scalars[CacheIdx] && "should overwrite existing value");
|
|
Scalars[CacheIdx] = V;
|
|
}
|
|
|
|
/// Reset an existing scalar value for \p Def and a given \p Lane.
|
|
void reset(VPValue *Def, Value *V, const VPLane &Lane) {
|
|
auto Iter = Data.VPV2Scalars.find(Def);
|
|
assert(Iter != Data.VPV2Scalars.end() &&
|
|
"need to overwrite existing value");
|
|
unsigned CacheIdx = Lane.mapToCacheIndex(VF);
|
|
assert(CacheIdx < Iter->second.size() &&
|
|
"need to overwrite existing value");
|
|
Iter->second[CacheIdx] = V;
|
|
}
|
|
|
|
/// Add additional metadata to \p To that was not present on \p Orig.
|
|
///
|
|
/// Currently this is used to add the noalias annotations based on the
|
|
/// inserted memchecks. Use this for instructions that are *cloned* into the
|
|
/// vector loop.
|
|
void addNewMetadata(Instruction *To, const Instruction *Orig);
|
|
|
|
/// Add metadata from one instruction to another.
|
|
///
|
|
/// This includes both the original MDs from \p From and additional ones (\see
|
|
/// addNewMetadata). Use this for *newly created* instructions in the vector
|
|
/// loop.
|
|
void addMetadata(Value *To, Instruction *From);
|
|
|
|
/// Set the debug location in the builder using the debug location \p DL.
|
|
void setDebugLocFrom(DebugLoc DL);
|
|
|
|
/// Construct the vector value of a scalarized value \p V one lane at a time.
|
|
void packScalarIntoVectorValue(VPValue *Def, const VPLane &Lane);
|
|
|
|
/// Hold state information used when constructing the CFG of the output IR,
|
|
/// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
|
|
struct CFGState {
|
|
/// The previous VPBasicBlock visited. Initially set to null.
|
|
VPBasicBlock *PrevVPBB = nullptr;
|
|
|
|
/// The previous IR BasicBlock created or used. Initially set to the new
|
|
/// header BasicBlock.
|
|
BasicBlock *PrevBB = nullptr;
|
|
|
|
/// The last IR BasicBlock in the output IR. Set to the exit block of the
|
|
/// vector loop.
|
|
BasicBlock *ExitBB = nullptr;
|
|
|
|
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
|
|
/// of replication, maps the BasicBlock of the last replica created.
|
|
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
|
|
|
|
/// Updater for the DominatorTree.
|
|
DomTreeUpdater DTU;
|
|
|
|
CFGState(DominatorTree *DT)
|
|
: DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy) {}
|
|
|
|
/// Returns the BasicBlock* mapped to the pre-header of the loop region
|
|
/// containing \p R.
|
|
BasicBlock *getPreheaderBBFor(VPRecipeBase *R);
|
|
} CFG;
|
|
|
|
/// Hold a pointer to LoopInfo to register new basic blocks in the loop.
|
|
LoopInfo *LI;
|
|
|
|
/// Hold a reference to the IRBuilder used to generate output IR code.
|
|
IRBuilderBase &Builder;
|
|
|
|
/// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
|
|
InnerLoopVectorizer *ILV;
|
|
|
|
/// Pointer to the VPlan code is generated for.
|
|
VPlan *Plan;
|
|
|
|
/// The parent loop object for the current scope, or nullptr.
|
|
Loop *CurrentParentLoop = nullptr;
|
|
|
|
/// LoopVersioning. It's only set up (non-null) if memchecks were
|
|
/// used.
|
|
///
|
|
/// This is currently only used to add no-alias metadata based on the
|
|
/// memchecks. The actually versioning is performed manually.
|
|
LoopVersioning *LVer = nullptr;
|
|
|
|
/// Map SCEVs to their expanded values. Populated when executing
|
|
/// VPExpandSCEVRecipes.
|
|
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
|
|
|
|
/// VPlan-based type analysis.
|
|
VPTypeAnalysis TypeAnalysis;
|
|
};
|
|
|
|
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
|
|
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
|
|
class VPBlockBase {
|
|
friend class VPBlockUtils;
|
|
|
|
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
|
|
|
|
/// An optional name for the block.
|
|
std::string Name;
|
|
|
|
/// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
|
|
/// it is a topmost VPBlockBase.
|
|
VPRegionBlock *Parent = nullptr;
|
|
|
|
/// List of predecessor blocks.
|
|
SmallVector<VPBlockBase *, 1> Predecessors;
|
|
|
|
/// List of successor blocks.
|
|
SmallVector<VPBlockBase *, 1> Successors;
|
|
|
|
/// VPlan containing the block. Can only be set on the entry block of the
|
|
/// plan.
|
|
VPlan *Plan = nullptr;
|
|
|
|
/// Add \p Successor as the last successor to this block.
|
|
void appendSuccessor(VPBlockBase *Successor) {
|
|
assert(Successor && "Cannot add nullptr successor!");
|
|
Successors.push_back(Successor);
|
|
}
|
|
|
|
/// Add \p Predecessor as the last predecessor to this block.
|
|
void appendPredecessor(VPBlockBase *Predecessor) {
|
|
assert(Predecessor && "Cannot add nullptr predecessor!");
|
|
Predecessors.push_back(Predecessor);
|
|
}
|
|
|
|
/// Remove \p Predecessor from the predecessors of this block.
|
|
void removePredecessor(VPBlockBase *Predecessor) {
|
|
auto Pos = find(Predecessors, Predecessor);
|
|
assert(Pos && "Predecessor does not exist");
|
|
Predecessors.erase(Pos);
|
|
}
|
|
|
|
/// Remove \p Successor from the successors of this block.
|
|
void removeSuccessor(VPBlockBase *Successor) {
|
|
auto Pos = find(Successors, Successor);
|
|
assert(Pos && "Successor does not exist");
|
|
Successors.erase(Pos);
|
|
}
|
|
|
|
/// This function replaces one predecessor with another, useful when
|
|
/// trying to replace an old block in the CFG with a new one.
|
|
void replacePredecessor(VPBlockBase *Old, VPBlockBase *New) {
|
|
auto I = find(Predecessors, Old);
|
|
assert(I != Predecessors.end());
|
|
assert(Old->getParent() == New->getParent() &&
|
|
"replaced predecessor must have the same parent");
|
|
*I = New;
|
|
}
|
|
|
|
/// This function replaces one successor with another, useful when
|
|
/// trying to replace an old block in the CFG with a new one.
|
|
void replaceSuccessor(VPBlockBase *Old, VPBlockBase *New) {
|
|
auto I = find(Successors, Old);
|
|
assert(I != Successors.end());
|
|
assert(Old->getParent() == New->getParent() &&
|
|
"replaced successor must have the same parent");
|
|
*I = New;
|
|
}
|
|
|
|
protected:
|
|
VPBlockBase(const unsigned char SC, const std::string &N)
|
|
: SubclassID(SC), Name(N) {}
|
|
|
|
public:
|
|
/// An enumeration for keeping track of the concrete subclass of VPBlockBase
|
|
/// that are actually instantiated. Values of this enumeration are kept in the
|
|
/// SubclassID field of the VPBlockBase objects. They are used for concrete
|
|
/// type identification.
|
|
using VPBlockTy = enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC };
|
|
|
|
using VPBlocksTy = SmallVectorImpl<VPBlockBase *>;
|
|
|
|
virtual ~VPBlockBase() = default;
|
|
|
|
const std::string &getName() const { return Name; }
|
|
|
|
void setName(const Twine &newName) { Name = newName.str(); }
|
|
|
|
/// \return an ID for the concrete type of this object.
|
|
/// This is used to implement the classof checks. This should not be used
|
|
/// for any other purpose, as the values may change as LLVM evolves.
|
|
unsigned getVPBlockID() const { return SubclassID; }
|
|
|
|
VPRegionBlock *getParent() { return Parent; }
|
|
const VPRegionBlock *getParent() const { return Parent; }
|
|
|
|
/// \return A pointer to the plan containing the current block.
|
|
VPlan *getPlan();
|
|
const VPlan *getPlan() const;
|
|
|
|
/// Sets the pointer of the plan containing the block. The block must be the
|
|
/// entry block into the VPlan.
|
|
void setPlan(VPlan *ParentPlan);
|
|
|
|
void setParent(VPRegionBlock *P) { Parent = P; }
|
|
|
|
/// \return the VPBasicBlock that is the entry of this VPBlockBase,
|
|
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
|
/// VPBlockBase is a VPBasicBlock, it is returned.
|
|
const VPBasicBlock *getEntryBasicBlock() const;
|
|
VPBasicBlock *getEntryBasicBlock();
|
|
|
|
/// \return the VPBasicBlock that is the exiting this VPBlockBase,
|
|
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
|
/// VPBlockBase is a VPBasicBlock, it is returned.
|
|
const VPBasicBlock *getExitingBasicBlock() const;
|
|
VPBasicBlock *getExitingBasicBlock();
|
|
|
|
const VPBlocksTy &getSuccessors() const { return Successors; }
|
|
VPBlocksTy &getSuccessors() { return Successors; }
|
|
|
|
iterator_range<VPBlockBase **> successors() { return Successors; }
|
|
iterator_range<VPBlockBase **> predecessors() { return Predecessors; }
|
|
|
|
const VPBlocksTy &getPredecessors() const { return Predecessors; }
|
|
VPBlocksTy &getPredecessors() { return Predecessors; }
|
|
|
|
/// \return the successor of this VPBlockBase if it has a single successor.
|
|
/// Otherwise return a null pointer.
|
|
VPBlockBase *getSingleSuccessor() const {
|
|
return (Successors.size() == 1 ? *Successors.begin() : nullptr);
|
|
}
|
|
|
|
/// \return the predecessor of this VPBlockBase if it has a single
|
|
/// predecessor. Otherwise return a null pointer.
|
|
VPBlockBase *getSinglePredecessor() const {
|
|
return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
|
|
}
|
|
|
|
size_t getNumSuccessors() const { return Successors.size(); }
|
|
size_t getNumPredecessors() const { return Predecessors.size(); }
|
|
|
|
/// An Enclosing Block of a block B is any block containing B, including B
|
|
/// itself. \return the closest enclosing block starting from "this", which
|
|
/// has successors. \return the root enclosing block if all enclosing blocks
|
|
/// have no successors.
|
|
VPBlockBase *getEnclosingBlockWithSuccessors();
|
|
|
|
/// \return the closest enclosing block starting from "this", which has
|
|
/// predecessors. \return the root enclosing block if all enclosing blocks
|
|
/// have no predecessors.
|
|
VPBlockBase *getEnclosingBlockWithPredecessors();
|
|
|
|
/// \return the successors either attached directly to this VPBlockBase or, if
|
|
/// this VPBlockBase is the exit block of a VPRegionBlock and has no
|
|
/// successors of its own, search recursively for the first enclosing
|
|
/// VPRegionBlock that has successors and return them. If no such
|
|
/// VPRegionBlock exists, return the (empty) successors of the topmost
|
|
/// VPBlockBase reached.
|
|
const VPBlocksTy &getHierarchicalSuccessors() {
|
|
return getEnclosingBlockWithSuccessors()->getSuccessors();
|
|
}
|
|
|
|
/// \return the hierarchical successor of this VPBlockBase if it has a single
|
|
/// hierarchical successor. Otherwise return a null pointer.
|
|
VPBlockBase *getSingleHierarchicalSuccessor() {
|
|
return getEnclosingBlockWithSuccessors()->getSingleSuccessor();
|
|
}
|
|
|
|
/// \return the predecessors either attached directly to this VPBlockBase or,
|
|
/// if this VPBlockBase is the entry block of a VPRegionBlock and has no
|
|
/// predecessors of its own, search recursively for the first enclosing
|
|
/// VPRegionBlock that has predecessors and return them. If no such
|
|
/// VPRegionBlock exists, return the (empty) predecessors of the topmost
|
|
/// VPBlockBase reached.
|
|
const VPBlocksTy &getHierarchicalPredecessors() {
|
|
return getEnclosingBlockWithPredecessors()->getPredecessors();
|
|
}
|
|
|
|
/// \return the hierarchical predecessor of this VPBlockBase if it has a
|
|
/// single hierarchical predecessor. Otherwise return a null pointer.
|
|
VPBlockBase *getSingleHierarchicalPredecessor() {
|
|
return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
|
|
}
|
|
|
|
/// Set a given VPBlockBase \p Successor as the single successor of this
|
|
/// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
|
|
/// This VPBlockBase must have no successors.
|
|
void setOneSuccessor(VPBlockBase *Successor) {
|
|
assert(Successors.empty() && "Setting one successor when others exist.");
|
|
assert(Successor->getParent() == getParent() &&
|
|
"connected blocks must have the same parent");
|
|
appendSuccessor(Successor);
|
|
}
|
|
|
|
/// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
|
|
/// successors of this VPBlockBase. This VPBlockBase is not added as
|
|
/// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
|
|
/// successors.
|
|
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
|
|
assert(Successors.empty() && "Setting two successors when others exist.");
|
|
appendSuccessor(IfTrue);
|
|
appendSuccessor(IfFalse);
|
|
}
|
|
|
|
/// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
|
|
/// This VPBlockBase must have no predecessors. This VPBlockBase is not added
|
|
/// as successor of any VPBasicBlock in \p NewPreds.
|
|
void setPredecessors(ArrayRef<VPBlockBase *> NewPreds) {
|
|
assert(Predecessors.empty() && "Block predecessors already set.");
|
|
for (auto *Pred : NewPreds)
|
|
appendPredecessor(Pred);
|
|
}
|
|
|
|
/// Set each VPBasicBlock in \p NewSuccss as successor of this VPBlockBase.
|
|
/// This VPBlockBase must have no successors. This VPBlockBase is not added
|
|
/// as predecessor of any VPBasicBlock in \p NewSuccs.
|
|
void setSuccessors(ArrayRef<VPBlockBase *> NewSuccs) {
|
|
assert(Successors.empty() && "Block successors already set.");
|
|
for (auto *Succ : NewSuccs)
|
|
appendSuccessor(Succ);
|
|
}
|
|
|
|
/// Remove all the predecessor of this block.
|
|
void clearPredecessors() { Predecessors.clear(); }
|
|
|
|
/// Remove all the successors of this block.
|
|
void clearSuccessors() { Successors.clear(); }
|
|
|
|
/// Swap successors of the block. The block must have exactly 2 successors.
|
|
// TODO: This should be part of introducing conditional branch recipes rather
|
|
// than being independent.
|
|
void swapSuccessors() {
|
|
assert(Successors.size() == 2 && "must have 2 successors to swap");
|
|
std::swap(Successors[0], Successors[1]);
|
|
}
|
|
|
|
/// The method which generates the output IR that correspond to this
|
|
/// VPBlockBase, thereby "executing" the VPlan.
|
|
virtual void execute(VPTransformState *State) = 0;
|
|
|
|
/// Return the cost of the block.
|
|
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0;
|
|
|
|
/// Return true if it is legal to hoist instructions into this block.
|
|
bool isLegalToHoistInto() {
|
|
// There are currently no constraints that prevent an instruction to be
|
|
// hoisted into a VPBlockBase.
|
|
return true;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
void printAsOperand(raw_ostream &OS, bool PrintType = false) const {
|
|
OS << getName();
|
|
}
|
|
|
|
/// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
|
|
/// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
|
|
/// consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual blocks is consistent with the whole VPlan printing.
|
|
virtual void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const = 0;
|
|
|
|
/// Print plain-text dump of this VPlan to \p O.
|
|
void print(raw_ostream &O) const {
|
|
VPSlotTracker SlotTracker(getPlan());
|
|
print(O, "", SlotTracker);
|
|
}
|
|
|
|
/// Print the successors of this block to \p O, prefixing all lines with \p
|
|
/// Indent.
|
|
void printSuccessors(raw_ostream &O, const Twine &Indent) const;
|
|
|
|
/// Dump this VPBlockBase to dbgs().
|
|
LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
|
|
#endif
|
|
|
|
/// Clone the current block and it's recipes without updating the operands of
|
|
/// the cloned recipes, including all blocks in the single-entry single-exit
|
|
/// region for VPRegionBlocks.
|
|
virtual VPBlockBase *clone() = 0;
|
|
};
|
|
|
|
/// Struct to hold various analysis needed for cost computations.
|
|
struct VPCostContext {
|
|
const TargetTransformInfo &TTI;
|
|
const TargetLibraryInfo &TLI;
|
|
VPTypeAnalysis Types;
|
|
LLVMContext &LLVMCtx;
|
|
LoopVectorizationCostModel &CM;
|
|
SmallPtrSet<Instruction *, 8> SkipCostComputation;
|
|
TargetTransformInfo::TargetCostKind CostKind;
|
|
|
|
VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
|
|
Type *CanIVTy, LoopVectorizationCostModel &CM,
|
|
TargetTransformInfo::TargetCostKind CostKind)
|
|
: TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
|
|
CM(CM), CostKind(CostKind) {}
|
|
|
|
/// Return the cost for \p UI with \p VF using the legacy cost model as
|
|
/// fallback until computing the cost of all recipes migrates to VPlan.
|
|
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const;
|
|
|
|
/// Return true if the cost for \p UI shouldn't be computed, e.g. because it
|
|
/// has already been pre-computed.
|
|
bool skipCostComputation(Instruction *UI, bool IsVector) const;
|
|
|
|
/// Returns the OperandInfo for \p V, if it is a live-in.
|
|
TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const;
|
|
};
|
|
|
|
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
|
|
/// instructions. VPRecipeBase owns the VPValues it defines through VPDef
|
|
/// and is responsible for deleting its defined values. Single-value
|
|
/// recipes must inherit from VPSingleDef instead of inheriting from both
|
|
/// VPRecipeBase and VPValue separately.
|
|
class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
|
|
public VPDef,
|
|
public VPUser {
|
|
friend VPBasicBlock;
|
|
friend class VPBlockUtils;
|
|
|
|
/// Each VPRecipe belongs to a single VPBasicBlock.
|
|
VPBasicBlock *Parent = nullptr;
|
|
|
|
/// The debug location for the recipe.
|
|
DebugLoc DL;
|
|
|
|
public:
|
|
VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
DebugLoc DL = {})
|
|
: VPDef(SC), VPUser(Operands), DL(DL) {}
|
|
|
|
template <typename IterT>
|
|
VPRecipeBase(const unsigned char SC, iterator_range<IterT> Operands,
|
|
DebugLoc DL = {})
|
|
: VPDef(SC), VPUser(Operands), DL(DL) {}
|
|
virtual ~VPRecipeBase() = default;
|
|
|
|
/// Clone the current recipe.
|
|
virtual VPRecipeBase *clone() = 0;
|
|
|
|
/// \return the VPBasicBlock which this VPRecipe belongs to.
|
|
VPBasicBlock *getParent() { return Parent; }
|
|
const VPBasicBlock *getParent() const { return Parent; }
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPRecipe, thereby "executing" the VPlan.
|
|
virtual void execute(VPTransformState &State) = 0;
|
|
|
|
/// Return the cost of this recipe, taking into account if the cost
|
|
/// computation should be skipped and the ForceTargetInstructionCost flag.
|
|
/// Also takes care of printing the cost for debugging.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
|
|
|
|
/// Insert an unlinked recipe into a basic block immediately before
|
|
/// the specified recipe.
|
|
void insertBefore(VPRecipeBase *InsertPos);
|
|
/// Insert an unlinked recipe into \p BB immediately before the insertion
|
|
/// point \p IP;
|
|
void insertBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator IP);
|
|
|
|
/// Insert an unlinked Recipe into a basic block immediately after
|
|
/// the specified Recipe.
|
|
void insertAfter(VPRecipeBase *InsertPos);
|
|
|
|
/// Unlink this recipe from its current VPBasicBlock and insert it into
|
|
/// the VPBasicBlock that MovePos lives in, right after MovePos.
|
|
void moveAfter(VPRecipeBase *MovePos);
|
|
|
|
/// Unlink this recipe and insert into BB before I.
|
|
///
|
|
/// \pre I is a valid iterator into BB.
|
|
void moveBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator I);
|
|
|
|
/// This method unlinks 'this' from the containing basic block, but does not
|
|
/// delete it.
|
|
void removeFromParent();
|
|
|
|
/// This method unlinks 'this' from the containing basic block and deletes it.
|
|
///
|
|
/// \returns an iterator pointing to the element after the erased one
|
|
iplist<VPRecipeBase>::iterator eraseFromParent();
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPDef *D) {
|
|
// All VPDefs are also VPRecipeBases.
|
|
return true;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) { return true; }
|
|
|
|
/// Returns true if the recipe may have side-effects.
|
|
bool mayHaveSideEffects() const;
|
|
|
|
/// Returns true for PHI-like recipes.
|
|
bool isPhi() const {
|
|
return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
|
|
}
|
|
|
|
/// Returns true if the recipe may read from memory.
|
|
bool mayReadFromMemory() const;
|
|
|
|
/// Returns true if the recipe may write to memory.
|
|
bool mayWriteToMemory() const;
|
|
|
|
/// Returns true if the recipe may read from or write to memory.
|
|
bool mayReadOrWriteMemory() const {
|
|
return mayReadFromMemory() || mayWriteToMemory();
|
|
}
|
|
|
|
/// Returns the debug location of the recipe.
|
|
DebugLoc getDebugLoc() const { return DL; }
|
|
|
|
protected:
|
|
/// Compute the cost of this recipe either using a recipe's specialized
|
|
/// implementation or using the legacy cost model and the underlying
|
|
/// instructions.
|
|
virtual InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const;
|
|
};
|
|
|
|
// Helper macro to define common classof implementations for recipes.
|
|
#define VP_CLASSOF_IMPL(VPDefID) \
|
|
static inline bool classof(const VPDef *D) { \
|
|
return D->getVPDefID() == VPDefID; \
|
|
} \
|
|
static inline bool classof(const VPValue *V) { \
|
|
auto *R = V->getDefiningRecipe(); \
|
|
return R && R->getVPDefID() == VPDefID; \
|
|
} \
|
|
static inline bool classof(const VPUser *U) { \
|
|
auto *R = dyn_cast<VPRecipeBase>(U); \
|
|
return R && R->getVPDefID() == VPDefID; \
|
|
} \
|
|
static inline bool classof(const VPRecipeBase *R) { \
|
|
return R->getVPDefID() == VPDefID; \
|
|
} \
|
|
static inline bool classof(const VPSingleDefRecipe *R) { \
|
|
return R->getVPDefID() == VPDefID; \
|
|
}
|
|
|
|
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
|
|
/// more output IR that define a single result VPValue.
|
|
/// Note that VPRecipeBase must be inherited from before VPValue.
|
|
class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
|
|
public:
|
|
template <typename IterT>
|
|
VPSingleDefRecipe(const unsigned char SC, IterT Operands, DebugLoc DL = {})
|
|
: VPRecipeBase(SC, Operands, DL), VPValue(this) {}
|
|
|
|
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
DebugLoc DL = {})
|
|
: VPRecipeBase(SC, Operands, DL), VPValue(this) {}
|
|
|
|
template <typename IterT>
|
|
VPSingleDefRecipe(const unsigned char SC, IterT Operands, Value *UV,
|
|
DebugLoc DL = {})
|
|
: VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
switch (R->getVPDefID()) {
|
|
case VPRecipeBase::VPDerivedIVSC:
|
|
case VPRecipeBase::VPEVLBasedIVPHISC:
|
|
case VPRecipeBase::VPExpandSCEVSC:
|
|
case VPRecipeBase::VPInstructionSC:
|
|
case VPRecipeBase::VPReductionEVLSC:
|
|
case VPRecipeBase::VPReductionSC:
|
|
case VPRecipeBase::VPReplicateSC:
|
|
case VPRecipeBase::VPScalarIVStepsSC:
|
|
case VPRecipeBase::VPVectorPointerSC:
|
|
case VPRecipeBase::VPReverseVectorPointerSC:
|
|
case VPRecipeBase::VPWidenCallSC:
|
|
case VPRecipeBase::VPWidenCanonicalIVSC:
|
|
case VPRecipeBase::VPWidenCastSC:
|
|
case VPRecipeBase::VPWidenGEPSC:
|
|
case VPRecipeBase::VPWidenIntrinsicSC:
|
|
case VPRecipeBase::VPWidenSC:
|
|
case VPRecipeBase::VPWidenEVLSC:
|
|
case VPRecipeBase::VPWidenSelectSC:
|
|
case VPRecipeBase::VPBlendSC:
|
|
case VPRecipeBase::VPPredInstPHISC:
|
|
case VPRecipeBase::VPCanonicalIVPHISC:
|
|
case VPRecipeBase::VPActiveLaneMaskPHISC:
|
|
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
|
|
case VPRecipeBase::VPWidenPHISC:
|
|
case VPRecipeBase::VPWidenIntOrFpInductionSC:
|
|
case VPRecipeBase::VPWidenPointerInductionSC:
|
|
case VPRecipeBase::VPReductionPHISC:
|
|
case VPRecipeBase::VPScalarCastSC:
|
|
case VPRecipeBase::VPPartialReductionSC:
|
|
return true;
|
|
case VPRecipeBase::VPBranchOnMaskSC:
|
|
case VPRecipeBase::VPInterleaveSC:
|
|
case VPRecipeBase::VPIRInstructionSC:
|
|
case VPRecipeBase::VPWidenLoadEVLSC:
|
|
case VPRecipeBase::VPWidenLoadSC:
|
|
case VPRecipeBase::VPWidenStoreEVLSC:
|
|
case VPRecipeBase::VPWidenStoreSC:
|
|
case VPRecipeBase::VPHistogramSC:
|
|
// TODO: Widened stores don't define a value, but widened loads do. Split
|
|
// the recipes to be able to make widened loads VPSingleDefRecipes.
|
|
return false;
|
|
}
|
|
llvm_unreachable("Unhandled VPDefID");
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
virtual VPSingleDefRecipe *clone() override = 0;
|
|
|
|
/// Returns the underlying instruction.
|
|
Instruction *getUnderlyingInstr() {
|
|
return cast<Instruction>(getUnderlyingValue());
|
|
}
|
|
const Instruction *getUnderlyingInstr() const {
|
|
return cast<Instruction>(getUnderlyingValue());
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPSingleDefRecipe to dbgs() (for debugging).
|
|
LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
};
|
|
|
|
/// Class to record LLVM IR flag for a recipe along with it.
|
|
class VPRecipeWithIRFlags : public VPSingleDefRecipe {
|
|
enum class OperationType : unsigned char {
|
|
Cmp,
|
|
OverflowingBinOp,
|
|
DisjointOp,
|
|
PossiblyExactOp,
|
|
GEPOp,
|
|
FPMathOp,
|
|
NonNegOp,
|
|
Other
|
|
};
|
|
|
|
public:
|
|
struct WrapFlagsTy {
|
|
char HasNUW : 1;
|
|
char HasNSW : 1;
|
|
|
|
WrapFlagsTy(bool HasNUW, bool HasNSW) : HasNUW(HasNUW), HasNSW(HasNSW) {}
|
|
};
|
|
|
|
struct DisjointFlagsTy {
|
|
char IsDisjoint : 1;
|
|
DisjointFlagsTy(bool IsDisjoint) : IsDisjoint(IsDisjoint) {}
|
|
};
|
|
|
|
private:
|
|
struct ExactFlagsTy {
|
|
char IsExact : 1;
|
|
};
|
|
struct NonNegFlagsTy {
|
|
char NonNeg : 1;
|
|
};
|
|
struct FastMathFlagsTy {
|
|
char AllowReassoc : 1;
|
|
char NoNaNs : 1;
|
|
char NoInfs : 1;
|
|
char NoSignedZeros : 1;
|
|
char AllowReciprocal : 1;
|
|
char AllowContract : 1;
|
|
char ApproxFunc : 1;
|
|
|
|
FastMathFlagsTy(const FastMathFlags &FMF);
|
|
};
|
|
|
|
OperationType OpType;
|
|
|
|
union {
|
|
CmpInst::Predicate CmpPredicate;
|
|
WrapFlagsTy WrapFlags;
|
|
DisjointFlagsTy DisjointFlags;
|
|
ExactFlagsTy ExactFlags;
|
|
GEPNoWrapFlags GEPFlags;
|
|
NonNegFlagsTy NonNegFlags;
|
|
FastMathFlagsTy FMFs;
|
|
unsigned AllFlags;
|
|
};
|
|
|
|
protected:
|
|
void transferFlags(VPRecipeWithIRFlags &Other) {
|
|
OpType = Other.OpType;
|
|
AllFlags = Other.AllFlags;
|
|
}
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL) {
|
|
OpType = OperationType::Other;
|
|
AllFlags = 0;
|
|
}
|
|
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, Instruction &I)
|
|
: VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()) {
|
|
if (auto *Op = dyn_cast<CmpInst>(&I)) {
|
|
OpType = OperationType::Cmp;
|
|
CmpPredicate = Op->getPredicate();
|
|
} else if (auto *Op = dyn_cast<PossiblyDisjointInst>(&I)) {
|
|
OpType = OperationType::DisjointOp;
|
|
DisjointFlags.IsDisjoint = Op->isDisjoint();
|
|
} else if (auto *Op = dyn_cast<OverflowingBinaryOperator>(&I)) {
|
|
OpType = OperationType::OverflowingBinOp;
|
|
WrapFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
|
|
} else if (auto *Op = dyn_cast<PossiblyExactOperator>(&I)) {
|
|
OpType = OperationType::PossiblyExactOp;
|
|
ExactFlags.IsExact = Op->isExact();
|
|
} else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
|
|
OpType = OperationType::GEPOp;
|
|
GEPFlags = GEP->getNoWrapFlags();
|
|
} else if (auto *PNNI = dyn_cast<PossiblyNonNegInst>(&I)) {
|
|
OpType = OperationType::NonNegOp;
|
|
NonNegFlags.NonNeg = PNNI->hasNonNeg();
|
|
} else if (auto *Op = dyn_cast<FPMathOperator>(&I)) {
|
|
OpType = OperationType::FPMathOp;
|
|
FMFs = Op->getFastMathFlags();
|
|
} else {
|
|
OpType = OperationType::Other;
|
|
AllFlags = 0;
|
|
}
|
|
}
|
|
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
|
|
CmpInst::Predicate Pred, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::Cmp),
|
|
CmpPredicate(Pred) {}
|
|
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
|
|
WrapFlagsTy WrapFlags, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL),
|
|
OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
|
|
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
|
|
FastMathFlags FMFs, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::FPMathOp),
|
|
FMFs(FMFs) {}
|
|
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
|
|
DisjointFlagsTy DisjointFlags, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::DisjointOp),
|
|
DisjointFlags(DisjointFlags) {}
|
|
|
|
protected:
|
|
template <typename IterT>
|
|
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
|
|
GEPNoWrapFlags GEPFlags, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::GEPOp),
|
|
GEPFlags(GEPFlags) {}
|
|
|
|
public:
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenEVLSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Drop all poison-generating flags.
|
|
void dropPoisonGeneratingFlags() {
|
|
// NOTE: This needs to be kept in-sync with
|
|
// Instruction::dropPoisonGeneratingFlags.
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
WrapFlags.HasNUW = false;
|
|
WrapFlags.HasNSW = false;
|
|
break;
|
|
case OperationType::DisjointOp:
|
|
DisjointFlags.IsDisjoint = false;
|
|
break;
|
|
case OperationType::PossiblyExactOp:
|
|
ExactFlags.IsExact = false;
|
|
break;
|
|
case OperationType::GEPOp:
|
|
GEPFlags = GEPNoWrapFlags::none();
|
|
break;
|
|
case OperationType::FPMathOp:
|
|
FMFs.NoNaNs = false;
|
|
FMFs.NoInfs = false;
|
|
break;
|
|
case OperationType::NonNegOp:
|
|
NonNegFlags.NonNeg = false;
|
|
break;
|
|
case OperationType::Cmp:
|
|
case OperationType::Other:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/// Set the IR flags for \p I.
|
|
void setFlags(Instruction *I) const {
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
I->setHasNoUnsignedWrap(WrapFlags.HasNUW);
|
|
I->setHasNoSignedWrap(WrapFlags.HasNSW);
|
|
break;
|
|
case OperationType::DisjointOp:
|
|
cast<PossiblyDisjointInst>(I)->setIsDisjoint(DisjointFlags.IsDisjoint);
|
|
break;
|
|
case OperationType::PossiblyExactOp:
|
|
I->setIsExact(ExactFlags.IsExact);
|
|
break;
|
|
case OperationType::GEPOp:
|
|
cast<GetElementPtrInst>(I)->setNoWrapFlags(GEPFlags);
|
|
break;
|
|
case OperationType::FPMathOp:
|
|
I->setHasAllowReassoc(FMFs.AllowReassoc);
|
|
I->setHasNoNaNs(FMFs.NoNaNs);
|
|
I->setHasNoInfs(FMFs.NoInfs);
|
|
I->setHasNoSignedZeros(FMFs.NoSignedZeros);
|
|
I->setHasAllowReciprocal(FMFs.AllowReciprocal);
|
|
I->setHasAllowContract(FMFs.AllowContract);
|
|
I->setHasApproxFunc(FMFs.ApproxFunc);
|
|
break;
|
|
case OperationType::NonNegOp:
|
|
I->setNonNeg(NonNegFlags.NonNeg);
|
|
break;
|
|
case OperationType::Cmp:
|
|
case OperationType::Other:
|
|
break;
|
|
}
|
|
}
|
|
|
|
CmpInst::Predicate getPredicate() const {
|
|
assert(OpType == OperationType::Cmp &&
|
|
"recipe doesn't have a compare predicate");
|
|
return CmpPredicate;
|
|
}
|
|
|
|
GEPNoWrapFlags getGEPNoWrapFlags() const { return GEPFlags; }
|
|
|
|
/// Returns true if the recipe has fast-math flags.
|
|
bool hasFastMathFlags() const { return OpType == OperationType::FPMathOp; }
|
|
|
|
FastMathFlags getFastMathFlags() const;
|
|
|
|
bool hasNoUnsignedWrap() const {
|
|
assert(OpType == OperationType::OverflowingBinOp &&
|
|
"recipe doesn't have a NUW flag");
|
|
return WrapFlags.HasNUW;
|
|
}
|
|
|
|
bool hasNoSignedWrap() const {
|
|
assert(OpType == OperationType::OverflowingBinOp &&
|
|
"recipe doesn't have a NSW flag");
|
|
return WrapFlags.HasNSW;
|
|
}
|
|
|
|
bool isDisjoint() const {
|
|
assert(OpType == OperationType::DisjointOp &&
|
|
"recipe cannot have a disjoing flag");
|
|
return DisjointFlags.IsDisjoint;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
void printFlags(raw_ostream &O) const;
|
|
#endif
|
|
};
|
|
|
|
/// Helper to access the operand that contains the unroll part for this recipe
|
|
/// after unrolling.
|
|
template <unsigned PartOpIdx> class VPUnrollPartAccessor {
|
|
protected:
|
|
/// Return the VPValue operand containing the unroll part or null if there is
|
|
/// no such operand.
|
|
VPValue *getUnrollPartOperand(VPUser &U) const;
|
|
|
|
/// Return the unroll part.
|
|
unsigned getUnrollPart(VPUser &U) const;
|
|
};
|
|
|
|
/// This is a concrete Recipe that models a single VPlan-level instruction.
|
|
/// While as any Recipe it may generate a sequence of IR instructions when
|
|
/// executed, these instructions would always form a single-def expression as
|
|
/// the VPInstruction is also a single def-use vertex.
|
|
class VPInstruction : public VPRecipeWithIRFlags,
|
|
public VPUnrollPartAccessor<1> {
|
|
friend class VPlanSlp;
|
|
|
|
public:
|
|
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
|
|
enum {
|
|
FirstOrderRecurrenceSplice =
|
|
Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
|
|
// values of a first-order recurrence.
|
|
Not,
|
|
SLPLoad,
|
|
SLPStore,
|
|
ActiveLaneMask,
|
|
ExplicitVectorLength,
|
|
/// Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
|
|
/// The first operand is the incoming value from the predecessor in VPlan,
|
|
/// the second operand is the incoming value for all other predecessors
|
|
/// (which are currently not modeled in VPlan).
|
|
ResumePhi,
|
|
CalculateTripCountMinusVF,
|
|
// Increment the canonical IV separately for each unrolled part.
|
|
CanonicalIVIncrementForPart,
|
|
BranchOnCount,
|
|
BranchOnCond,
|
|
ComputeReductionResult,
|
|
// Takes the VPValue to extract from as first operand and the lane or part
|
|
// to extract as second operand, counting from the end starting with 1 for
|
|
// last. The second operand must be a positive constant and <= VF.
|
|
ExtractFromEnd,
|
|
LogicalAnd, // Non-poison propagating logical And.
|
|
// Add an offset in bytes (second operand) to a base pointer (first
|
|
// operand). Only generates scalar values (either for the first lane only or
|
|
// for all lanes, depending on its uses).
|
|
PtrAdd,
|
|
// Returns a scalar boolean value, which is true if any lane of its (only
|
|
// boolean) vector operand is true.
|
|
AnyOf,
|
|
// Extracts the first active lane of a vector, where the first operand is
|
|
// the predicate, and the second operand is the vector to extract.
|
|
ExtractFirstActive,
|
|
};
|
|
|
|
private:
|
|
typedef unsigned char OpcodeTy;
|
|
OpcodeTy Opcode;
|
|
|
|
/// An optional name that can be used for the generated IR instruction.
|
|
const std::string Name;
|
|
|
|
/// Returns true if this VPInstruction generates scalar values for all lanes.
|
|
/// Most VPInstructions generate a single value per part, either vector or
|
|
/// scalar. VPReplicateRecipe takes care of generating multiple (scalar)
|
|
/// values per all lanes, stemming from an original ingredient. This method
|
|
/// identifies the (rare) cases of VPInstructions that do so as well, w/o an
|
|
/// underlying ingredient.
|
|
bool doesGeneratePerAllLanes() const;
|
|
|
|
/// Returns true if we can generate a scalar for the first lane only if
|
|
/// needed.
|
|
bool canGenerateScalarForFirstLane() const;
|
|
|
|
/// Utility methods serving execute(): generates a single vector instance of
|
|
/// the modeled instruction. \returns the generated value. . In some cases an
|
|
/// existing value is returned rather than a generated one.
|
|
Value *generate(VPTransformState &State);
|
|
|
|
/// Utility methods serving execute(): generates a scalar single instance of
|
|
/// the modeled instruction for a given lane. \returns the scalar generated
|
|
/// value for lane \p Lane.
|
|
Value *generatePerLane(VPTransformState &State, const VPLane &Lane);
|
|
|
|
#if !defined(NDEBUG)
|
|
/// Return true if the VPInstruction is a floating point math operation, i.e.
|
|
/// has fast-math flags.
|
|
bool isFPMathOp() const;
|
|
#endif
|
|
|
|
public:
|
|
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
|
|
const Twine &Name = "")
|
|
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
|
|
Opcode(Opcode), Name(Name.str()) {}
|
|
|
|
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
|
|
DebugLoc DL = {}, const Twine &Name = "")
|
|
: VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
|
|
|
|
VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
|
|
VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
|
|
|
|
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
|
|
WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
|
|
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
|
|
Opcode(Opcode), Name(Name.str()) {}
|
|
|
|
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
|
|
DisjointFlagsTy DisjointFlag, DebugLoc DL = {},
|
|
const Twine &Name = "")
|
|
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL),
|
|
Opcode(Opcode), Name(Name.str()) {
|
|
assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint");
|
|
}
|
|
|
|
VPInstruction(VPValue *Ptr, VPValue *Offset, GEPNoWrapFlags Flags,
|
|
DebugLoc DL = {}, const Twine &Name = "")
|
|
: VPRecipeWithIRFlags(VPDef::VPInstructionSC,
|
|
ArrayRef<VPValue *>({Ptr, Offset}), Flags, DL),
|
|
Opcode(VPInstruction::PtrAdd), Name(Name.str()) {}
|
|
|
|
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
|
|
FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
|
|
|
|
VPInstruction *clone() override {
|
|
SmallVector<VPValue *, 2> Operands(operands());
|
|
auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
|
|
New->transferFlags(*this);
|
|
return New;
|
|
}
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
/// Generate the instruction.
|
|
/// TODO: We currently execute only per-part unless a specific instance is
|
|
/// provided.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPInstruction.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the VPInstruction to \p O.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
|
|
/// Print the VPInstruction to dbgs() (for debugging).
|
|
LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
|
|
bool hasResult() const {
|
|
// CallInst may or may not have a result, depending on the called function.
|
|
// Conservatively return calls have results for now.
|
|
switch (getOpcode()) {
|
|
case Instruction::Ret:
|
|
case Instruction::Br:
|
|
case Instruction::Store:
|
|
case Instruction::Switch:
|
|
case Instruction::IndirectBr:
|
|
case Instruction::Resume:
|
|
case Instruction::CatchRet:
|
|
case Instruction::Unreachable:
|
|
case Instruction::Fence:
|
|
case Instruction::AtomicRMW:
|
|
case VPInstruction::BranchOnCond:
|
|
case VPInstruction::BranchOnCount:
|
|
return false;
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// Returns true if the underlying opcode may read from or write to memory.
|
|
bool opcodeMayReadOrWriteFromMemory() const;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override;
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool onlyFirstPartUsed(const VPValue *Op) const override;
|
|
|
|
/// Returns true if this VPInstruction produces a scalar value from a vector,
|
|
/// e.g. by performing a reduction or extracting a lane.
|
|
bool isVectorToScalar() const;
|
|
|
|
/// Returns true if this VPInstruction's operands are single scalars and the
|
|
/// result is also a single scalar.
|
|
bool isSingleScalar() const;
|
|
|
|
/// Returns the symbolic name assigned to the VPInstruction.
|
|
StringRef getName() const { return Name; }
|
|
};
|
|
|
|
/// A recipe to wrap on original IR instruction not to be modified during
|
|
/// execution, execept for PHIs. For PHIs, a single VPValue operand is allowed,
|
|
/// and it is used to add a new incoming value for the single predecessor VPBB.
|
|
/// Expect PHIs, VPIRInstructions cannot have any operands.
|
|
class VPIRInstruction : public VPRecipeBase {
|
|
Instruction &I;
|
|
|
|
public:
|
|
VPIRInstruction(Instruction &I)
|
|
: VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
|
|
|
|
~VPIRInstruction() override = default;
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPIRInstructionSC)
|
|
|
|
VPIRInstruction *clone() override {
|
|
auto *R = new VPIRInstruction(I);
|
|
for (auto *Op : operands())
|
|
R->addOperand(Op);
|
|
return R;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPIRInstruction.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
Instruction &getInstruction() const { return I; }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
bool onlyFirstPartUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Update the recipes single operand to the last lane of the operand using \p
|
|
/// Builder. Must only be used for single operand VPIRInstructions wrapping a
|
|
/// PHINode.
|
|
void extractLastLaneOfOperand(VPBuilder &Builder);
|
|
};
|
|
|
|
/// VPWidenRecipe is a recipe for producing a widened instruction using the
|
|
/// opcode and operands of the recipe. This recipe covers most of the
|
|
/// traditional vectorization cases where each recipe transforms into a
|
|
/// vectorized version of itself.
|
|
class VPWidenRecipe : public VPRecipeWithIRFlags {
|
|
unsigned Opcode;
|
|
|
|
protected:
|
|
template <typename IterT>
|
|
VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
|
|
iterator_range<IterT> Operands)
|
|
: VPRecipeWithIRFlags(VPDefOpcode, Operands, I), Opcode(I.getOpcode()) {}
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
|
|
: VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
|
|
|
|
~VPWidenRecipe() override = default;
|
|
|
|
VPWidenRecipe *clone() override {
|
|
auto *R = new VPWidenRecipe(*getUnderlyingInstr(), operands());
|
|
R->transferFlags(*this);
|
|
return R;
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPDefID() == VPRecipeBase::VPWidenSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Produce a widened instruction using the opcode and operands of the recipe,
|
|
/// processing State.VF elements.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening operations with vector-predication intrinsics with
|
|
/// explicit vector length (EVL).
|
|
class VPWidenEVLRecipe : public VPWidenRecipe {
|
|
using VPRecipeWithIRFlags::transferFlags;
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPWidenEVLRecipe(Instruction &I, iterator_range<IterT> Operands, VPValue &EVL)
|
|
: VPWidenRecipe(VPDef::VPWidenEVLSC, I, Operands) {
|
|
addOperand(&EVL);
|
|
}
|
|
VPWidenEVLRecipe(VPWidenRecipe &W, VPValue &EVL)
|
|
: VPWidenEVLRecipe(*W.getUnderlyingInstr(), W.operands(), EVL) {
|
|
transferFlags(W);
|
|
}
|
|
|
|
~VPWidenEVLRecipe() override = default;
|
|
|
|
VPWidenRecipe *clone() override final {
|
|
llvm_unreachable("VPWidenEVLRecipe cannot be cloned");
|
|
return nullptr;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenEVLSC);
|
|
|
|
VPValue *getEVL() { return getOperand(getNumOperands() - 1); }
|
|
const VPValue *getEVL() const { return getOperand(getNumOperands() - 1); }
|
|
|
|
/// Produce a vp-intrinsic using the opcode and operands of the recipe,
|
|
/// processing EVL elements.
|
|
void execute(VPTransformState &State) override final;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// EVL in that recipe is always the last operand, thus any use before means
|
|
// the VPValue should be vectorized.
|
|
return getEVL() == Op;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override final;
|
|
#endif
|
|
};
|
|
|
|
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
|
|
class VPWidenCastRecipe : public VPRecipeWithIRFlags {
|
|
/// Cast instruction opcode.
|
|
Instruction::CastOps Opcode;
|
|
|
|
/// Result type for the cast.
|
|
Type *ResultTy;
|
|
|
|
public:
|
|
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
|
|
CastInst &UI)
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
|
|
ResultTy(ResultTy) {
|
|
assert(UI.getOpcode() == Opcode &&
|
|
"opcode of underlying cast doesn't match");
|
|
}
|
|
|
|
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
|
|
ResultTy(ResultTy) {}
|
|
|
|
~VPWidenCastRecipe() override = default;
|
|
|
|
VPWidenCastRecipe *clone() override {
|
|
if (auto *UV = getUnderlyingValue())
|
|
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
|
|
*cast<CastInst>(UV));
|
|
|
|
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
|
|
|
|
/// Produce widened copies of the cast.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCastRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
Instruction::CastOps getOpcode() const { return Opcode; }
|
|
|
|
/// Returns the result type of the cast.
|
|
Type *getResultType() const { return ResultTy; }
|
|
};
|
|
|
|
/// VPScalarCastRecipe is a recipe to create scalar cast instructions.
|
|
class VPScalarCastRecipe : public VPSingleDefRecipe {
|
|
Instruction::CastOps Opcode;
|
|
|
|
Type *ResultTy;
|
|
|
|
Value *generate(VPTransformState &State);
|
|
|
|
public:
|
|
VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
|
|
DebugLoc DL)
|
|
: VPSingleDefRecipe(VPDef::VPScalarCastSC, {Op}, DL), Opcode(Opcode),
|
|
ResultTy(ResultTy) {}
|
|
|
|
~VPScalarCastRecipe() override = default;
|
|
|
|
VPScalarCastRecipe *clone() override {
|
|
return new VPScalarCastRecipe(Opcode, getOperand(0), ResultTy,
|
|
getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPScalarCastSC)
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPScalarCastRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns the result type of the cast.
|
|
Type *getResultType() const { return ResultTy; }
|
|
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
// At the moment, only uniform codegen is implemented.
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// A recipe for widening vector intrinsics.
|
|
class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
|
|
/// ID of the vector intrinsic to widen.
|
|
Intrinsic::ID VectorIntrinsicID;
|
|
|
|
/// Scalar return type of the intrinsic.
|
|
Type *ResultTy;
|
|
|
|
/// True if the intrinsic may read from memory.
|
|
bool MayReadFromMemory;
|
|
|
|
/// True if the intrinsic may read write to memory.
|
|
bool MayWriteToMemory;
|
|
|
|
/// True if the intrinsic may have side-effects.
|
|
bool MayHaveSideEffects;
|
|
|
|
public:
|
|
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
|
|
ArrayRef<VPValue *> CallArguments, Type *Ty,
|
|
DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
|
|
VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
|
|
MayReadFromMemory(CI.mayReadFromMemory()),
|
|
MayWriteToMemory(CI.mayWriteToMemory()),
|
|
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
|
|
|
|
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
|
|
ArrayRef<VPValue *> CallArguments, Type *Ty,
|
|
DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
|
|
VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
|
|
LLVMContext &Ctx = Ty->getContext();
|
|
AttributeList Attrs = Intrinsic::getAttributes(Ctx, VectorIntrinsicID);
|
|
MemoryEffects ME = Attrs.getMemoryEffects();
|
|
MayReadFromMemory = ME.onlyWritesMemory();
|
|
MayWriteToMemory = ME.onlyReadsMemory();
|
|
MayHaveSideEffects = MayWriteToMemory ||
|
|
!Attrs.hasFnAttr(Attribute::NoUnwind) ||
|
|
!Attrs.hasFnAttr(Attribute::WillReturn);
|
|
}
|
|
|
|
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
|
|
std::initializer_list<VPValue *> CallArguments,
|
|
Type *Ty, DebugLoc DL = {})
|
|
: VPWidenIntrinsicRecipe(VectorIntrinsicID,
|
|
ArrayRef<VPValue *>(CallArguments), Ty, DL) {}
|
|
|
|
~VPWidenIntrinsicRecipe() override = default;
|
|
|
|
VPWidenIntrinsicRecipe *clone() override {
|
|
return new VPWidenIntrinsicRecipe(*cast<CallInst>(getUnderlyingValue()),
|
|
VectorIntrinsicID, {op_begin(), op_end()},
|
|
ResultTy, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
|
|
|
|
/// Produce a widened version of the vector intrinsic.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this vector intrinsic.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Return the ID of the intrinsic.
|
|
Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
|
|
|
|
/// Return the scalar return type of the intrinsic.
|
|
Type *getResultType() const { return ResultTy; }
|
|
|
|
/// Return to name of the intrinsic as string.
|
|
StringRef getIntrinsicName() const;
|
|
|
|
/// Returns true if the intrinsic may read from memory.
|
|
bool mayReadFromMemory() const { return MayReadFromMemory; }
|
|
|
|
/// Returns true if the intrinsic may write to memory.
|
|
bool mayWriteToMemory() const { return MayWriteToMemory; }
|
|
|
|
/// Returns true if the intrinsic may have side-effects.
|
|
bool mayHaveSideEffects() const { return MayHaveSideEffects; }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override;
|
|
};
|
|
|
|
/// A recipe for widening Call instructions using library calls.
|
|
class VPWidenCallRecipe : public VPRecipeWithIRFlags {
|
|
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
|
|
/// between a given VF and the chosen vectorized variant, so there will be a
|
|
/// different VPlan for each VF with a valid variant.
|
|
Function *Variant;
|
|
|
|
public:
|
|
VPWidenCallRecipe(Value *UV, Function *Variant,
|
|
ArrayRef<VPValue *> CallArguments, DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
|
|
*cast<Instruction>(UV)),
|
|
Variant(Variant) {
|
|
assert(
|
|
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
|
|
"last operand must be the called function");
|
|
}
|
|
|
|
~VPWidenCallRecipe() override = default;
|
|
|
|
VPWidenCallRecipe *clone() override {
|
|
return new VPWidenCallRecipe(getUnderlyingValue(), Variant,
|
|
{op_begin(), op_end()}, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
|
|
|
|
/// Produce a widened version of the call instruction.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCallRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
Function *getCalledScalarFunction() const {
|
|
return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
|
|
}
|
|
|
|
operand_range arg_operands() {
|
|
return make_range(op_begin(), op_begin() + getNumOperands() - 1);
|
|
}
|
|
const_operand_range arg_operands() const {
|
|
return make_range(op_begin(), op_begin() + getNumOperands() - 1);
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe representing a sequence of load -> update -> store as part of
|
|
/// a histogram operation. This means there may be aliasing between vector
|
|
/// lanes, which is handled by the llvm.experimental.vector.histogram family
|
|
/// of intrinsics. The only update operations currently supported are
|
|
/// 'add' and 'sub' where the other term is loop-invariant.
|
|
class VPHistogramRecipe : public VPRecipeBase {
|
|
/// Opcode of the update operation, currently either add or sub.
|
|
unsigned Opcode;
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPHistogramRecipe(unsigned Opcode, iterator_range<IterT> Operands,
|
|
DebugLoc DL = {})
|
|
: VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Opcode(Opcode) {}
|
|
|
|
~VPHistogramRecipe() override = default;
|
|
|
|
VPHistogramRecipe *clone() override {
|
|
return new VPHistogramRecipe(Opcode, operands(), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
|
|
|
|
/// Produce a vectorized histogram operation.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPHistogramRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
/// Return the mask operand if one was provided, or a null pointer if all
|
|
/// lanes should be executed unconditionally.
|
|
VPValue *getMask() const {
|
|
return getNumOperands() == 3 ? getOperand(2) : nullptr;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening select instructions.
|
|
struct VPWidenSelectRecipe : public VPRecipeWithIRFlags {
|
|
template <typename IterT>
|
|
VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands)
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I) {}
|
|
|
|
~VPWidenSelectRecipe() override = default;
|
|
|
|
VPWidenSelectRecipe *clone() override {
|
|
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
|
|
operands());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
|
|
|
|
/// Produce a widened version of the select instruction.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenSelectRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
VPValue *getCond() const {
|
|
return getOperand(0);
|
|
}
|
|
|
|
bool isInvariantCond() const {
|
|
return getCond()->isDefinedOutsideLoopRegions();
|
|
}
|
|
};
|
|
|
|
/// A recipe for handling GEP instructions.
|
|
class VPWidenGEPRecipe : public VPRecipeWithIRFlags {
|
|
bool isPointerLoopInvariant() const {
|
|
return getOperand(0)->isDefinedOutsideLoopRegions();
|
|
}
|
|
|
|
bool isIndexLoopInvariant(unsigned I) const {
|
|
return getOperand(I + 1)->isDefinedOutsideLoopRegions();
|
|
}
|
|
|
|
bool areAllOperandsInvariant() const {
|
|
return all_of(operands(), [](VPValue *Op) {
|
|
return Op->isDefinedOutsideLoopRegions();
|
|
});
|
|
}
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range<IterT> Operands)
|
|
: VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP) {}
|
|
|
|
~VPWidenGEPRecipe() override = default;
|
|
|
|
VPWidenGEPRecipe *clone() override {
|
|
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
|
|
operands());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
|
|
|
|
/// Generate the gep nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenGEPRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to compute the pointers for widened memory accesses of IndexTy
|
|
/// in reverse order.
|
|
class VPReverseVectorPointerRecipe : public VPRecipeWithIRFlags,
|
|
public VPUnrollPartAccessor<2> {
|
|
Type *IndexedTy;
|
|
|
|
public:
|
|
VPReverseVectorPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy,
|
|
GEPNoWrapFlags GEPFlags, DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPDef::VPReverseVectorPointerSC,
|
|
ArrayRef<VPValue *>({Ptr, VF}), GEPFlags, DL),
|
|
IndexedTy(IndexedTy) {}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPReverseVectorPointerSC)
|
|
|
|
VPValue *getVFValue() { return getOperand(1); }
|
|
const VPValue *getVFValue() const { return getOperand(1); }
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Return the cost of this VPVectorPointerRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool onlyFirstPartUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
assert(getNumOperands() <= 2 && "must have at most two operands");
|
|
return true;
|
|
}
|
|
|
|
VPReverseVectorPointerRecipe *clone() override {
|
|
return new VPReverseVectorPointerRecipe(getOperand(0), getVFValue(),
|
|
IndexedTy, getGEPNoWrapFlags(),
|
|
getDebugLoc());
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to compute the pointers for widened memory accesses of IndexTy.
|
|
class VPVectorPointerRecipe : public VPRecipeWithIRFlags,
|
|
public VPUnrollPartAccessor<1> {
|
|
Type *IndexedTy;
|
|
|
|
public:
|
|
VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, GEPNoWrapFlags GEPFlags,
|
|
DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
|
|
GEPFlags, DL),
|
|
IndexedTy(IndexedTy) {}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPVectorPointerSC)
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool onlyFirstPartUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
assert(getNumOperands() <= 2 && "must have at most two operands");
|
|
return true;
|
|
}
|
|
|
|
VPVectorPointerRecipe *clone() override {
|
|
return new VPVectorPointerRecipe(getOperand(0), IndexedTy,
|
|
getGEPNoWrapFlags(), getDebugLoc());
|
|
}
|
|
|
|
/// Return the cost of this VPHeaderPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A pure virtual base class for all recipes modeling header phis, including
|
|
/// phis for first order recurrences, pointer inductions and reductions. The
|
|
/// start value is the first operand of the recipe and the incoming value from
|
|
/// the backedge is the second operand.
|
|
///
|
|
/// Inductions are modeled using the following sub-classes:
|
|
/// * VPCanonicalIVPHIRecipe: Canonical scalar induction of the vector loop,
|
|
/// starting at a specified value (zero for the main vector loop, the resume
|
|
/// value for the epilogue vector loop) and stepping by 1. The induction
|
|
/// controls exiting of the vector loop by comparing against the vector trip
|
|
/// count. Produces a single scalar PHI for the induction value per
|
|
/// iteration.
|
|
/// * VPWidenIntOrFpInductionRecipe: Generates vector values for integer and
|
|
/// floating point inductions with arbitrary start and step values. Produces
|
|
/// a vector PHI per-part.
|
|
/// * VPDerivedIVRecipe: Converts the canonical IV value to the corresponding
|
|
/// value of an IV with different start and step values. Produces a single
|
|
/// scalar value per iteration
|
|
/// * VPScalarIVStepsRecipe: Generates scalar values per-lane based on a
|
|
/// canonical or derived induction.
|
|
/// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a
|
|
/// pointer induction. Produces either a vector PHI per-part or scalar values
|
|
/// per-lane based on the canonical induction.
|
|
class VPHeaderPHIRecipe : public VPSingleDefRecipe {
|
|
protected:
|
|
VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
|
|
VPValue *Start = nullptr, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>(), UnderlyingInstr, DL) {
|
|
if (Start)
|
|
addOperand(Start);
|
|
}
|
|
|
|
public:
|
|
~VPHeaderPHIRecipe() override = default;
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPRecipeBase *B) {
|
|
return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
|
|
B->getVPDefID() <= VPDef::VPLastHeaderPHISC;
|
|
}
|
|
static inline bool classof(const VPValue *V) {
|
|
auto *B = V->getDefiningRecipe();
|
|
return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
|
|
B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC;
|
|
}
|
|
|
|
/// Generate the phi nodes.
|
|
void execute(VPTransformState &State) override = 0;
|
|
|
|
/// Return the cost of this header phi recipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override = 0;
|
|
#endif
|
|
|
|
/// Returns the start value of the phi, if one is set.
|
|
VPValue *getStartValue() {
|
|
return getNumOperands() == 0 ? nullptr : getOperand(0);
|
|
}
|
|
VPValue *getStartValue() const {
|
|
return getNumOperands() == 0 ? nullptr : getOperand(0);
|
|
}
|
|
|
|
/// Update the start value of the recipe.
|
|
void setStartValue(VPValue *V) { setOperand(0, V); }
|
|
|
|
/// Returns the incoming value from the loop backedge.
|
|
virtual VPValue *getBackedgeValue() {
|
|
return getOperand(1);
|
|
}
|
|
|
|
/// Returns the backedge value as a recipe. The backedge value is guaranteed
|
|
/// to be a recipe.
|
|
virtual VPRecipeBase &getBackedgeRecipe() {
|
|
return *getBackedgeValue()->getDefiningRecipe();
|
|
}
|
|
};
|
|
|
|
/// Base class for widened induction (VPWidenIntOrFpInductionRecipe and
|
|
/// VPWidenPointerInductionRecipe), providing shared functionality, including
|
|
/// retrieving the step value, induction descriptor and original phi node.
|
|
class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
|
|
const InductionDescriptor &IndDesc;
|
|
|
|
public:
|
|
VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
|
|
VPValue *Step, const InductionDescriptor &IndDesc,
|
|
DebugLoc DL)
|
|
: VPHeaderPHIRecipe(Kind, IV, Start, DL), IndDesc(IndDesc) {
|
|
addOperand(Step);
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC ||
|
|
R->getVPDefID() == VPDef::VPWidenPointerInductionSC;
|
|
}
|
|
|
|
static inline bool classof(const VPValue *V) {
|
|
auto *R = V->getDefiningRecipe();
|
|
return R && classof(R);
|
|
}
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *R) {
|
|
return classof(static_cast<const VPRecipeBase *>(R));
|
|
}
|
|
|
|
virtual void execute(VPTransformState &State) override = 0;
|
|
|
|
/// Returns the step value of the induction.
|
|
VPValue *getStepValue() { return getOperand(1); }
|
|
const VPValue *getStepValue() const { return getOperand(1); }
|
|
|
|
PHINode *getPHINode() const { return cast<PHINode>(getUnderlyingValue()); }
|
|
|
|
/// Returns the induction descriptor for the recipe.
|
|
const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
|
|
|
|
VPValue *getBackedgeValue() override {
|
|
// TODO: All operands of base recipe must exist and be at same index in
|
|
// derived recipe.
|
|
llvm_unreachable(
|
|
"VPWidenIntOrFpInductionRecipe generates its own backedge value");
|
|
}
|
|
|
|
VPRecipeBase &getBackedgeRecipe() override {
|
|
// TODO: All operands of base recipe must exist and be at same index in
|
|
// derived recipe.
|
|
llvm_unreachable(
|
|
"VPWidenIntOrFpInductionRecipe generates its own backedge value");
|
|
}
|
|
};
|
|
|
|
/// A recipe for handling phi nodes of integer and floating-point inductions,
|
|
/// producing their vector values.
|
|
class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe {
|
|
TruncInst *Trunc;
|
|
|
|
public:
|
|
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
|
|
VPValue *VF, const InductionDescriptor &IndDesc,
|
|
DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
|
|
Step, IndDesc, DL),
|
|
Trunc(nullptr) {
|
|
addOperand(VF);
|
|
}
|
|
|
|
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
|
|
VPValue *VF, const InductionDescriptor &IndDesc,
|
|
TruncInst *Trunc, DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
|
|
Step, IndDesc, DL),
|
|
Trunc(Trunc) {
|
|
addOperand(VF);
|
|
}
|
|
|
|
~VPWidenIntOrFpInductionRecipe() override = default;
|
|
|
|
VPWidenIntOrFpInductionRecipe *clone() override {
|
|
return new VPWidenIntOrFpInductionRecipe(
|
|
getPHINode(), getStartValue(), getStepValue(), getVFValue(),
|
|
getInductionDescriptor(), Trunc, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)
|
|
|
|
/// Generate the vectorized and scalarized versions of the phi node as
|
|
/// needed by their users.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
VPValue *getVFValue() { return getOperand(2); }
|
|
const VPValue *getVFValue() const { return getOperand(2); }
|
|
|
|
VPValue *getSplatVFValue() {
|
|
// If the recipe has been unrolled (4 operands), return the VPValue for the
|
|
// induction increment.
|
|
return getNumOperands() == 5 ? getOperand(3) : nullptr;
|
|
}
|
|
|
|
/// Returns the first defined value as TruncInst, if it is one or nullptr
|
|
/// otherwise.
|
|
TruncInst *getTruncInst() { return Trunc; }
|
|
const TruncInst *getTruncInst() const { return Trunc; }
|
|
|
|
/// Returns true if the induction is canonical, i.e. starting at 0 and
|
|
/// incremented by UF * VF (= the original IV is incremented by 1) and has the
|
|
/// same type as the canonical induction.
|
|
bool isCanonical() const;
|
|
|
|
/// Returns the scalar type of the induction.
|
|
Type *getScalarType() const {
|
|
return Trunc ? Trunc->getType() : getPHINode()->getType();
|
|
}
|
|
|
|
/// Returns the VPValue representing the value of this induction at
|
|
/// the last unrolled part, if it exists. Returns itself if unrolling did not
|
|
/// take place.
|
|
VPValue *getLastUnrolledPartOperand() {
|
|
return getNumOperands() == 5 ? getOperand(4) : this;
|
|
}
|
|
};
|
|
|
|
class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
|
|
public VPUnrollPartAccessor<3> {
|
|
bool IsScalarAfterVectorization;
|
|
|
|
public:
|
|
/// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
|
|
/// Start.
|
|
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
|
|
const InductionDescriptor &IndDesc,
|
|
bool IsScalarAfterVectorization, DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start,
|
|
Step, IndDesc, DL),
|
|
IsScalarAfterVectorization(IsScalarAfterVectorization) {}
|
|
|
|
~VPWidenPointerInductionRecipe() override = default;
|
|
|
|
VPWidenPointerInductionRecipe *clone() override {
|
|
return new VPWidenPointerInductionRecipe(
|
|
cast<PHINode>(getUnderlyingInstr()), getOperand(0), getOperand(1),
|
|
getInductionDescriptor(), IsScalarAfterVectorization, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)
|
|
|
|
/// Generate vector values for the pointer induction.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Returns true if only scalar values will be generated.
|
|
bool onlyScalarsGenerated(bool IsScalable);
|
|
|
|
/// Returns the VPValue representing the value of this induction at
|
|
/// the first unrolled part, if it exists. Returns itself if unrolling did not
|
|
/// take place.
|
|
VPValue *getFirstUnrolledPartOperand() {
|
|
return getUnrollPart(*this) == 0 ? this : getOperand(2);
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Recipe to generate a scalar PHI. Used to generate code for recipes that
|
|
/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
|
|
/// VPEVLBasedIVPHIRecipe.
|
|
class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
|
|
std::string Name;
|
|
|
|
public:
|
|
VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
|
|
StringRef Name)
|
|
: VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
|
|
Name(Name.str()) {
|
|
addOperand(BackedgeValue);
|
|
}
|
|
|
|
~VPScalarPHIRecipe() override = default;
|
|
|
|
VPScalarPHIRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for handling phis that are widened in the vector loop.
|
|
/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
|
|
/// managed in the recipe directly.
|
|
class VPWidenPHIRecipe : public VPSingleDefRecipe {
|
|
/// List of incoming blocks. Only used in the VPlan native path.
|
|
SmallVector<VPBasicBlock *, 2> IncomingBlocks;
|
|
|
|
public:
|
|
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
|
|
/// debug location \p DL.
|
|
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr, DebugLoc DL = {})
|
|
: VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef<VPValue *>(), Phi, DL) {
|
|
if (Start)
|
|
addOperand(Start);
|
|
}
|
|
|
|
VPWidenPHIRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
~VPWidenPHIRecipe() override = default;
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenPHISC)
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi.
|
|
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
|
|
addOperand(IncomingV);
|
|
IncomingBlocks.push_back(IncomingBlock);
|
|
}
|
|
|
|
/// Returns the \p I th incoming VPBasicBlock.
|
|
VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
|
|
|
|
/// Returns the \p I th incoming VPValue.
|
|
VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
|
|
};
|
|
|
|
/// A recipe for handling first-order recurrence phis. The start value is the
|
|
/// first operand of the recipe and the incoming value from the backedge is the
|
|
/// second operand.
|
|
struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
|
|
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
|
|
: VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC)
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *R) {
|
|
return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC;
|
|
}
|
|
|
|
VPFirstOrderRecurrencePHIRecipe *clone() override {
|
|
return new VPFirstOrderRecurrencePHIRecipe(
|
|
cast<PHINode>(getUnderlyingInstr()), *getOperand(0));
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this first-order recurrence phi recipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for handling reduction phis. The start value is the first operand
|
|
/// of the recipe and the incoming value from the backedge is the second
|
|
/// operand.
|
|
class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
|
|
public VPUnrollPartAccessor<2> {
|
|
/// Descriptor for the reduction.
|
|
const RecurrenceDescriptor &RdxDesc;
|
|
|
|
/// The phi is part of an in-loop reduction.
|
|
bool IsInLoop;
|
|
|
|
/// The phi is part of an ordered reduction. Requires IsInLoop to be true.
|
|
bool IsOrdered;
|
|
|
|
/// When expanding the reduction PHI, the plan's VF element count is divided
|
|
/// by this factor to form the reduction phi's VF.
|
|
unsigned VFScaleFactor = 1;
|
|
|
|
public:
|
|
/// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
|
|
/// RdxDesc.
|
|
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
|
|
VPValue &Start, bool IsInLoop = false,
|
|
bool IsOrdered = false, unsigned VFScaleFactor = 1)
|
|
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
|
|
RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
|
|
VFScaleFactor(VFScaleFactor) {
|
|
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
|
|
}
|
|
|
|
~VPReductionPHIRecipe() override = default;
|
|
|
|
VPReductionPHIRecipe *clone() override {
|
|
auto *R = new VPReductionPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
|
|
RdxDesc, *getOperand(0), IsInLoop,
|
|
IsOrdered, VFScaleFactor);
|
|
R->addOperand(getBackedgeValue());
|
|
return R;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPReductionPHISC)
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *R) {
|
|
return R->getVPDefID() == VPDef::VPReductionPHISC;
|
|
}
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const RecurrenceDescriptor &getRecurrenceDescriptor() const {
|
|
return RdxDesc;
|
|
}
|
|
|
|
/// Returns true, if the phi is part of an ordered reduction.
|
|
bool isOrdered() const { return IsOrdered; }
|
|
|
|
/// Returns true, if the phi is part of an in-loop reduction.
|
|
bool isInLoop() const { return IsInLoop; }
|
|
};
|
|
|
|
/// A recipe for forming partial reductions. In the loop, an accumulator and
|
|
/// vector operand are added together and passed to the next iteration as the
|
|
/// next accumulator. After the loop body, the accumulator is reduced to a
|
|
/// scalar value.
|
|
class VPPartialReductionRecipe : public VPSingleDefRecipe {
|
|
unsigned Opcode;
|
|
|
|
public:
|
|
VPPartialReductionRecipe(Instruction *ReductionInst, VPValue *Op0,
|
|
VPValue *Op1)
|
|
: VPPartialReductionRecipe(ReductionInst->getOpcode(), Op0, Op1,
|
|
ReductionInst) {}
|
|
VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1,
|
|
Instruction *ReductionInst = nullptr)
|
|
: VPSingleDefRecipe(VPDef::VPPartialReductionSC,
|
|
ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
|
|
Opcode(Opcode) {
|
|
[[maybe_unused]] auto *AccumulatorRecipe =
|
|
getOperand(1)->getDefiningRecipe();
|
|
assert((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
|
|
isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
|
|
"Unexpected operand order for partial reduction recipe");
|
|
}
|
|
~VPPartialReductionRecipe() override = default;
|
|
|
|
VPPartialReductionRecipe *clone() override {
|
|
return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1),
|
|
getUnderlyingInstr());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPPartialReductionSC)
|
|
|
|
/// Generate the reduction in the loop.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPPartialReductionRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Get the binary op's opcode.
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
|
|
/// instructions.
|
|
class VPBlendRecipe : public VPSingleDefRecipe {
|
|
public:
|
|
/// The blend operation is a User of the incoming values and of their
|
|
/// respective masks, ordered [I0, M0, I1, M1, I2, M2, ...]. Note that M0 can
|
|
/// be omitted (implied by passing an odd number of operands) in which case
|
|
/// all other incoming values are merged into it.
|
|
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
|
|
: VPSingleDefRecipe(VPDef::VPBlendSC, Operands, Phi, Phi->getDebugLoc()) {
|
|
assert(Operands.size() > 0 && "Expected at least one operand!");
|
|
}
|
|
|
|
VPBlendRecipe *clone() override {
|
|
SmallVector<VPValue *> Ops(operands());
|
|
return new VPBlendRecipe(cast<PHINode>(getUnderlyingValue()), Ops);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPBlendSC)
|
|
|
|
/// A normalized blend is one that has an odd number of operands, whereby the
|
|
/// first operand does not have an associated mask.
|
|
bool isNormalized() const { return getNumOperands() % 2; }
|
|
|
|
/// Return the number of incoming values, taking into account when normalized
|
|
/// the first incoming value will have no mask.
|
|
unsigned getNumIncomingValues() const {
|
|
return (getNumOperands() + isNormalized()) / 2;
|
|
}
|
|
|
|
/// Return incoming value number \p Idx.
|
|
VPValue *getIncomingValue(unsigned Idx) const {
|
|
return Idx == 0 ? getOperand(0) : getOperand(Idx * 2 - isNormalized());
|
|
}
|
|
|
|
/// Return mask number \p Idx.
|
|
VPValue *getMask(unsigned Idx) const {
|
|
assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
|
|
return Idx == 0 ? getOperand(1) : getOperand(Idx * 2 + !isNormalized());
|
|
}
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenMemoryRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Recursing through Blend recipes only, must terminate at header phi's the
|
|
// latest.
|
|
return all_of(users(),
|
|
[this](VPUser *U) { return U->onlyFirstLaneUsed(this); });
|
|
}
|
|
};
|
|
|
|
/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
|
|
/// or stores into one wide load/store and shuffles. The first operand of a
|
|
/// VPInterleave recipe is the address, followed by the stored values, followed
|
|
/// by an optional mask.
|
|
class VPInterleaveRecipe : public VPRecipeBase {
|
|
const InterleaveGroup<Instruction> *IG;
|
|
|
|
/// Indicates if the interleave group is in a conditional block and requires a
|
|
/// mask.
|
|
bool HasMask = false;
|
|
|
|
/// Indicates if gaps between members of the group need to be masked out or if
|
|
/// unusued gaps can be loaded speculatively.
|
|
bool NeedsMaskForGaps = false;
|
|
|
|
public:
|
|
VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
|
|
ArrayRef<VPValue *> StoredValues, VPValue *Mask,
|
|
bool NeedsMaskForGaps)
|
|
: VPRecipeBase(VPDef::VPInterleaveSC, {Addr}), IG(IG),
|
|
NeedsMaskForGaps(NeedsMaskForGaps) {
|
|
for (unsigned i = 0; i < IG->getFactor(); ++i)
|
|
if (Instruction *I = IG->getMember(i)) {
|
|
if (I->getType()->isVoidTy())
|
|
continue;
|
|
new VPValue(I, this);
|
|
}
|
|
|
|
for (auto *SV : StoredValues)
|
|
addOperand(SV);
|
|
if (Mask) {
|
|
HasMask = true;
|
|
addOperand(Mask);
|
|
}
|
|
}
|
|
~VPInterleaveRecipe() override = default;
|
|
|
|
VPInterleaveRecipe *clone() override {
|
|
return new VPInterleaveRecipe(IG, getAddr(), getStoredValues(), getMask(),
|
|
NeedsMaskForGaps);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPInterleaveSC)
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getAddr() const {
|
|
return getOperand(0); // Address is the 1st, mandatory operand.
|
|
}
|
|
|
|
/// Return the mask used by this recipe. Note that a full mask is represented
|
|
/// by a nullptr.
|
|
VPValue *getMask() const {
|
|
// Mask is optional and therefore the last, currently 2nd operand.
|
|
return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
|
|
/// Return the VPValues stored by this interleave group. If it is a load
|
|
/// interleave group, return an empty ArrayRef.
|
|
ArrayRef<VPValue *> getStoredValues() const {
|
|
// The first operand is the address, followed by the stored values, followed
|
|
// by an optional mask.
|
|
return ArrayRef<VPValue *>(op_begin(), getNumOperands())
|
|
.slice(1, getNumStoreOperands());
|
|
}
|
|
|
|
/// Generate the wide load or store, and shuffles.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPInterleaveRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const InterleaveGroup<Instruction> *getInterleaveGroup() { return IG; }
|
|
|
|
/// Returns the number of stored operands of this interleave group. Returns 0
|
|
/// for load interleave groups.
|
|
unsigned getNumStoreOperands() const {
|
|
return getNumOperands() - (HasMask ? 2 : 1);
|
|
}
|
|
|
|
/// The recipe only uses the first lane of the address.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
|
|
}
|
|
|
|
Instruction *getInsertPos() const { return IG->getInsertPos(); }
|
|
};
|
|
|
|
/// A recipe to represent inloop reduction operations, performing a reduction on
|
|
/// a vector operand into a scalar value, and adding the result to a chain.
|
|
/// The Operands are {ChainOp, VecOp, [Condition]}.
|
|
class VPReductionRecipe : public VPSingleDefRecipe {
|
|
/// The recurrence decriptor for the reduction in question.
|
|
const RecurrenceDescriptor &RdxDesc;
|
|
bool IsOrdered;
|
|
/// Whether the reduction is conditional.
|
|
bool IsConditional = false;
|
|
|
|
protected:
|
|
VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
|
|
Instruction *I, ArrayRef<VPValue *> Operands,
|
|
VPValue *CondOp, bool IsOrdered, DebugLoc DL)
|
|
: VPSingleDefRecipe(SC, Operands, I, DL), RdxDesc(R),
|
|
IsOrdered(IsOrdered) {
|
|
if (CondOp) {
|
|
IsConditional = true;
|
|
addOperand(CondOp);
|
|
}
|
|
}
|
|
|
|
public:
|
|
VPReductionRecipe(const RecurrenceDescriptor &R, Instruction *I,
|
|
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
|
|
bool IsOrdered, DebugLoc DL = {})
|
|
: VPReductionRecipe(VPDef::VPReductionSC, R, I,
|
|
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
|
|
IsOrdered, DL) {}
|
|
|
|
~VPReductionRecipe() override = default;
|
|
|
|
VPReductionRecipe *clone() override {
|
|
return new VPReductionRecipe(RdxDesc, getUnderlyingInstr(), getChainOp(),
|
|
getVecOp(), getCondOp(), IsOrdered,
|
|
getDebugLoc());
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Generate the reduction in the loop.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of VPReductionRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Return the recurrence decriptor for the in-loop reduction.
|
|
const RecurrenceDescriptor &getRecurrenceDescriptor() const {
|
|
return RdxDesc;
|
|
}
|
|
/// Return true if the in-loop reduction is ordered.
|
|
bool isOrdered() const { return IsOrdered; };
|
|
/// Return true if the in-loop reduction is conditional.
|
|
bool isConditional() const { return IsConditional; };
|
|
/// The VPValue of the scalar Chain being accumulated.
|
|
VPValue *getChainOp() const { return getOperand(0); }
|
|
/// The VPValue of the vector value to be reduced.
|
|
VPValue *getVecOp() const { return getOperand(1); }
|
|
/// The VPValue of the condition for the block.
|
|
VPValue *getCondOp() const {
|
|
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
};
|
|
|
|
/// A recipe to represent inloop reduction operations with vector-predication
|
|
/// intrinsics, performing a reduction on a vector operand with the explicit
|
|
/// vector length (EVL) into a scalar value, and adding the result to a chain.
|
|
/// The Operands are {ChainOp, VecOp, EVL, [Condition]}.
|
|
class VPReductionEVLRecipe : public VPReductionRecipe {
|
|
public:
|
|
VPReductionEVLRecipe(VPReductionRecipe &R, VPValue &EVL, VPValue *CondOp)
|
|
: VPReductionRecipe(
|
|
VPDef::VPReductionEVLSC, R.getRecurrenceDescriptor(),
|
|
cast_or_null<Instruction>(R.getUnderlyingValue()),
|
|
ArrayRef<VPValue *>({R.getChainOp(), R.getVecOp(), &EVL}), CondOp,
|
|
R.isOrdered(), R.getDebugLoc()) {}
|
|
|
|
~VPReductionEVLRecipe() override = default;
|
|
|
|
VPReductionEVLRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPReductionEVLSC)
|
|
|
|
/// Generate the reduction in the loop
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// The VPValue of the explicit vector length.
|
|
VPValue *getEVL() const { return getOperand(2); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Op == getEVL();
|
|
}
|
|
};
|
|
|
|
/// VPReplicateRecipe replicates a given instruction producing multiple scalar
|
|
/// copies of the original scalar type, one per lane, instead of producing a
|
|
/// single copy of widened type for all lanes. If the instruction is known to be
|
|
/// uniform only one copy, per lane zero, will be generated.
|
|
class VPReplicateRecipe : public VPRecipeWithIRFlags {
|
|
/// Indicator if only a single replica per lane is needed.
|
|
bool IsUniform;
|
|
|
|
/// Indicator if the replicas are also predicated.
|
|
bool IsPredicated;
|
|
|
|
public:
|
|
template <typename IterT>
|
|
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
|
|
bool IsUniform, VPValue *Mask = nullptr)
|
|
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
|
|
IsUniform(IsUniform), IsPredicated(Mask) {
|
|
if (Mask)
|
|
addOperand(Mask);
|
|
}
|
|
|
|
~VPReplicateRecipe() override = default;
|
|
|
|
VPReplicateRecipe *clone() override {
|
|
auto *Copy =
|
|
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
|
|
isPredicated() ? getMask() : nullptr);
|
|
Copy->transferFlags(*this);
|
|
return Copy;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPReplicateSC)
|
|
|
|
/// Generate replicas of the desired Ingredient. Replicas will be generated
|
|
/// for all parts and lanes unless a specific part and lane are specified in
|
|
/// the \p State.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPReplicateRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
bool isUniform() const { return IsUniform; }
|
|
|
|
bool isPredicated() const { return IsPredicated; }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return isUniform();
|
|
}
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe is used by a widened recipe via an intervening
|
|
/// VPPredInstPHIRecipe. In this case, the scalar values should also be packed
|
|
/// in a vector.
|
|
bool shouldPack() const;
|
|
|
|
/// Return the mask of a predicated VPReplicateRecipe.
|
|
VPValue *getMask() {
|
|
assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
|
|
return getOperand(getNumOperands() - 1);
|
|
}
|
|
|
|
unsigned getOpcode() const { return getUnderlyingInstr()->getOpcode(); }
|
|
};
|
|
|
|
/// A recipe for generating conditional branches on the bits of a mask.
|
|
class VPBranchOnMaskRecipe : public VPRecipeBase {
|
|
public:
|
|
VPBranchOnMaskRecipe(VPValue *BlockInMask)
|
|
: VPRecipeBase(VPDef::VPBranchOnMaskSC, {}) {
|
|
if (BlockInMask) // nullptr means all-one mask.
|
|
addOperand(BlockInMask);
|
|
}
|
|
|
|
VPBranchOnMaskRecipe *clone() override {
|
|
return new VPBranchOnMaskRecipe(getOperand(0));
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPBranchOnMaskSC)
|
|
|
|
/// Generate the extraction of the appropriate bit from the block mask and the
|
|
/// conditional branch.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPBranchOnMaskRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override {
|
|
O << Indent << "BRANCH-ON-MASK ";
|
|
if (VPValue *Mask = getMask())
|
|
Mask->printAsOperand(O, SlotTracker);
|
|
else
|
|
O << " All-One";
|
|
}
|
|
#endif
|
|
|
|
/// Return the mask used by this recipe. Note that a full mask is represented
|
|
/// by a nullptr.
|
|
VPValue *getMask() const {
|
|
assert(getNumOperands() <= 1 && "should have either 0 or 1 operands");
|
|
// Mask is optional.
|
|
return getNumOperands() == 1 ? getOperand(0) : nullptr;
|
|
}
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
|
|
/// control converges back from a Branch-on-Mask. The phi nodes are needed in
|
|
/// order to merge values that are set under such a branch and feed their uses.
|
|
/// The phi nodes can be scalar or vector depending on the users of the value.
|
|
/// This recipe works in concert with VPBranchOnMaskRecipe.
|
|
class VPPredInstPHIRecipe : public VPSingleDefRecipe {
|
|
public:
|
|
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
|
|
/// nodes after merging back from a Branch-on-Mask.
|
|
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
|
|
: VPSingleDefRecipe(VPDef::VPPredInstPHISC, PredV, DL) {}
|
|
~VPPredInstPHIRecipe() override = default;
|
|
|
|
VPPredInstPHIRecipe *clone() override {
|
|
return new VPPredInstPHIRecipe(getOperand(0), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC)
|
|
|
|
/// Generates phi nodes for live-outs (from a replicate region) as needed to
|
|
/// retain SSA form.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPPredInstPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// A common base class for widening memory operations. An optional mask can be
|
|
/// provided as the last operand.
|
|
class VPWidenMemoryRecipe : public VPRecipeBase {
|
|
protected:
|
|
Instruction &Ingredient;
|
|
|
|
/// Whether the accessed addresses are consecutive.
|
|
bool Consecutive;
|
|
|
|
/// Whether the consecutive accessed addresses are in reverse order.
|
|
bool Reverse;
|
|
|
|
/// Whether the memory access is masked.
|
|
bool IsMasked = false;
|
|
|
|
void setMask(VPValue *Mask) {
|
|
assert(!IsMasked && "cannot re-set mask");
|
|
if (!Mask)
|
|
return;
|
|
addOperand(Mask);
|
|
IsMasked = true;
|
|
}
|
|
|
|
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
|
|
std::initializer_list<VPValue *> Operands,
|
|
bool Consecutive, bool Reverse, DebugLoc DL)
|
|
: VPRecipeBase(SC, Operands, DL), Ingredient(I), Consecutive(Consecutive),
|
|
Reverse(Reverse) {
|
|
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
|
|
}
|
|
|
|
public:
|
|
VPWidenMemoryRecipe *clone() override {
|
|
llvm_unreachable("cloning not supported");
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPDefID() == VPRecipeBase::VPWidenLoadSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenStoreSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenLoadEVLSC ||
|
|
R->getVPDefID() == VPRecipeBase::VPWidenStoreEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Return whether the loaded-from / stored-to addresses are consecutive.
|
|
bool isConsecutive() const { return Consecutive; }
|
|
|
|
/// Return whether the consecutive loaded/stored addresses are in reverse
|
|
/// order.
|
|
bool isReverse() const { return Reverse; }
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getAddr() const { return getOperand(0); }
|
|
|
|
/// Returns true if the recipe is masked.
|
|
bool isMasked() const { return IsMasked; }
|
|
|
|
/// Return the mask used by this recipe. Note that a full mask is represented
|
|
/// by a nullptr.
|
|
VPValue *getMask() const {
|
|
// Mask is optional and therefore the last operand.
|
|
return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
|
|
/// Generate the wide load/store.
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
|
|
}
|
|
|
|
/// Return the cost of this VPWidenMemoryRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
Instruction &getIngredient() const { return Ingredient; }
|
|
};
|
|
|
|
/// A recipe for widening load operations, using the address to load from and an
|
|
/// optional mask.
|
|
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
|
|
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
|
|
bool Consecutive, bool Reverse, DebugLoc DL)
|
|
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
|
|
Reverse, DL),
|
|
VPValue(this, &Load) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VPWidenLoadRecipe *clone() override {
|
|
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
|
|
getMask(), Consecutive, Reverse,
|
|
getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC);
|
|
|
|
/// Generate a wide load or gather.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened, consecutive loads operations only demand the first lane of
|
|
// their address.
|
|
return Op == getAddr() && isConsecutive();
|
|
}
|
|
};
|
|
|
|
/// A recipe for widening load operations with vector-predication intrinsics,
|
|
/// using the address to load from, the explicit vector length and an optional
|
|
/// mask.
|
|
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
|
|
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
|
|
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
|
|
{L.getAddr(), &EVL}, L.isConsecutive(),
|
|
L.isReverse(), L.getDebugLoc()),
|
|
VPValue(this, &getIngredient()) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenLoadEVLSC)
|
|
|
|
/// Return the EVL operand.
|
|
VPValue *getEVL() const { return getOperand(1); }
|
|
|
|
/// Generate the wide load or gather.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenLoadEVLRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened loads only demand the first lane of EVL and consecutive loads
|
|
// only demand the first lane of their address.
|
|
return Op == getEVL() || (Op == getAddr() && isConsecutive());
|
|
}
|
|
};
|
|
|
|
/// A recipe for widening store operations, using the stored value, the address
|
|
/// to store to and an optional mask.
|
|
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
|
|
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
|
|
VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
|
|
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
|
|
Consecutive, Reverse, DL) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VPWidenStoreRecipe *clone() override {
|
|
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
|
|
getStoredValue(), getMask(), Consecutive,
|
|
Reverse, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
|
|
|
|
/// Return the value stored by this recipe.
|
|
VPValue *getStoredValue() const { return getOperand(1); }
|
|
|
|
/// Generate a wide store or scatter.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened, consecutive stores only demand the first lane of their address,
|
|
// unless the same operand is also stored.
|
|
return Op == getAddr() && isConsecutive() && Op != getStoredValue();
|
|
}
|
|
};
|
|
|
|
/// A recipe for widening store operations with vector-predication intrinsics,
|
|
/// using the value to store, the address to store to, the explicit vector
|
|
/// length and an optional mask.
|
|
struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
|
|
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
|
|
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
|
|
{S.getAddr(), S.getStoredValue(), &EVL},
|
|
S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenStoreEVLSC)
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getStoredValue() const { return getOperand(1); }
|
|
|
|
/// Return the EVL operand.
|
|
VPValue *getEVL() const { return getOperand(2); }
|
|
|
|
/// Generate the wide store or scatter.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenStoreEVLRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
if (Op == getEVL()) {
|
|
assert(getStoredValue() != Op && "unexpected store of EVL");
|
|
return true;
|
|
}
|
|
// Widened, consecutive memory operations only demand the first lane of
|
|
// their address, unless the same operand is also stored. That latter can
|
|
// happen with opaque pointers.
|
|
return Op == getAddr() && isConsecutive() && Op != getStoredValue();
|
|
}
|
|
};
|
|
|
|
/// Recipe to expand a SCEV expression.
|
|
class VPExpandSCEVRecipe : public VPSingleDefRecipe {
|
|
const SCEV *Expr;
|
|
ScalarEvolution &SE;
|
|
|
|
public:
|
|
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
|
|
: VPSingleDefRecipe(VPDef::VPExpandSCEVSC, {}), Expr(Expr), SE(SE) {}
|
|
|
|
~VPExpandSCEVRecipe() override = default;
|
|
|
|
VPExpandSCEVRecipe *clone() override {
|
|
return new VPExpandSCEVRecipe(Expr, SE);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPExpandSCEVSC)
|
|
|
|
/// Generate a canonical vector induction variable of the vector loop, with
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPExpandSCEVRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const SCEV *getSCEV() const { return Expr; }
|
|
};
|
|
|
|
/// Canonical scalar induction phi of the vector loop. Starting at the specified
|
|
/// start value (either 0 or the resume value when vectorizing the epilogue
|
|
/// loop). VPWidenCanonicalIVRecipe represents the vector version of the
|
|
/// canonical induction variable.
|
|
class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV, DL) {}
|
|
|
|
~VPCanonicalIVPHIRecipe() override = default;
|
|
|
|
VPCanonicalIVPHIRecipe *clone() override {
|
|
auto *R = new VPCanonicalIVPHIRecipe(getOperand(0), getDebugLoc());
|
|
R->addOperand(getBackedgeValue());
|
|
return R;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC)
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *D) {
|
|
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable(
|
|
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns the scalar type of the induction.
|
|
Type *getScalarType() const {
|
|
return getStartValue()->getLiveInIRValue()->getType();
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool onlyFirstPartUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Return the cost of this VPCanonicalIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// For now, match the behavior of the legacy cost model.
|
|
return 0;
|
|
}
|
|
};
|
|
|
|
/// A recipe for generating the active lane mask for the vector loop that is
|
|
/// used to predicate the vector operations.
|
|
/// TODO: It would be good to use the existing VPWidenPHIRecipe instead and
|
|
/// remove VPActiveLaneMaskPHIRecipe.
|
|
class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask,
|
|
DL) {}
|
|
|
|
~VPActiveLaneMaskPHIRecipe() override = default;
|
|
|
|
VPActiveLaneMaskPHIRecipe *clone() override {
|
|
auto *R = new VPActiveLaneMaskPHIRecipe(getOperand(0), getDebugLoc());
|
|
if (getNumOperands() == 2)
|
|
R->addOperand(getOperand(1));
|
|
return R;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC)
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *D) {
|
|
return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC;
|
|
}
|
|
|
|
/// Generate the active lane mask phi of the vector loop.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for generating the phi node for the current index of elements,
|
|
/// adjusted in accordance with EVL value. It starts at the start value of the
|
|
/// canonical induction and gets incremented by EVL in each iteration of the
|
|
/// vector loop.
|
|
class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPDef::VPEVLBasedIVPHISC, nullptr, StartIV, DL) {}
|
|
|
|
~VPEVLBasedIVPHIRecipe() override = default;
|
|
|
|
VPEVLBasedIVPHIRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPEVLBasedIVPHISC)
|
|
|
|
static inline bool classof(const VPHeaderPHIRecipe *D) {
|
|
return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable(
|
|
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
|
|
}
|
|
|
|
/// Return the cost of this VPEVLBasedIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// For now, match the behavior of the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A Recipe for widening the canonical induction variable of the vector loop.
|
|
class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
|
|
public VPUnrollPartAccessor<1> {
|
|
public:
|
|
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
|
|
: VPSingleDefRecipe(VPDef::VPWidenCanonicalIVSC, {CanonicalIV}) {}
|
|
|
|
~VPWidenCanonicalIVRecipe() override = default;
|
|
|
|
VPWidenCanonicalIVRecipe *clone() override {
|
|
return new VPWidenCanonicalIVRecipe(
|
|
cast<VPCanonicalIVPHIRecipe>(getOperand(0)));
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC)
|
|
|
|
/// Generate a canonical vector induction variable of the vector loop, with
|
|
/// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
|
|
/// step = <VF*UF, VF*UF, ..., VF*UF>.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCanonicalIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for converting the input value \p IV value to the corresponding
|
|
/// value of an IV with different start and step values, using Start + IV *
|
|
/// Step.
|
|
class VPDerivedIVRecipe : public VPSingleDefRecipe {
|
|
/// Kind of the induction.
|
|
const InductionDescriptor::InductionKind Kind;
|
|
/// If not nullptr, the floating point induction binary operator. Must be set
|
|
/// for floating point inductions.
|
|
const FPMathOperator *FPBinOp;
|
|
|
|
/// Name to use for the generated IR instruction for the derived IV.
|
|
std::string Name;
|
|
|
|
public:
|
|
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start,
|
|
VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
|
|
const Twine &Name = "")
|
|
: VPDerivedIVRecipe(
|
|
IndDesc.getKind(),
|
|
dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp()),
|
|
Start, CanonicalIV, Step, Name) {}
|
|
|
|
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
|
|
const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV,
|
|
VPValue *Step, const Twine &Name = "")
|
|
: VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, IV, Step}), Kind(Kind),
|
|
FPBinOp(FPBinOp), Name(Name.str()) {}
|
|
|
|
~VPDerivedIVRecipe() override = default;
|
|
|
|
VPDerivedIVRecipe *clone() override {
|
|
return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
|
|
getStepValue());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC)
|
|
|
|
/// Generate the transformed value of the induction at offset StartValue (1.
|
|
/// operand) + IV (2. operand) * StepValue (3, operand).
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPDerivedIVRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
Type *getScalarType() const {
|
|
return getStartValue()->getLiveInIRValue()->getType();
|
|
}
|
|
|
|
VPValue *getStartValue() const { return getOperand(0); }
|
|
VPValue *getStepValue() const { return getOperand(2); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// A recipe for handling phi nodes of integer and floating-point inductions,
|
|
/// producing their scalar values.
|
|
class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags,
|
|
public VPUnrollPartAccessor<2> {
|
|
Instruction::BinaryOps InductionOpcode;
|
|
|
|
public:
|
|
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step,
|
|
Instruction::BinaryOps Opcode, FastMathFlags FMFs)
|
|
: VPRecipeWithIRFlags(VPDef::VPScalarIVStepsSC,
|
|
ArrayRef<VPValue *>({IV, Step}), FMFs),
|
|
InductionOpcode(Opcode) {}
|
|
|
|
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
|
|
VPValue *Step)
|
|
: VPScalarIVStepsRecipe(
|
|
IV, Step, IndDesc.getInductionOpcode(),
|
|
dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp())
|
|
? IndDesc.getInductionBinOp()->getFastMathFlags()
|
|
: FastMathFlags()) {}
|
|
|
|
~VPScalarIVStepsRecipe() override = default;
|
|
|
|
VPScalarIVStepsRecipe *clone() override {
|
|
return new VPScalarIVStepsRecipe(
|
|
getOperand(0), getOperand(1), InductionOpcode,
|
|
hasFastMathFlags() ? getFastMathFlags() : FastMathFlags());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPDef::VPScalarIVStepsSC)
|
|
|
|
/// Generate the scalarized versions of the phi node as needed by their users.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPScalarIVStepsRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
VPValue *getStepValue() const { return getOperand(1); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool onlyFirstLaneUsed(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
|
|
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
|
|
/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
|
|
class VPBasicBlock : public VPBlockBase {
|
|
friend class VPlan;
|
|
|
|
/// Use VPlan::createVPBasicBlock to create VPBasicBlocks.
|
|
VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
|
|
: VPBlockBase(VPBasicBlockSC, Name.str()) {
|
|
if (Recipe)
|
|
appendRecipe(Recipe);
|
|
}
|
|
|
|
public:
|
|
using RecipeListTy = iplist<VPRecipeBase>;
|
|
|
|
protected:
|
|
/// The VPRecipes held in the order of output instructions to generate.
|
|
RecipeListTy Recipes;
|
|
|
|
VPBasicBlock(const unsigned char BlockSC, const Twine &Name = "")
|
|
: VPBlockBase(BlockSC, Name.str()) {}
|
|
|
|
public:
|
|
~VPBasicBlock() override {
|
|
while (!Recipes.empty())
|
|
Recipes.pop_back();
|
|
}
|
|
|
|
/// Instruction iterators...
|
|
using iterator = RecipeListTy::iterator;
|
|
using const_iterator = RecipeListTy::const_iterator;
|
|
using reverse_iterator = RecipeListTy::reverse_iterator;
|
|
using const_reverse_iterator = RecipeListTy::const_reverse_iterator;
|
|
|
|
//===--------------------------------------------------------------------===//
|
|
/// Recipe iterator methods
|
|
///
|
|
inline iterator begin() { return Recipes.begin(); }
|
|
inline const_iterator begin() const { return Recipes.begin(); }
|
|
inline iterator end() { return Recipes.end(); }
|
|
inline const_iterator end() const { return Recipes.end(); }
|
|
|
|
inline reverse_iterator rbegin() { return Recipes.rbegin(); }
|
|
inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
|
|
inline reverse_iterator rend() { return Recipes.rend(); }
|
|
inline const_reverse_iterator rend() const { return Recipes.rend(); }
|
|
|
|
inline size_t size() const { return Recipes.size(); }
|
|
inline bool empty() const { return Recipes.empty(); }
|
|
inline const VPRecipeBase &front() const { return Recipes.front(); }
|
|
inline VPRecipeBase &front() { return Recipes.front(); }
|
|
inline const VPRecipeBase &back() const { return Recipes.back(); }
|
|
inline VPRecipeBase &back() { return Recipes.back(); }
|
|
|
|
/// Returns a reference to the list of recipes.
|
|
RecipeListTy &getRecipeList() { return Recipes; }
|
|
|
|
/// Returns a pointer to a member of the recipe list.
|
|
static RecipeListTy VPBasicBlock::*getSublistAccess(VPRecipeBase *) {
|
|
return &VPBasicBlock::Recipes;
|
|
}
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC ||
|
|
V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
|
|
}
|
|
|
|
void insert(VPRecipeBase *Recipe, iterator InsertPt) {
|
|
assert(Recipe && "No recipe to append.");
|
|
assert(!Recipe->Parent && "Recipe already in VPlan");
|
|
Recipe->Parent = this;
|
|
Recipes.insert(InsertPt, Recipe);
|
|
}
|
|
|
|
/// Augment the existing recipes of a VPBasicBlock with an additional
|
|
/// \p Recipe as the last recipe.
|
|
void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPBasicBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
/// Return the cost of this VPBasicBlock.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
|
|
|
|
/// Return the position of the first non-phi node recipe in the block.
|
|
iterator getFirstNonPhi();
|
|
|
|
/// Returns an iterator range over the PHI-like recipes in the block.
|
|
iterator_range<iterator> phis() {
|
|
return make_range(begin(), getFirstNonPhi());
|
|
}
|
|
|
|
/// Split current block at \p SplitAt by inserting a new block between the
|
|
/// current block and its successors and moving all recipes starting at
|
|
/// SplitAt to the new block. Returns the new block.
|
|
VPBasicBlock *splitAt(iterator SplitAt);
|
|
|
|
VPRegionBlock *getEnclosingLoopRegion();
|
|
const VPRegionBlock *getEnclosingLoopRegion() const;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
|
|
/// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual blocks is consistent with the whole VPlan printing.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
using VPBlockBase::print; // Get the print(raw_stream &O) version.
|
|
#endif
|
|
|
|
/// If the block has multiple successors, return the branch recipe terminating
|
|
/// the block. If there are no or only a single successor, return nullptr;
|
|
VPRecipeBase *getTerminator();
|
|
const VPRecipeBase *getTerminator() const;
|
|
|
|
/// Returns true if the block is exiting it's parent region.
|
|
bool isExiting() const;
|
|
|
|
/// Clone the current block and it's recipes, without updating the operands of
|
|
/// the cloned recipes.
|
|
VPBasicBlock *clone() override;
|
|
|
|
protected:
|
|
/// Execute the recipes in the IR basic block \p BB.
|
|
void executeRecipes(VPTransformState *State, BasicBlock *BB);
|
|
|
|
/// Connect the VPBBs predecessors' in the VPlan CFG to the IR basic block
|
|
/// generated for this VPBB.
|
|
void connectToPredecessors(VPTransformState::CFGState &CFG);
|
|
|
|
private:
|
|
/// Create an IR BasicBlock to hold the output instructions generated by this
|
|
/// VPBasicBlock, and return it. Update the CFGState accordingly.
|
|
BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
|
|
};
|
|
|
|
/// A special type of VPBasicBlock that wraps an existing IR basic block.
|
|
/// Recipes of the block get added before the first non-phi instruction in the
|
|
/// wrapped block.
|
|
/// Note: At the moment, VPIRBasicBlock can only be used to wrap VPlan's
|
|
/// preheader block.
|
|
class VPIRBasicBlock : public VPBasicBlock {
|
|
friend class VPlan;
|
|
|
|
BasicBlock *IRBB;
|
|
|
|
/// Use VPlan::createVPIRBasicBlock to create VPIRBasicBlocks.
|
|
VPIRBasicBlock(BasicBlock *IRBB)
|
|
: VPBasicBlock(VPIRBasicBlockSC,
|
|
(Twine("ir-bb<") + IRBB->getName() + Twine(">")).str()),
|
|
IRBB(IRBB) {}
|
|
|
|
public:
|
|
~VPIRBasicBlock() override {}
|
|
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
|
|
}
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPBasicBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
VPIRBasicBlock *clone() override;
|
|
|
|
BasicBlock *getIRBasicBlock() const { return IRBB; }
|
|
};
|
|
|
|
/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
|
|
/// which form a Single-Entry-Single-Exiting subgraph of the output IR CFG.
|
|
/// A VPRegionBlock may indicate that its contents are to be replicated several
|
|
/// times. This is designed to support predicated scalarization, in which a
|
|
/// scalar if-then code structure needs to be generated VF * UF times. Having
|
|
/// this replication indicator helps to keep a single model for multiple
|
|
/// candidate VF's. The actual replication takes place only once the desired VF
|
|
/// and UF have been determined.
|
|
class VPRegionBlock : public VPBlockBase {
|
|
friend class VPlan;
|
|
|
|
/// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
|
|
VPBlockBase *Entry;
|
|
|
|
/// Hold the Single Exiting block of the SESE region modelled by the
|
|
/// VPRegionBlock.
|
|
VPBlockBase *Exiting;
|
|
|
|
/// An indicator whether this region is to generate multiple replicated
|
|
/// instances of output IR corresponding to its VPBlockBases.
|
|
bool IsReplicator;
|
|
|
|
/// Use VPlan::createVPRegionBlock to create VPRegionBlocks.
|
|
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting,
|
|
const std::string &Name = "", bool IsReplicator = false)
|
|
: VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
|
|
IsReplicator(IsReplicator) {
|
|
assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
|
|
assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
|
|
Entry->setParent(this);
|
|
Exiting->setParent(this);
|
|
}
|
|
VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
|
|
: VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
|
|
IsReplicator(IsReplicator) {}
|
|
|
|
public:
|
|
~VPRegionBlock() override {}
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
|
|
}
|
|
|
|
const VPBlockBase *getEntry() const { return Entry; }
|
|
VPBlockBase *getEntry() { return Entry; }
|
|
|
|
/// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
|
|
/// EntryBlock must have no predecessors.
|
|
void setEntry(VPBlockBase *EntryBlock) {
|
|
assert(EntryBlock->getPredecessors().empty() &&
|
|
"Entry block cannot have predecessors.");
|
|
Entry = EntryBlock;
|
|
EntryBlock->setParent(this);
|
|
}
|
|
|
|
const VPBlockBase *getExiting() const { return Exiting; }
|
|
VPBlockBase *getExiting() { return Exiting; }
|
|
|
|
/// Set \p ExitingBlock as the exiting VPBlockBase of this VPRegionBlock. \p
|
|
/// ExitingBlock must have no successors.
|
|
void setExiting(VPBlockBase *ExitingBlock) {
|
|
assert(ExitingBlock->getSuccessors().empty() &&
|
|
"Exit block cannot have successors.");
|
|
Exiting = ExitingBlock;
|
|
ExitingBlock->setParent(this);
|
|
}
|
|
|
|
/// Returns the pre-header VPBasicBlock of the loop region.
|
|
VPBasicBlock *getPreheaderVPBB() {
|
|
assert(!isReplicator() && "should only get pre-header of loop regions");
|
|
return getSinglePredecessor()->getExitingBasicBlock();
|
|
}
|
|
|
|
/// An indicator whether this region is to generate multiple replicated
|
|
/// instances of output IR corresponding to its VPBlockBases.
|
|
bool isReplicator() const { return IsReplicator; }
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPRegionBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
// Return the cost of this region.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
|
|
/// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
|
|
/// consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual regions is consistent with the whole VPlan printing.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
using VPBlockBase::print; // Get the print(raw_stream &O) version.
|
|
#endif
|
|
|
|
/// Clone all blocks in the single-entry single-exit region of the block and
|
|
/// their recipes without updating the operands of the cloned recipes.
|
|
VPRegionBlock *clone() override;
|
|
};
|
|
|
|
/// VPlan models a candidate for vectorization, encoding various decisions take
|
|
/// to produce efficient output IR, including which branches, basic-blocks and
|
|
/// output IR instructions to generate, and their cost. VPlan holds a
|
|
/// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
|
|
/// VPBasicBlock.
|
|
class VPlan {
|
|
friend class VPlanPrinter;
|
|
friend class VPSlotTracker;
|
|
|
|
/// VPBasicBlock corresponding to the original preheader. Used to place
|
|
/// VPExpandSCEV recipes for expressions used during skeleton creation and the
|
|
/// rest of VPlan execution.
|
|
/// When this VPlan is used for the epilogue vector loop, the entry will be
|
|
/// replaced by a new entry block created during skeleton creation.
|
|
VPBasicBlock *Entry;
|
|
|
|
/// VPIRBasicBlock wrapping the header of the original scalar loop.
|
|
VPIRBasicBlock *ScalarHeader;
|
|
|
|
/// Holds the VFs applicable to this VPlan.
|
|
SmallSetVector<ElementCount, 2> VFs;
|
|
|
|
/// Holds the UFs applicable to this VPlan. If empty, the VPlan is valid for
|
|
/// any UF.
|
|
SmallSetVector<unsigned, 2> UFs;
|
|
|
|
/// Holds the name of the VPlan, for printing.
|
|
std::string Name;
|
|
|
|
/// Represents the trip count of the original loop, for folding
|
|
/// the tail.
|
|
VPValue *TripCount = nullptr;
|
|
|
|
/// Represents the backedge taken count of the original loop, for folding
|
|
/// the tail. It equals TripCount - 1.
|
|
VPValue *BackedgeTakenCount = nullptr;
|
|
|
|
/// Represents the vector trip count.
|
|
VPValue VectorTripCount;
|
|
|
|
/// Represents the vectorization factor of the loop.
|
|
VPValue VF;
|
|
|
|
/// Represents the loop-invariant VF * UF of the vector loop region.
|
|
VPValue VFxUF;
|
|
|
|
/// Holds a mapping between Values and their corresponding VPValue inside
|
|
/// VPlan.
|
|
Value2VPValueTy Value2VPValue;
|
|
|
|
/// Contains all the external definitions created for this VPlan. External
|
|
/// definitions are VPValues that hold a pointer to their underlying IR.
|
|
SmallVector<VPValue *, 16> VPLiveInsToFree;
|
|
|
|
/// Mapping from SCEVs to the VPValues representing their expansions.
|
|
/// NOTE: This mapping is temporary and will be removed once all users have
|
|
/// been modeled in VPlan directly.
|
|
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
|
|
|
|
/// Blocks allocated and owned by the VPlan. They will be deleted once the
|
|
/// VPlan is destroyed.
|
|
SmallVector<VPBlockBase *> CreatedBlocks;
|
|
|
|
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
|
|
/// wrapping the original header of the scalar loop.
|
|
VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
|
|
: Entry(Entry), ScalarHeader(ScalarHeader) {
|
|
Entry->setPlan(this);
|
|
assert(ScalarHeader->getNumSuccessors() == 0 &&
|
|
"scalar header must be a leaf node");
|
|
}
|
|
|
|
public:
|
|
/// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
|
|
/// original preheader and scalar header of \p L, to be used as entry and
|
|
/// scalar header blocks of the new VPlan.
|
|
VPlan(Loop *L);
|
|
|
|
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
|
|
/// wrapping \p ScalarHeaderBB and a trip count of \p TC.
|
|
VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
|
|
setEntry(createVPBasicBlock("preheader"));
|
|
ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
|
|
TripCount = TC;
|
|
}
|
|
|
|
~VPlan();
|
|
|
|
void setEntry(VPBasicBlock *VPBB) {
|
|
Entry = VPBB;
|
|
VPBB->setPlan(this);
|
|
}
|
|
|
|
/// Create initial VPlan, having an "entry" VPBasicBlock (wrapping
|
|
/// original scalar pre-header) which contains SCEV expansions that need
|
|
/// to happen before the CFG is modified (when executing a VPlan for the
|
|
/// epilogue vector loop, the original entry needs to be replaced by a new
|
|
/// one); a VPBasicBlock for the vector pre-header, followed by a region for
|
|
/// the vector loop, followed by the middle VPBasicBlock. If a check is needed
|
|
/// to guard executing the scalar epilogue loop, it will be added to the
|
|
/// middle block, together with VPBasicBlocks for the scalar preheader and
|
|
/// exit blocks. \p InductionTy is the type of the canonical induction and
|
|
/// used for related values, like the trip count expression.
|
|
static VPlanPtr createInitialVPlan(Type *InductionTy,
|
|
PredicatedScalarEvolution &PSE,
|
|
bool RequiresScalarEpilogueCheck,
|
|
bool TailFolded, Loop *TheLoop);
|
|
|
|
/// Prepare the plan for execution, setting up the required live-in values.
|
|
void prepareToExecute(Value *TripCount, Value *VectorTripCount,
|
|
VPTransformState &State);
|
|
|
|
/// Generate the IR code for this VPlan.
|
|
void execute(VPTransformState *State);
|
|
|
|
/// Return the cost of this plan.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
|
|
|
|
VPBasicBlock *getEntry() { return Entry; }
|
|
const VPBasicBlock *getEntry() const { return Entry; }
|
|
|
|
/// Returns the preheader of the vector loop region, if one exists, or null
|
|
/// otherwise.
|
|
VPBasicBlock *getVectorPreheader() {
|
|
VPRegionBlock *VectorRegion = getVectorLoopRegion();
|
|
return VectorRegion
|
|
? cast<VPBasicBlock>(VectorRegion->getSinglePredecessor())
|
|
: nullptr;
|
|
}
|
|
|
|
/// Returns the VPRegionBlock of the vector loop.
|
|
VPRegionBlock *getVectorLoopRegion();
|
|
const VPRegionBlock *getVectorLoopRegion() const;
|
|
|
|
/// Returns the 'middle' block of the plan, that is the block that selects
|
|
/// whether to execute the scalar tail loop or the exit block from the loop
|
|
/// latch.
|
|
const VPBasicBlock *getMiddleBlock() const {
|
|
return cast<VPBasicBlock>(getScalarPreheader()->getPredecessors().front());
|
|
}
|
|
VPBasicBlock *getMiddleBlock() {
|
|
return cast<VPBasicBlock>(getScalarPreheader()->getPredecessors().front());
|
|
}
|
|
|
|
/// Return the VPBasicBlock for the preheader of the scalar loop.
|
|
VPBasicBlock *getScalarPreheader() const {
|
|
return cast<VPBasicBlock>(getScalarHeader()->getSinglePredecessor());
|
|
}
|
|
|
|
/// Return the VPIRBasicBlock wrapping the header of the scalar loop.
|
|
VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
|
|
|
|
/// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
|
|
/// of the VPlan, that is leaf nodes except the scalar header. Defined in
|
|
/// VPlanHCFG, as the definition of the type needs access to the definitions
|
|
/// of VPBlockShallowTraversalWrapper.
|
|
auto getExitBlocks();
|
|
|
|
/// Returns true if \p VPBB is an exit block.
|
|
bool isExitBlock(VPBlockBase *VPBB);
|
|
|
|
/// The trip count of the original loop.
|
|
VPValue *getTripCount() const {
|
|
assert(TripCount && "trip count needs to be set before accessing it");
|
|
return TripCount;
|
|
}
|
|
|
|
/// Resets the trip count for the VPlan. The caller must make sure all uses of
|
|
/// the original trip count have been replaced.
|
|
void resetTripCount(VPValue *NewTripCount) {
|
|
assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
|
|
"TripCount always must be set");
|
|
TripCount = NewTripCount;
|
|
}
|
|
|
|
/// The backedge taken count of the original loop.
|
|
VPValue *getOrCreateBackedgeTakenCount() {
|
|
if (!BackedgeTakenCount)
|
|
BackedgeTakenCount = new VPValue();
|
|
return BackedgeTakenCount;
|
|
}
|
|
|
|
/// The vector trip count.
|
|
VPValue &getVectorTripCount() { return VectorTripCount; }
|
|
|
|
/// Returns the VF of the vector loop region.
|
|
VPValue &getVF() { return VF; };
|
|
|
|
/// Returns VF * UF of the vector loop region.
|
|
VPValue &getVFxUF() { return VFxUF; }
|
|
|
|
void addVF(ElementCount VF) { VFs.insert(VF); }
|
|
|
|
void setVF(ElementCount VF) {
|
|
assert(hasVF(VF) && "Cannot set VF not already in plan");
|
|
VFs.clear();
|
|
VFs.insert(VF);
|
|
}
|
|
|
|
bool hasVF(ElementCount VF) { return VFs.count(VF); }
|
|
bool hasScalableVF() {
|
|
return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
|
|
}
|
|
|
|
/// Returns an iterator range over all VFs of the plan.
|
|
iterator_range<SmallSetVector<ElementCount, 2>::iterator>
|
|
vectorFactors() const {
|
|
return {VFs.begin(), VFs.end()};
|
|
}
|
|
|
|
bool hasScalarVFOnly() const { return VFs.size() == 1 && VFs[0].isScalar(); }
|
|
|
|
bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
|
|
|
|
unsigned getUF() const {
|
|
assert(UFs.size() == 1 && "Expected a single UF");
|
|
return UFs[0];
|
|
}
|
|
|
|
void setUF(unsigned UF) {
|
|
assert(hasUF(UF) && "Cannot set the UF not already in plan");
|
|
UFs.clear();
|
|
UFs.insert(UF);
|
|
}
|
|
|
|
/// Return a string with the name of the plan and the applicable VFs and UFs.
|
|
std::string getName() const;
|
|
|
|
void setName(const Twine &newName) { Name = newName.str(); }
|
|
|
|
/// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
|
|
/// yet) for \p V.
|
|
VPValue *getOrAddLiveIn(Value *V) {
|
|
assert(V && "Trying to get or add the VPValue of a null Value");
|
|
if (!Value2VPValue.count(V)) {
|
|
VPValue *VPV = new VPValue(V);
|
|
VPLiveInsToFree.push_back(VPV);
|
|
assert(VPV->isLiveIn() && "VPV must be a live-in.");
|
|
assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
|
|
Value2VPValue[V] = VPV;
|
|
}
|
|
|
|
assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
|
|
assert(Value2VPValue[V]->isLiveIn() &&
|
|
"Only live-ins should be in mapping");
|
|
return Value2VPValue[V];
|
|
}
|
|
|
|
/// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
|
|
VPValue *getLiveIn(Value *V) const { return Value2VPValue.lookup(V); }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the live-ins of this VPlan to \p O.
|
|
void printLiveIns(raw_ostream &O) const;
|
|
|
|
/// Print this VPlan to \p O.
|
|
void print(raw_ostream &O) const;
|
|
|
|
/// Print this VPlan in DOT format to \p O.
|
|
void printDOT(raw_ostream &O) const;
|
|
|
|
/// Dump the plan to stderr (for debugging).
|
|
LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
|
|
/// Returns the canonical induction recipe of the vector loop.
|
|
VPCanonicalIVPHIRecipe *getCanonicalIV() {
|
|
VPBasicBlock *EntryVPBB = getVectorLoopRegion()->getEntryBasicBlock();
|
|
if (EntryVPBB->empty()) {
|
|
// VPlan native path.
|
|
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
|
|
}
|
|
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
|
|
}
|
|
|
|
VPValue *getSCEVExpansion(const SCEV *S) const {
|
|
return SCEVToExpansion.lookup(S);
|
|
}
|
|
|
|
void addSCEVExpansion(const SCEV *S, VPValue *V) {
|
|
assert(!SCEVToExpansion.contains(S) && "SCEV already expanded");
|
|
SCEVToExpansion[S] = V;
|
|
}
|
|
|
|
/// Clone the current VPlan, update all VPValues of the new VPlan and cloned
|
|
/// recipes to refer to the clones, and return it.
|
|
VPlan *duplicate();
|
|
|
|
/// Create a new VPBasicBlock with \p Name and containing \p Recipe if
|
|
/// present. The returned block is owned by the VPlan and deleted once the
|
|
/// VPlan is destroyed.
|
|
VPBasicBlock *createVPBasicBlock(const Twine &Name,
|
|
VPRecipeBase *Recipe = nullptr) {
|
|
auto *VPB = new VPBasicBlock(Name, Recipe);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a new VPRegionBlock with \p Entry, \p Exiting and \p Name. If \p
|
|
/// IsReplicator is true, the region is a replicate region. The returned block
|
|
/// is owned by the VPlan and deleted once the VPlan is destroyed.
|
|
VPRegionBlock *createVPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting,
|
|
const std::string &Name = "",
|
|
bool IsReplicator = false) {
|
|
auto *VPB = new VPRegionBlock(Entry, Exiting, Name, IsReplicator);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a new VPRegionBlock with \p Name and entry and exiting blocks set
|
|
/// to nullptr. If \p IsReplicator is true, the region is a replicate region.
|
|
/// The returned block is owned by the VPlan and deleted once the VPlan is
|
|
/// destroyed.
|
|
VPRegionBlock *createVPRegionBlock(const std::string &Name = "",
|
|
bool IsReplicator = false) {
|
|
auto *VPB = new VPRegionBlock(Name, IsReplicator);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a VPIRBasicBlock wrapping \p IRBB, but do not create
|
|
/// VPIRInstructions wrapping the instructions in t\p IRBB. The returned
|
|
/// block is owned by the VPlan and deleted once the VPlan is destroyed.
|
|
VPIRBasicBlock *createEmptyVPIRBasicBlock(BasicBlock *IRBB);
|
|
|
|
/// Create a VPIRBasicBlock from \p IRBB containing VPIRInstructions for all
|
|
/// instructions in \p IRBB, except its terminator which is managed by the
|
|
/// successors of the block in VPlan. The returned block is owned by the VPlan
|
|
/// and deleted once the VPlan is destroyed.
|
|
VPIRBasicBlock *createVPIRBasicBlock(BasicBlock *IRBB);
|
|
};
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// VPlanPrinter prints a given VPlan to a given output stream. The printing is
|
|
/// indented and follows the dot format.
|
|
class VPlanPrinter {
|
|
raw_ostream &OS;
|
|
const VPlan &Plan;
|
|
unsigned Depth = 0;
|
|
unsigned TabWidth = 2;
|
|
std::string Indent;
|
|
unsigned BID = 0;
|
|
SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
|
|
|
|
VPSlotTracker SlotTracker;
|
|
|
|
/// Handle indentation.
|
|
void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
|
|
|
|
/// Print a given \p Block of the Plan.
|
|
void dumpBlock(const VPBlockBase *Block);
|
|
|
|
/// Print the information related to the CFG edges going out of a given
|
|
/// \p Block, followed by printing the successor blocks themselves.
|
|
void dumpEdges(const VPBlockBase *Block);
|
|
|
|
/// Print a given \p BasicBlock, including its VPRecipes, followed by printing
|
|
/// its successor blocks.
|
|
void dumpBasicBlock(const VPBasicBlock *BasicBlock);
|
|
|
|
/// Print a given \p Region of the Plan.
|
|
void dumpRegion(const VPRegionBlock *Region);
|
|
|
|
unsigned getOrCreateBID(const VPBlockBase *Block) {
|
|
return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
|
|
}
|
|
|
|
Twine getOrCreateName(const VPBlockBase *Block);
|
|
|
|
Twine getUID(const VPBlockBase *Block);
|
|
|
|
/// Print the information related to a CFG edge between two VPBlockBases.
|
|
void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
|
|
const Twine &Label);
|
|
|
|
public:
|
|
VPlanPrinter(raw_ostream &O, const VPlan &P)
|
|
: OS(O), Plan(P), SlotTracker(&P) {}
|
|
|
|
LLVM_DUMP_METHOD void dump();
|
|
};
|
|
|
|
struct VPlanIngredient {
|
|
const Value *V;
|
|
|
|
VPlanIngredient(const Value *V) : V(V) {}
|
|
|
|
void print(raw_ostream &O) const;
|
|
};
|
|
|
|
inline raw_ostream &operator<<(raw_ostream &OS, const VPlanIngredient &I) {
|
|
I.print(OS);
|
|
return OS;
|
|
}
|
|
|
|
inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
|
|
Plan.print(OS);
|
|
return OS;
|
|
}
|
|
#endif
|
|
|
|
class VPInterleavedAccessInfo {
|
|
DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
|
|
InterleaveGroupMap;
|
|
|
|
/// Type for mapping of instruction based interleave groups to VPInstruction
|
|
/// interleave groups
|
|
using Old2NewTy = DenseMap<InterleaveGroup<Instruction> *,
|
|
InterleaveGroup<VPInstruction> *>;
|
|
|
|
/// Recursively \p Region and populate VPlan based interleave groups based on
|
|
/// \p IAI.
|
|
void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
|
|
InterleavedAccessInfo &IAI);
|
|
/// Recursively traverse \p Block and populate VPlan based interleave groups
|
|
/// based on \p IAI.
|
|
void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
|
|
InterleavedAccessInfo &IAI);
|
|
|
|
public:
|
|
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
|
|
|
|
~VPInterleavedAccessInfo() {
|
|
SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet;
|
|
// Avoid releasing a pointer twice.
|
|
for (auto &I : InterleaveGroupMap)
|
|
DelSet.insert(I.second);
|
|
for (auto *Ptr : DelSet)
|
|
delete Ptr;
|
|
}
|
|
|
|
/// Get the interleave group that \p Instr belongs to.
|
|
///
|
|
/// \returns nullptr if doesn't have such group.
|
|
InterleaveGroup<VPInstruction> *
|
|
getInterleaveGroup(VPInstruction *Instr) const {
|
|
return InterleaveGroupMap.lookup(Instr);
|
|
}
|
|
};
|
|
|
|
/// Class that maps (parts of) an existing VPlan to trees of combined
|
|
/// VPInstructions.
|
|
class VPlanSlp {
|
|
enum class OpMode { Failed, Load, Opcode };
|
|
|
|
/// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
|
|
/// DenseMap keys.
|
|
struct BundleDenseMapInfo {
|
|
static SmallVector<VPValue *, 4> getEmptyKey() {
|
|
return {reinterpret_cast<VPValue *>(-1)};
|
|
}
|
|
|
|
static SmallVector<VPValue *, 4> getTombstoneKey() {
|
|
return {reinterpret_cast<VPValue *>(-2)};
|
|
}
|
|
|
|
static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
|
|
return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
|
|
}
|
|
|
|
static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
|
|
const SmallVector<VPValue *, 4> &RHS) {
|
|
return LHS == RHS;
|
|
}
|
|
};
|
|
|
|
/// Mapping of values in the original VPlan to a combined VPInstruction.
|
|
DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
|
|
BundleToCombined;
|
|
|
|
VPInterleavedAccessInfo &IAI;
|
|
|
|
/// Basic block to operate on. For now, only instructions in a single BB are
|
|
/// considered.
|
|
const VPBasicBlock &BB;
|
|
|
|
/// Indicates whether we managed to combine all visited instructions or not.
|
|
bool CompletelySLP = true;
|
|
|
|
/// Width of the widest combined bundle in bits.
|
|
unsigned WidestBundleBits = 0;
|
|
|
|
using MultiNodeOpTy =
|
|
typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
|
|
|
|
// Input operand bundles for the current multi node. Each multi node operand
|
|
// bundle contains values not matching the multi node's opcode. They will
|
|
// be reordered in reorderMultiNodeOps, once we completed building a
|
|
// multi node.
|
|
SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
|
|
|
|
/// Indicates whether we are building a multi node currently.
|
|
bool MultiNodeActive = false;
|
|
|
|
/// Check if we can vectorize Operands together.
|
|
bool areVectorizable(ArrayRef<VPValue *> Operands) const;
|
|
|
|
/// Add combined instruction \p New for the bundle \p Operands.
|
|
void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
|
|
|
|
/// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
|
|
VPInstruction *markFailed();
|
|
|
|
/// Reorder operands in the multi node to maximize sequential memory access
|
|
/// and commutative operations.
|
|
SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
|
|
|
|
/// Choose the best candidate to use for the lane after \p Last. The set of
|
|
/// candidates to choose from are values with an opcode matching \p Last's
|
|
/// or loads consecutive to \p Last.
|
|
std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
|
|
SmallPtrSetImpl<VPValue *> &Candidates,
|
|
VPInterleavedAccessInfo &IAI);
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print bundle \p Values to dbgs().
|
|
void dumpBundle(ArrayRef<VPValue *> Values);
|
|
#endif
|
|
|
|
public:
|
|
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {}
|
|
|
|
~VPlanSlp() = default;
|
|
|
|
/// Tries to build an SLP tree rooted at \p Operands and returns a
|
|
/// VPInstruction combining \p Operands, if they can be combined.
|
|
VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
|
|
|
|
/// Return the width of the widest combined bundle in bits.
|
|
unsigned getWidestBundleBits() const { return WidestBundleBits; }
|
|
|
|
/// Return true if all visited instruction can be combined.
|
|
bool isCompletelySLP() const { return CompletelySLP; }
|
|
};
|
|
} // end namespace llvm
|
|
|
|
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|