[SLP] Support vectorizing 2^N-1 reductions (#106266)
Build on the -slp-vectorize-non-power-of-2 experimental option, and support vectorizing reductions with 2^N-1 sized vector. Specifically, two related changes: 1) When searching for a profitable VL, start with the 2^N-1 reduction width. If cost model does not select that VL, return to power of two boundaries when halfing the search VL. The later is mostly for simplicity. 2) Reduce the minimum reduction width from 4 to 3 when supporting non-power of two vectors. This is required to support <3 x Ty> cases. One thing which isn't directly related to this change, but I want to note for clarity is that the non-power-of-two vectorization appears to be sensative to operand order of reduction. I haven't yet fully figured out why, but I suspect this is non-power-of-two specific.
This commit is contained in:
Philip Reames
@@ -17583,7 +17583,7 @@ public:
|
||||
/// Attempt to vectorize the tree found by matchAssociativeReduction.
|
||||
Value *tryToReduce(BoUpSLP &V, const DataLayout &DL, TargetTransformInfo *TTI,
|
||||
const TargetLibraryInfo &TLI) {
|
||||
constexpr int ReductionLimit = 4;
|
||||
const unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4;
|
||||
constexpr unsigned RegMaxNumber = 4;
|
||||
constexpr unsigned RedValsMaxNumber = 128;
|
||||
// If there are a sufficient number of reduction values, reduce
|
||||
@@ -17799,13 +17799,15 @@ public:
|
||||
|
||||
unsigned MaxVecRegSize = V.getMaxVecRegSize();
|
||||
unsigned EltSize = V.getVectorElementSize(Candidates[0]);
|
||||
unsigned MaxElts =
|
||||
RegMaxNumber * llvm::bit_floor(MaxVecRegSize / EltSize);
|
||||
const unsigned MaxElts = std::clamp<unsigned>(
|
||||
llvm::bit_floor(MaxVecRegSize / EltSize), RedValsMaxNumber,
|
||||
RegMaxNumber * RedValsMaxNumber);
|
||||
|
||||
unsigned ReduxWidth = NumReducedVals;
|
||||
if (!VectorizeNonPowerOf2 || !has_single_bit(ReduxWidth + 1))
|
||||
ReduxWidth = bit_floor(ReduxWidth);
|
||||
ReduxWidth = std::min(ReduxWidth, MaxElts);
|
||||
|
||||
unsigned ReduxWidth = std::min<unsigned>(
|
||||
llvm::bit_floor(NumReducedVals),
|
||||
std::clamp<unsigned>(MaxElts, RedValsMaxNumber,
|
||||
RegMaxNumber * RedValsMaxNumber));
|
||||
unsigned Start = 0;
|
||||
unsigned Pos = Start;
|
||||
// Restarts vectorization attempt with lower vector factor.
|
||||
@@ -17825,7 +17827,7 @@ public:
|
||||
if (Pos < NumReducedVals - ReduxWidth + 1)
|
||||
return IsAnyRedOpGathered;
|
||||
Pos = Start;
|
||||
ReduxWidth /= 2;
|
||||
ReduxWidth = bit_ceil(ReduxWidth) / 2;
|
||||
return IsAnyRedOpGathered;
|
||||
};
|
||||
bool AnyVectorized = false;
|
||||
@@ -18014,12 +18016,10 @@ public:
|
||||
createStrideMask(I, ScalarTyNumElements, VL.size());
|
||||
Value *Lane = Builder.CreateShuffleVector(VectorizedRoot, Mask);
|
||||
ReducedSubTree = Builder.CreateInsertElement(
|
||||
ReducedSubTree, emitReduction(Lane, Builder, ReduxWidth, TTI),
|
||||
I);
|
||||
ReducedSubTree, emitReduction(Lane, Builder, TTI), I);
|
||||
}
|
||||
} else {
|
||||
ReducedSubTree =
|
||||
emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
|
||||
ReducedSubTree = emitReduction(VectorizedRoot, Builder, TTI);
|
||||
}
|
||||
if (ReducedSubTree->getType() != VL.front()->getType()) {
|
||||
assert(ReducedSubTree->getType() != VL.front()->getType() &&
|
||||
@@ -18301,10 +18301,8 @@ private:
|
||||
|
||||
/// Emit a horizontal reduction of the vectorized value.
|
||||
Value *emitReduction(Value *VectorizedValue, IRBuilderBase &Builder,
|
||||
unsigned ReduxWidth, const TargetTransformInfo *TTI) {
|
||||
const TargetTransformInfo *TTI) {
|
||||
assert(VectorizedValue && "Need to have a vectorized tree node");
|
||||
assert(has_single_bit(ReduxWidth) &&
|
||||
"We only handle power-of-two reductions for now");
|
||||
assert(RdxKind != RecurKind::FMulAdd &&
|
||||
"A call to the llvm.fmuladd intrinsic is not handled yet");
|
||||
|
||||
|
||||
@@ -434,16 +434,22 @@ define i32 @reduce_add(ptr %src) {
|
||||
}
|
||||
|
||||
define float @reduce_fadd(ptr %src) {
|
||||
; CHECK-LABEL: @reduce_fadd(
|
||||
; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
|
||||
; CHECK-NEXT: [[L_SRC_0:%.*]] = load float, ptr [[GEP_SRC_0]], align 4
|
||||
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 1
|
||||
; CHECK-NEXT: [[L_SRC_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
|
||||
; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
|
||||
; CHECK-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
|
||||
; CHECK-NEXT: [[ADD_0:%.*]] = fadd fast float [[L_SRC_0]], [[L_SRC_1]]
|
||||
; CHECK-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[L_SRC_2]]
|
||||
; CHECK-NEXT: ret float [[ADD_1]]
|
||||
; NON-POW2-LABEL: @reduce_fadd(
|
||||
; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v3f32(float -0.000000e+00, <3 x float> [[TMP1]])
|
||||
; NON-POW2-NEXT: ret float [[TMP2]]
|
||||
;
|
||||
; POW2-ONLY-LABEL: @reduce_fadd(
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[L_SRC_0:%.*]] = load float, ptr [[GEP_SRC_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 1
|
||||
; POW2-ONLY-NEXT: [[L_SRC_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast float [[L_SRC_0]], [[L_SRC_1]]
|
||||
; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[L_SRC_2]]
|
||||
; POW2-ONLY-NEXT: ret float [[ADD_1]]
|
||||
;
|
||||
%gep.src.0 = getelementptr inbounds float, ptr %src, i32 0
|
||||
%l.src.0 = load float, ptr %gep.src.0, align 4
|
||||
@@ -458,19 +464,26 @@ define float @reduce_fadd(ptr %src) {
|
||||
}
|
||||
|
||||
define i32 @reduce_add_after_mul(ptr %src) {
|
||||
; CHECK-LABEL: @reduce_add_after_mul(
|
||||
; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
|
||||
; CHECK-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4
|
||||
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1
|
||||
; CHECK-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4
|
||||
; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2
|
||||
; CHECK-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
|
||||
; CHECK-NEXT: [[MUL_0:%.*]] = mul nsw i32 [[L_SRC_0]], 10
|
||||
; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[L_SRC_1]], 10
|
||||
; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10
|
||||
; CHECK-NEXT: [[ADD_0:%.*]] = add i32 [[MUL_0]], [[MUL_1]]
|
||||
; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]
|
||||
; CHECK-NEXT: ret i32 [[ADD_1]]
|
||||
; NON-POW2-LABEL: @reduce_add_after_mul(
|
||||
; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP1]], <i32 10, i32 10, i32 10>
|
||||
; NON-POW2-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP2]])
|
||||
; NON-POW2-NEXT: ret i32 [[TMP3]]
|
||||
;
|
||||
; POW2-ONLY-LABEL: @reduce_add_after_mul(
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1
|
||||
; POW2-ONLY-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[MUL_0:%.*]] = mul nsw i32 [[L_SRC_0]], 10
|
||||
; POW2-ONLY-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[L_SRC_1]], 10
|
||||
; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10
|
||||
; POW2-ONLY-NEXT: [[ADD_0:%.*]] = add i32 [[MUL_0]], [[MUL_1]]
|
||||
; POW2-ONLY-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]
|
||||
; POW2-ONLY-NEXT: ret i32 [[ADD_1]]
|
||||
;
|
||||
%gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0
|
||||
%l.src.0 = load i32, ptr %gep.src.0, align 4
|
||||
@@ -489,25 +502,34 @@ define i32 @reduce_add_after_mul(ptr %src) {
|
||||
}
|
||||
|
||||
define i32 @dot_product_i32(ptr %a, ptr %b) {
|
||||
; CHECK-LABEL: @dot_product_i32(
|
||||
; CHECK-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0
|
||||
; CHECK-NEXT: [[L_A_0:%.*]] = load i32, ptr [[GEP_A_0]], align 4
|
||||
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 1
|
||||
; CHECK-NEXT: [[L_A_1:%.*]] = load i32, ptr [[GEP_A_1]], align 4
|
||||
; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2
|
||||
; CHECK-NEXT: [[L_A_2:%.*]] = load i32, ptr [[GEP_A_2]], align 4
|
||||
; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0
|
||||
; CHECK-NEXT: [[L_B_0:%.*]] = load i32, ptr [[GEP_B_0]], align 4
|
||||
; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 1
|
||||
; CHECK-NEXT: [[L_B_1:%.*]] = load i32, ptr [[GEP_B_1]], align 4
|
||||
; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2
|
||||
; CHECK-NEXT: [[L_B_2:%.*]] = load i32, ptr [[GEP_B_2]], align 4
|
||||
; CHECK-NEXT: [[MUL_0:%.*]] = mul nsw i32 [[L_A_0]], [[L_B_0]]
|
||||
; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[L_A_1]], [[L_B_1]]
|
||||
; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_A_2]], [[L_B_2]]
|
||||
; CHECK-NEXT: [[ADD_0:%.*]] = add i32 [[MUL_0]], [[MUL_1]]
|
||||
; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]
|
||||
; CHECK-NEXT: ret i32 [[ADD_1]]
|
||||
; NON-POW2-LABEL: @dot_product_i32(
|
||||
; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_A_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x i32>, ptr [[GEP_B_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP3:%.*]] = mul nsw <3 x i32> [[TMP1]], [[TMP2]]
|
||||
; NON-POW2-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP3]])
|
||||
; NON-POW2-NEXT: ret i32 [[TMP4]]
|
||||
;
|
||||
; POW2-ONLY-LABEL: @dot_product_i32(
|
||||
; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[L_A_0:%.*]] = load i32, ptr [[GEP_A_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 1
|
||||
; POW2-ONLY-NEXT: [[L_A_1:%.*]] = load i32, ptr [[GEP_A_1]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load i32, ptr [[GEP_A_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[L_B_0:%.*]] = load i32, ptr [[GEP_B_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 1
|
||||
; POW2-ONLY-NEXT: [[L_B_1:%.*]] = load i32, ptr [[GEP_B_1]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load i32, ptr [[GEP_B_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[MUL_0:%.*]] = mul nsw i32 [[L_A_0]], [[L_B_0]]
|
||||
; POW2-ONLY-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[L_A_1]], [[L_B_1]]
|
||||
; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_A_2]], [[L_B_2]]
|
||||
; POW2-ONLY-NEXT: [[ADD_0:%.*]] = add i32 [[MUL_0]], [[MUL_1]]
|
||||
; POW2-ONLY-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]
|
||||
; POW2-ONLY-NEXT: ret i32 [[ADD_1]]
|
||||
;
|
||||
%gep.a.0 = getelementptr inbounds i32, ptr %a, i32 0
|
||||
%l.a.0 = load i32, ptr %gep.a.0, align 4
|
||||
@@ -533,22 +555,31 @@ define i32 @dot_product_i32(ptr %a, ptr %b) {
|
||||
}
|
||||
|
||||
define float @dot_product_fp32(ptr %a, ptr %b) {
|
||||
; CHECK-LABEL: @dot_product_fp32(
|
||||
; CHECK-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0
|
||||
; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
|
||||
; CHECK-NEXT: [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4
|
||||
; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0
|
||||
; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
|
||||
; CHECK-NEXT: [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
|
||||
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_A_0]], align 4
|
||||
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[GEP_B_0]], align 4
|
||||
; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP2]]
|
||||
; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]
|
||||
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
|
||||
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
|
||||
; CHECK-NEXT: [[ADD_0:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
|
||||
; CHECK-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]
|
||||
; CHECK-NEXT: ret float [[ADD_1]]
|
||||
; NON-POW2-LABEL: @dot_product_fp32(
|
||||
; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[GEP_A_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[GEP_B_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP3:%.*]] = fmul fast <3 x float> [[TMP1]], [[TMP2]]
|
||||
; NON-POW2-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v3f32(float -0.000000e+00, <3 x float> [[TMP3]])
|
||||
; NON-POW2-NEXT: ret float [[TMP4]]
|
||||
;
|
||||
; POW2-ONLY-LABEL: @dot_product_fp32(
|
||||
; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_A_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[GEP_B_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP2]]
|
||||
; POW2-ONLY-NEXT: [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]
|
||||
; POW2-ONLY-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
|
||||
; POW2-ONLY-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
|
||||
; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
|
||||
; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]
|
||||
; POW2-ONLY-NEXT: ret float [[ADD_1]]
|
||||
;
|
||||
%gep.a.0 = getelementptr inbounds float, ptr %a, i32 0
|
||||
%l.a.0 = load float, ptr %gep.a.0, align 4
|
||||
@@ -574,22 +605,31 @@ define float @dot_product_fp32(ptr %a, ptr %b) {
|
||||
}
|
||||
|
||||
define double @dot_product_fp64(ptr %a, ptr %b) {
|
||||
; CHECK-LABEL: @dot_product_fp64(
|
||||
; CHECK-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i32 0
|
||||
; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds double, ptr [[A]], i32 2
|
||||
; CHECK-NEXT: [[L_A_2:%.*]] = load double, ptr [[GEP_A_2]], align 4
|
||||
; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i32 0
|
||||
; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds double, ptr [[B]], i32 2
|
||||
; CHECK-NEXT: [[L_B_2:%.*]] = load double, ptr [[GEP_B_2]], align 4
|
||||
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[GEP_A_0]], align 4
|
||||
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[GEP_B_0]], align 4
|
||||
; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
|
||||
; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast double [[L_A_2]], [[L_B_2]]
|
||||
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
|
||||
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
|
||||
; CHECK-NEXT: [[ADD_0:%.*]] = fadd fast double [[TMP4]], [[TMP5]]
|
||||
; CHECK-NEXT: [[ADD_1:%.*]] = fadd fast double [[ADD_0]], [[MUL_2]]
|
||||
; CHECK-NEXT: ret double [[ADD_1]]
|
||||
; NON-POW2-LABEL: @dot_product_fp64(
|
||||
; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i32 0
|
||||
; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x double>, ptr [[GEP_A_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x double>, ptr [[GEP_B_0]], align 4
|
||||
; NON-POW2-NEXT: [[TMP3:%.*]] = fmul fast <3 x double> [[TMP1]], [[TMP2]]
|
||||
; NON-POW2-NEXT: [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v3f64(double -0.000000e+00, <3 x double> [[TMP3]])
|
||||
; NON-POW2-NEXT: ret double [[TMP4]]
|
||||
;
|
||||
; POW2-ONLY-LABEL: @dot_product_fp64(
|
||||
; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds double, ptr [[A]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load double, ptr [[GEP_A_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i32 0
|
||||
; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds double, ptr [[B]], i32 2
|
||||
; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load double, ptr [[GEP_B_2]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[GEP_A_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[GEP_B_0]], align 4
|
||||
; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
|
||||
; POW2-ONLY-NEXT: [[MUL_2:%.*]] = fmul fast double [[L_A_2]], [[L_B_2]]
|
||||
; POW2-ONLY-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
|
||||
; POW2-ONLY-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
|
||||
; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast double [[TMP4]], [[TMP5]]
|
||||
; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast double [[ADD_0]], [[MUL_2]]
|
||||
; POW2-ONLY-NEXT: ret double [[ADD_1]]
|
||||
;
|
||||
%gep.a.0 = getelementptr inbounds double, ptr %a, i32 0
|
||||
%l.a.0 = load double, ptr %gep.a.0, align 4
|
||||
|
||||
Reference in New Issue
Block a user