caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ef2cdfe393d01cd4935c806387ac912b5a2c8ced
clang-p2996/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
Mauri Mustonen 0de8aeae72 [VPlan] Support to widen select intructions in VPlan native path 
Add support to widen select instructions in VPlan native path by using a correct recipe when such instructions are encountered. This is already used by inner loop vectorizer.

Previously select instructions get handled by the wrong recipe and resulted in unreachable instruction errors like this one: https://bugs.llvm.org/show_bug.cgi?id=48139.

Reviewed By: fhahn

Differential Revision: https://reviews.llvm.org/D97136
2021-03-10 20:59:53 +00:00

201 lines
11 KiB
LLVM
Raw Blame History

; RUN: opt -loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
; Test that VPlan native path is able to widen select instruction in the
; innermost loop under different conditions when outer loop is marked to be
; vectorized. These conditions include following:
; * Inner and outer loop invariant select condition
; * Select condition depending on outer loop iteration variable.
; * Select condidition depending on inner loop iteration variable.
; * Select conditition depending on both outer and inner loop iteration
; variables.
define void @loop_invariant_select(double* noalias nocapture %out, i1 %select, double %a, double %b) {
; CHECK-LABEL: @loop_invariant_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]]
; CHECK: for2.header1:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP2:%.*]], [[FOR2_HEADER1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SELECT:%.*]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> [[TMP1]], <4 x double*> [[TMP0]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
entry:
br label %for1.header
for1.header:
%indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
%ptr = getelementptr inbounds double, double* %out, i64 %indvar1
br label %for2.header
for2.header:
%indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]
; Select condition is loop invariant for both inner and outer loop.
%select.b = select i1 %select, double %a, double %b
store double %select.b, double* %ptr, align 8
%indvar21 = add nuw nsw i64 %indvar2, 1
%for2.cond = icmp eq i64 %indvar21, 10000
br i1 %for2.cond, label %for1.latch, label %for2.header
for1.latch:
%indvar11 = add nuw nsw i64 %indvar1, 1
%for1.cond = icmp eq i64 %indvar11, 1000
br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
exit:
ret void
}
define void @outer_loop_dependant_select(double* noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: @outer_loop_dependant_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]]
; CHECK: for2.header1:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i1>
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> [[TMP2]], <4 x double*> [[TMP0]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
entry:
br label %for1.header
for1.header:
%indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
%ptr = getelementptr inbounds double, double* %out, i64 %indvar1
br label %for2.header
for2.header:
%indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]
%select = trunc i64 %indvar1 to i1
; Select condition only depends on outer loop iteration variable.
%select.b = select i1 %select, double %a, double %b
store double %select.b, double* %ptr, align 8
%indvar21 = add nuw nsw i64 %indvar2, 1
%for2.cond = icmp eq i64 %indvar21, 10000
br i1 %for2.cond, label %for1.latch, label %for2.header
for1.latch:
%indvar11 = add nuw nsw i64 %indvar1, 1
%for1.cond = icmp eq i64 %indvar11, 1000
br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
exit:
ret void
}
define void @inner_loop_dependant_select(double* noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: @inner_loop_dependant_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]]
; CHECK: for2.header1:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_PHI]] to <4 x i1>
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> [[TMP2]], <4 x double*> [[TMP0]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
entry:
br label %for1.header
for1.header:
%indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
%ptr = getelementptr inbounds double, double* %out, i64 %indvar1
br label %for2.header
for2.header:
%indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]
%select = trunc i64 %indvar2 to i1
; Select condition only depends on inner loop iteration variable.
%select.b = select i1 %select, double %a, double %b
store double %select.b, double* %ptr, align 8
%indvar21 = add nuw nsw i64 %indvar2, 1
%for2.cond = icmp eq i64 %indvar21, 10000
br i1 %for2.cond, label %for1.latch, label %for2.header
for1.latch:
%indvar11 = add nuw nsw i64 %indvar1, 1
%for1.cond = icmp eq i64 %indvar11, 1000
br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
exit:
ret void
}
define void @outer_and_inner_loop_dependant_select(double* noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: @outer_and_inner_loop_dependant_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]]
; CHECK: for2.header1:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR2_HEADER1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i1>
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> [[TMP3]], <4 x double*> [[TMP0]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
entry:
br label %for1.header
for1.header:
%indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
%ptr = getelementptr inbounds double, double* %out, i64 %indvar1
br label %for2.header
for2.header:
%indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]
%sum = add nuw nsw i64 %indvar1, %indvar2
%select = trunc i64 %sum to i1
; Select condition depends on both inner and outer loop iteration variables.
%select.b = select i1 %select, double %a, double %b
store double %select.b, double* %ptr, align 8
%indvar21 = add nuw nsw i64 %indvar2, 1
%for2.cond = icmp eq i64 %indvar21, 10000
br i1 %for2.cond, label %for1.latch, label %for2.header
for1.latch:
%indvar11 = add nuw nsw i64 %indvar1, 1
%for1.cond = icmp eq i64 %indvar11, 1000
br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
exit:
ret void
}
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
Reference in New Issue View Git Blame Copy Permalink