86f18d394e
Having FindFirstIV tests in if-reduction.ll is misleading, and iv-select-cmp.ll is already too large.
203 lines
10 KiB
LLVM
203 lines
10 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
|
|
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
|
|
; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s
|
|
; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s
|
|
|
|
define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
|
|
; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
|
|
; CHECK-SAME: ptr [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
|
|
; CHECK-NEXT: [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
|
|
; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
|
|
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
|
|
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
|
|
; CHECK-NEXT: ret i64 [[SPEC_SELECT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop: ; preds = %entry, %loop
|
|
%iv = phi i64 [ 19999, %entry ], [ %iv.next, %loop ]
|
|
%rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ]
|
|
%gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv
|
|
%ld.a = load i64, ptr %gep.a.iv, align 8
|
|
%cmp.a.3 = icmp sgt i64 %ld.a, 3
|
|
%spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx
|
|
%iv.next = add nsw i64 %iv, -1
|
|
%exit.cond = icmp eq i64 %iv, 0
|
|
br i1 %exit.cond, label %exit, label %loop
|
|
|
|
exit: ; preds = %loop
|
|
ret i64 %spec.select
|
|
}
|
|
|
|
@table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1
|
|
|
|
define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
|
|
; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
|
|
; CHECK-SAME: i16 noundef [[VAL:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
|
|
; CHECK-NEXT: [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
|
|
; CHECK-NEXT: [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1
|
|
; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
|
|
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
|
|
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
|
|
; CHECK-NEXT: ret i16 [[SPEC_SELECT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop: ; preds = %entry, %loop
|
|
%iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ]
|
|
%rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ]
|
|
%gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv
|
|
%ld.table = load i16, ptr %gep.table.iv, align 1
|
|
%cmp.table.val = icmp ugt i16 %ld.table, %val
|
|
%iv.next = add nsw i16 %iv, -1
|
|
%spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx
|
|
%exit.cond = icmp eq i16 %iv.next, 0
|
|
br i1 %exit.cond, label %exit, label %loop
|
|
|
|
exit: ; preds = %loop
|
|
%spec.select.lcssa = phi i16 [ %spec.select, %loop ]
|
|
ret i16 %spec.select.lcssa
|
|
}
|
|
|
|
@tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1
|
|
|
|
define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
|
|
; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
|
|
; CHECK-SAME: half noundef [[VAL:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
|
|
; CHECK-NEXT: [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
|
|
; CHECK-NEXT: [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1
|
|
; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
|
|
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
|
|
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
|
|
; CHECK-NEXT: ret i16 [[SPEC_SELECT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop: ; preds = %entry, %loop
|
|
%iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ]
|
|
%rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ]
|
|
%gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv
|
|
%ld.table = load half, ptr %gep.table.iv, align 1
|
|
%cmp.table.val = fcmp ugt half %ld.table, %val
|
|
%iv.next = add nsw i16 %iv, -1
|
|
%spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx
|
|
%exit.cond = icmp eq i16 %iv.next, 0
|
|
br i1 %exit.cond, label %exit, label %loop
|
|
|
|
exit: ; preds = %loop
|
|
%spec.select.lcssa = phi i16 [ %spec.select, %loop ]
|
|
ret i16 %spec.select.lcssa
|
|
}
|
|
|
|
define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
|
|
; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[LOOP]] ], [ [[RDX_START]], %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
|
|
; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_NEXT]]
|
|
; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
|
|
; CHECK-NEXT: [[GEP_B_IV:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV_NEXT]]
|
|
; CHECK-NEXT: [[LD_B:%.*]] = load i64, ptr [[GEP_B_IV]], align 8
|
|
; CHECK-NEXT: [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]]
|
|
; CHECK-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[IV_NEXT]], i64 [[RDX]]
|
|
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1
|
|
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT:.*]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ]
|
|
; CHECK-NEXT: ret i64 [[COND_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop: ; preds = %entry, %loop
|
|
%iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ]
|
|
%rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ]
|
|
%iv.next = add nsw i64 %iv, -1
|
|
%gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv.next
|
|
%ld.a = load i64, ptr %gep.a.iv, align 8
|
|
%gep.b.iv = getelementptr inbounds i64, ptr %b, i64 %iv.next
|
|
%ld.b = load i64, ptr %gep.b.iv, align 8
|
|
%cmp.a.b = icmp sgt i64 %ld.a, %ld.b
|
|
%cond = select i1 %cmp.a.b, i64 %iv.next, i64 %rdx
|
|
%exit.cond = icmp ugt i64 %iv, 1
|
|
br i1 %exit.cond, label %loop, label %exit
|
|
|
|
exit: ; preds = %loop
|
|
ret i64 %cond
|
|
}
|
|
|
|
; The sentinel value for decreasing-IV vectorization is LONG_MAX, and since
|
|
; the IV hits this value, it is impossible to vectorize this case.
|
|
define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(ptr %a) {
|
|
; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(
|
|
; CHECK-SAME: ptr [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
|
|
; CHECK-NEXT: [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
|
|
; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
|
|
; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
|
|
; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
|
|
; CHECK-NEXT: ret i64 [[SPEC_SELECT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop: ; preds = %entry, %loop
|
|
%iv = phi i64 [ 9223372036854775807, %entry ], [ %iv.next, %loop ]
|
|
%rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ]
|
|
%gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv
|
|
%ld.a = load i64, ptr %gep.a.iv, align 8
|
|
%cmp.a.3 = icmp sgt i64 %ld.a, 3
|
|
%spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx
|
|
%iv.next = add nsw i64 %iv, -1
|
|
%exit.cond = icmp eq i64 %iv, 0
|
|
br i1 %exit.cond, label %exit, label %loop
|
|
|
|
exit: ; preds = %loop
|
|
ret i64 %spec.select
|
|
}
|
|
|