65ca32b83c
This patch allows the formation of interleaved access groups in loops containing predicated blocks. However, the predicated accesses are prevented from forming groups. Differential Revision: https://reviews.llvm.org/D19694 llvm-svn: 275471
165 lines
5.3 KiB
LLVM
165 lines
5.3 KiB
LLVM
; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses -vectorize-num-stores-pred=1 -enable-cond-stores-vec < %s | FileCheck %s
|
|
|
|
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
|
|
%pair = type { i64, i64 }
|
|
|
|
; Ensure that we vectorize the interleaved load group even though the loop
|
|
; contains a conditional store. The store group contains gaps and is not
|
|
; vectorized.
|
|
;
|
|
; CHECK-LABEL: @interleaved_with_cond_store_0(
|
|
;
|
|
; CHECK: min.iters.checked
|
|
; CHECK: %n.mod.vf = and i64 %[[N:.+]], 1
|
|
; CHECK: %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
|
|
; CHECK: %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
|
|
; CHECK: %n.vec = sub i64 %[[N]], %[[R]]
|
|
;
|
|
; CHECK: vector.body:
|
|
; CHECK: %wide.vec = load <4 x i64>, <4 x i64>* %{{.*}}
|
|
; CHECK: %strided.vec = shufflevector <4 x i64> %wide.vec, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
|
|
; CHECK: store i64 %[[X1]], {{.*}}
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
|
|
; CHECK: store i64 %[[X2]], {{.*}}
|
|
|
|
define void @interleaved_with_cond_store_0(%pair *%p, i64 %x, i64 %n) {
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
|
|
%p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
|
|
%0 = load i64, i64* %p.1, align 8
|
|
%1 = icmp eq i64 %0, %x
|
|
br i1 %1, label %if.then, label %if.merge
|
|
|
|
if.then:
|
|
store i64 %0, i64* %p.1, align 8
|
|
br label %if.merge
|
|
|
|
if.merge:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%cond = icmp slt i64 %i.next, %n
|
|
br i1 %cond, label %for.body, label %for.end
|
|
|
|
for.end:
|
|
ret void
|
|
}
|
|
|
|
; Ensure that we don't form a single interleaved group for the two loads. The
|
|
; conditional store prevents the second load from being hoisted. The two load
|
|
; groups are separately vectorized. The store group contains gaps and is not
|
|
; vectorized.
|
|
;
|
|
; CHECK-LABEL: @interleaved_with_cond_store_1(
|
|
;
|
|
; CHECK: min.iters.checked
|
|
; CHECK: %n.mod.vf = and i64 %[[N:.+]], 1
|
|
; CHECK: %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
|
|
; CHECK: %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
|
|
; CHECK: %n.vec = sub i64 %[[N]], %[[R]]
|
|
;
|
|
; CHECK: vector.body:
|
|
; CHECK: %[[L1:.+]] = load <4 x i64>, <4 x i64>* %{{.*}}
|
|
; CHECK: %strided.vec = shufflevector <4 x i64> %[[L1]], <4 x i64> undef, <2 x i32> <i32 0, i32 2>
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
|
|
; CHECK: store i64 %[[X1]], {{.*}}
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
|
|
; CHECK: store i64 %[[X2]], {{.*}}
|
|
;
|
|
; CHECK: pred.store.continue
|
|
; CHECK: %[[L2:.+]] = load <4 x i64>, <4 x i64>* {{.*}}
|
|
; CHECK: %[[X3:.+]] = extractelement <4 x i64> %[[L2]], i32 0
|
|
; CHECK: store i64 %[[X3]], {{.*}}
|
|
; CHECK: %[[X4:.+]] = extractelement <4 x i64> %[[L2]], i32 2
|
|
; CHECK: store i64 %[[X4]], {{.*}}
|
|
|
|
define void @interleaved_with_cond_store_1(%pair *%p, i64 %x, i64 %n) {
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
|
|
%p.0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
|
|
%p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
|
|
%0 = load i64, i64* %p.1, align 8
|
|
%1 = icmp eq i64 %0, %x
|
|
br i1 %1, label %if.then, label %if.merge
|
|
|
|
if.then:
|
|
store i64 %0, i64* %p.0, align 8
|
|
br label %if.merge
|
|
|
|
if.merge:
|
|
%2 = load i64, i64* %p.0, align 8
|
|
store i64 %2, i64 *%p.1, align 8
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%cond = icmp slt i64 %i.next, %n
|
|
br i1 %cond, label %for.body, label %for.end
|
|
|
|
for.end:
|
|
ret void
|
|
}
|
|
|
|
; Ensure that we don't create a single interleaved group for the two stores.
|
|
; The second store is conditional and we can't sink the first store inside the
|
|
; predicated block. The load group is vectorized, and the store groups contain
|
|
; gaps and are not vectorized.
|
|
;
|
|
; CHECK-LABEL: @interleaved_with_cond_store_2(
|
|
;
|
|
; CHECK: min.iters.checked
|
|
; CHECK: %n.mod.vf = and i64 %[[N:.+]], 1
|
|
; CHECK: %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
|
|
; CHECK: %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
|
|
; CHECK: %n.vec = sub i64 %[[N]], %[[R]]
|
|
;
|
|
; CHECK: vector.body:
|
|
; CHECK: %[[L1:.+]] = load <4 x i64>, <4 x i64>* %{{.*}}
|
|
; CHECK: %strided.vec = shufflevector <4 x i64> %[[L1]], <4 x i64> undef, <2 x i32> <i32 0, i32 2>
|
|
; CHECK: store i64 %x, {{.*}}
|
|
; CHECK: store i64 %x, {{.*}}
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
|
|
; CHECK: store i64 %[[X1]], {{.*}}
|
|
;
|
|
; CHECK: pred.store.if
|
|
; CHECK: %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
|
|
; CHECK: store i64 %[[X2]], {{.*}}
|
|
|
|
define void @interleaved_with_cond_store_2(%pair *%p, i64 %x, i64 %n) {
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
|
|
%p.0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
|
|
%p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
|
|
%0 = load i64, i64* %p.1, align 8
|
|
store i64 %x, i64* %p.0, align 8
|
|
%1 = icmp eq i64 %0, %x
|
|
br i1 %1, label %if.then, label %if.merge
|
|
|
|
if.then:
|
|
store i64 %0, i64* %p.1, align 8
|
|
br label %if.merge
|
|
|
|
if.merge:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%cond = icmp slt i64 %i.next, %n
|
|
br i1 %cond, label %for.body, label %for.end
|
|
|
|
for.end:
|
|
ret void
|
|
}
|