90d09eb300
Support for peeling with multiple exit blocks was added in D63921/77bb3a486fa6.
So far it has only been enabled for loops where all non-latch exits are
'de-optimizing' exits (D63923). But peeling of multi-exit loops can be
highly beneficial in other cases too, like if all non-latch exiting
blocks are unreachable.
The motivating case are loops with runtime checks, like the C++ example
below. The main issue preventing vectorization is that the invariant
accesses to load the bounds of B is conditionally executed in the loop
and cannot be hoisted out. If we peel off the first iteration, they
become dereferenceable in the loop, because they must execute before the
loop is executed, as all non-latch exits are terminated with
unreachable. This subsequently allows hoisting the loads and runtime
checks out of the loop, allowing vectorization of the loop.
int sum(std::vector<int> *A, std::vector<int> *B, int N) {
int cost = 0;
for (int i = 0; i < N; ++i)
cost += A->at(i) + B->at(i);
return cost;
}
This gives a ~20-30% increase of score for Geekbench5/HDR on AArch64.
Note that this requires a follow-up improvement to the peeling cost
model to actually peel iterations off loops as above. I will share that
shortly.
Also, peeling of multi-exits might be beneficial for exit blocks with
other terminators, but I would like to keep the scope limited to known
high-reward cases for now.
I removed the option to disable peeling for multi-deopt exits because
the code is more general now. Alternatively, the option could also be
generalized, but I am not sure if there's much value in the option?
Reviewed By: reames
Differential Revision: https://reviews.llvm.org/D108108
56 lines
1.9 KiB
LLVM
56 lines
1.9 KiB
LLVM
; REQUIRES: asserts
|
|
; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime 2>&1 | FileCheck %s
|
|
; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,loop-unroll)' 2>&1 | FileCheck %s
|
|
|
|
; Regression test for setting the correct idom for exit blocks.
|
|
|
|
; CHECK: Loop Unroll: F[basic]
|
|
; CHECK: PEELING loop %for.body with iteration count 2!
|
|
|
|
define i32 @basic(i32* %p, i32 %k, i1 %c1, i1 %c2) #0 !prof !3 {
|
|
entry:
|
|
%cmp3 = icmp slt i32 0, %k
|
|
br i1 %cmp3, label %for.body.lr.ph, label %for.end
|
|
|
|
for.body.lr.ph: ; preds = %entry
|
|
br label %for.body
|
|
|
|
for.body: ; preds = %for.body.lr.ph, %for.body
|
|
%i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %latch ]
|
|
%p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %latch ]
|
|
%incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
|
|
store i32 %i.05, i32* %p.addr.04, align 4
|
|
%inc = add nsw i32 %i.05, 1
|
|
%cmp = icmp slt i32 %inc, %k
|
|
br i1 %c1, label %continue, label %to_side_exit
|
|
|
|
continue:
|
|
br i1 %c2, label %latch, label %side_exit, !prof !2
|
|
|
|
latch:
|
|
br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !1
|
|
|
|
for.cond.for.end_crit_edge: ; preds = %for.body
|
|
br label %for.end
|
|
|
|
to_side_exit:
|
|
br i1 %c2, label %continue, label %side_exit, !prof !2
|
|
|
|
|
|
for.end: ; preds = %for.cond.for.end_crit_edge, %entry
|
|
%res = phi i32 [ 0, %entry ], [ %inc, %for.cond.for.end_crit_edge ]
|
|
ret i32 %res
|
|
|
|
side_exit:
|
|
%rval = call i32(...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %inc) ]
|
|
ret i32 %rval
|
|
}
|
|
|
|
declare i32 @llvm.experimental.deoptimize.i32(...)
|
|
|
|
attributes #0 = { nounwind }
|
|
|
|
!1 = !{!"branch_weights", i32 1, i32 1}
|
|
!2 = !{!"branch_weights", i32 1, i32 0}
|
|
!3 = !{!"function_entry_count", i64 1}
|