a8c2ba105d
After the switch to the new pass manager, we have observed multiple instances of catastrophic inlining, where the inliner produces huge functions with many hundreds of thousands of instructions from small input IR. We were forced to back out the switch to the new pass manager for this reason. This patch fixes at least one of the root cause issues. LLVM uses a bottom-up inliner, and the fact that functions are processed bottom-up is not just a question of optimality -- it is an imporant requirement to prevent runaway inlining. The premise of the current inlining approach and cost model is that after all calls inside a function have been inlined, it may get large enough that inlining it into its callers is no longer considered profitable. This safeguard does not exist if inlining doesn't happen bottom-up, as inlining the callees, and their callees, and their callees etc. will always seem individually profitable, and the inliner can easily flatten the whole call tree. There are instances where we necessarily have to deviate from bottom-up inlining: When inlining in an SCC there is no natural "bottom", so inlining effectively happens top-down. This requires special care, and the inliner avoids exponential blowup by ensuring that functions in the SCC grow in a balanced way and will eventually hit the threshold. However, there is one instance where the inlining advisor explicitly violates the bottom-up principle: Deferred inlining tries to "defer" inlining a call if it determines that inlining the caller into all its call-sites would be more profitable. Something very important to understand about deferred inlining is that it doesn't make one inlining choice in place of another -- it effectively chooses to do both. If we have a call chain A -> B -> C and cost modelling tells us that inlining B -> C is profitable, but we defer this and instead inline A -> B first, then we'll now have a call A -> C, and the cost model will (a few special cases notwithstanding) still tell us that this is profitable. So the end result is that we inlined *both* B and C, even though under the usual cost model function B would have been too large to further inline after C has been integrated into it. Because deferred inlining violates the bottom-up invariant of the inliner, it can result in exponential inlining. The exponential-deferred-inlining.ll test case illustrates this on a simple example (see https://gist.github.com/nikic/1262b5f7d27278e1b34a190ae10947f5 for a much more catastrophic case with about 5000x size blowup). If the call chain A -> B -> C is not a chain but a tree of calls, then we end up deferring inlining across the tree and end up flattening everything into the root node. This patch proposes to address this by disabling deferred inlining entirely (currently still behind an option). Beyond the issue of exponential inlining, I don't think that the whole concept makes sense, at least as long as deferred inlining still ends up inlining both call edges. I believe the motivation for having deferred inlining in the first place is that you might have a small wrapper function with local linkage that could be eliminated if inlined. This would automatically happen if there was a single caller, due to the large "last call to local" bonus. However, this bonus is not extended if there are multiple callers, even if we would eventually end up inlining into all of them (if the bonus were extended). Now, unlike the normal inlining cost model, the deferred inlining cost model does look at all callers, and will extend the "last call to local" bonus if it determines that we could inline all of them as long as we defer the current inlining decision. This makes very little sense. The "last call to local" bonus doesn't really cost model anything. It's basically an "infinite" bonus that ensures we always inline the last call to a local. The fact that it's not literally infinite just prevents inlining of huge functions, which can easily result in scalability issues. I very much doubt that it was an intentional cost-modelling choice to say that getting rid of a small local function is worth adding 15000 instructions elsewhere, yet this is exactly how this value is getting used here. The main alternative I see to complete removal is to change deferred inlining to an actual either/or decision. That is, to mark deferred calls as noinline so we're actually trading off one inlining decision against another, and not just adding a side-channel to the cost model to do both. Apart from fixing the catastrophic inlining case, the effect on rustc is a modest compile-time improvement on average (up to 8% for a parsing-type crate, where tree-like calls are expected) and pretty neutral where run-time performance is concerned (mix of small wins and losses, usually in the sub-1% category). Differential Revision: https://reviews.llvm.org/D115497
196 lines
6.0 KiB
LLVM
196 lines
6.0 KiB
LLVM
; Test for a subtle bug when computing analyses during inlining and mutating
|
|
; the SCC structure. Without care, this can fail to invalidate analyses.
|
|
;
|
|
; RUN: opt < %s -aa-pipeline= -passes='cgscc(inline,function(verify<domtree>))' -debug-pass-manager -inline-deferral -S 2>&1 | FileCheck %s
|
|
|
|
; First we check that the passes run in the way we expect. Otherwise this test
|
|
; may stop testing anything.
|
|
;
|
|
; CHECK: Running pass: InlinerPass on (test1_f, test1_g, test1_h)
|
|
; CHECK: Running analysis: DominatorTreeAnalysis on test1_f
|
|
; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_f
|
|
; CHECK: Invalidating analysis: LoopAnalysis on test1_f
|
|
; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_f
|
|
; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_f
|
|
; CHECK: Running analysis: DominatorTreeAnalysis on test1_g
|
|
; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_g
|
|
; CHECK: Invalidating analysis: LoopAnalysis on test1_g
|
|
; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_g
|
|
; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_g
|
|
; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_h
|
|
; CHECK: Invalidating analysis: LoopAnalysis on test1_h
|
|
; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_h
|
|
; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_h
|
|
; CHECK-NOT: Invalidating analysis:
|
|
; CHECK: Running pass: DominatorTreeVerifierPass on test1_g
|
|
; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on test1_g
|
|
; CHECK-NOT: Invalidating analysis:
|
|
; CHECK: Running pass: DominatorTreeVerifierPass on test1_h
|
|
; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on test1_h
|
|
; CHECK-NOT: Invalidating analysis:
|
|
; CHECK: Running pass: DominatorTreeVerifierPass on test1_f
|
|
|
|
; An external function used to control branches.
|
|
declare i1 @flag()
|
|
; CHECK-LABEL: declare i1 @flag()
|
|
|
|
; The utility function with interesting control flow that gets inlined below to
|
|
; perturb the dominator tree.
|
|
define internal void @callee() {
|
|
entry:
|
|
%ptr = alloca i8
|
|
%flag = call i1 @flag()
|
|
br i1 %flag, label %then, label %else
|
|
|
|
then:
|
|
store volatile i8 42, i8* %ptr
|
|
br label %return
|
|
|
|
else:
|
|
store volatile i8 -42, i8* %ptr
|
|
br label %return
|
|
|
|
return:
|
|
ret void
|
|
}
|
|
|
|
; The 'test1_' prefixed functions work to carefully test that incrementally
|
|
; reducing an SCC in the inliner cannot accidentially leave stale function
|
|
; analysis results due to failing to invalidate them for all the functions.
|
|
|
|
; We visit this function first in the inliner, and while we inline callee
|
|
; perturbing the CFG, we don't inline anything else and the SCC structure
|
|
; remains in tact.
|
|
define void @test1_f() {
|
|
; CHECK-LABEL: define void @test1_f()
|
|
entry:
|
|
; We force this edge to survive inlining.
|
|
call void @test1_g() noinline
|
|
; CHECK: call void @test1_g()
|
|
|
|
; Pull interesting CFG into this function.
|
|
call void @callee()
|
|
; CHECK-NOT: call void @callee()
|
|
|
|
ret void
|
|
; CHECK: ret void
|
|
}
|
|
|
|
; We visit this function second and here we inline the edge to 'test1_f'
|
|
; separating it into its own SCC. The current SCC is now just 'test1_g' and
|
|
; 'test1_h'.
|
|
define void @test1_g() {
|
|
; CHECK-LABEL: define void @test1_g()
|
|
entry:
|
|
; This edge gets inlined away.
|
|
call void @test1_f()
|
|
; CHECK-NOT: call void @test1_f()
|
|
; CHECK: call void @test1_g()
|
|
|
|
; We force this edge to survive inlining.
|
|
call void @test1_h() noinline
|
|
; CHECK: call void @test1_h()
|
|
|
|
; Pull interesting CFG into this function.
|
|
call void @callee()
|
|
; CHECK-NOT: call void @callee()
|
|
|
|
ret void
|
|
; CHECK: ret void
|
|
}
|
|
|
|
; The inliner visits this last function. It can't actually break any cycles
|
|
; here, but because we visit this function we compute fresh analyses for it.
|
|
; These analyses are then invalidated when we inline callee disrupting the
|
|
; CFG, and it is important that they be freed.
|
|
define void @test1_h() {
|
|
; CHECK-LABEL: define void @test1_h()
|
|
entry:
|
|
call void @test1_g()
|
|
; CHECK: call void @test1_g()
|
|
|
|
; Pull interesting CFG into this function.
|
|
call void @callee()
|
|
; CHECK-NOT: call void @callee()
|
|
|
|
ret void
|
|
; CHECK: ret void
|
|
}
|
|
|
|
; The 'test2_' prefixed code works to carefully trigger forming an SCC with
|
|
; a dominator tree for one of the functions but not the other and without even
|
|
; a function analysis manager proxy for the SCC that things get merged into.
|
|
; Without proper handling when updating the call graph this will find a stale
|
|
; dominator tree.
|
|
|
|
@test2_global = external global i32, align 4
|
|
|
|
define void @test2_hoge(i1 (i32*)* %arg) {
|
|
; CHECK-LABEL: define void @test2_hoge(
|
|
bb:
|
|
%tmp2 = call zeroext i1 %arg(i32* @test2_global)
|
|
; CHECK: call zeroext i1 %arg(
|
|
br label %bb3
|
|
|
|
bb3:
|
|
%tmp5 = call zeroext i1 %arg(i32* @test2_global)
|
|
; CHECK: call zeroext i1 %arg(
|
|
br i1 %tmp5, label %bb3, label %bb6
|
|
|
|
bb6:
|
|
ret void
|
|
}
|
|
|
|
define zeroext i1 @test2_widget(i32* %arg) {
|
|
; CHECK-LABEL: define zeroext i1 @test2_widget(
|
|
bb:
|
|
%tmp1 = alloca i8, align 1
|
|
%tmp2 = alloca i32, align 4
|
|
call void @test2_quux()
|
|
; CHECK-NOT: call
|
|
;
|
|
; CHECK: call zeroext i1 @test2_widget(i32* @test2_global)
|
|
; CHECK-NEXT: br label %[[NEW_BB:.*]]
|
|
;
|
|
; CHECK: [[NEW_BB]]:
|
|
; CHECK-NEXT: call zeroext i1 @test2_widget(i32* @test2_global)
|
|
;
|
|
; CHECK: {{.*}}:
|
|
|
|
call void @test2_hoge.1(i32* %arg)
|
|
; CHECK-NEXT: call void @test2_hoge.1(
|
|
|
|
%tmp4 = call zeroext i1 @test2_barney(i32* %tmp2)
|
|
%tmp5 = zext i1 %tmp4 to i32
|
|
store i32 %tmp5, i32* %tmp2, align 4
|
|
%tmp6 = call zeroext i1 @test2_barney(i32* null)
|
|
call void @test2_ham(i8* %tmp1)
|
|
; CHECK: call void @test2_ham(
|
|
|
|
call void @test2_quux()
|
|
; CHECK-NOT: call
|
|
;
|
|
; CHECK: call zeroext i1 @test2_widget(i32* @test2_global)
|
|
; CHECK-NEXT: br label %[[NEW_BB:.*]]
|
|
;
|
|
; CHECK: [[NEW_BB]]:
|
|
; CHECK-NEXT: call zeroext i1 @test2_widget(i32* @test2_global)
|
|
;
|
|
; CHECK: {{.*}}:
|
|
ret i1 true
|
|
; CHECK-NEXT: ret i1 true
|
|
}
|
|
|
|
define internal void @test2_quux() {
|
|
; CHECK-NOT: @test2_quux
|
|
bb:
|
|
call void @test2_hoge(i1 (i32*)* @test2_widget)
|
|
ret void
|
|
}
|
|
|
|
declare void @test2_hoge.1(i32*)
|
|
|
|
declare zeroext i1 @test2_barney(i32*)
|
|
|
|
declare void @test2_ham(i8*)
|