1518b260ce
Clang's `-fwhole-program-vtables` is required for this optimization to
take place. If `-fwhole-program-vtables` is not enabled, this change is
no-op.
* Function-comparison (before):
```
%vtable = load ptr, ptr %obj
%vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
%func = load ptr, ptr %vfn
%cond = icmp eq ptr %func, @callee
br i1 %cond, label bb1, label bb2:
bb1:
call @callee
bb2:
call %func
```
* VTable-comparison (after):
```
%vtable = load ptr, ptr %obj
%cond = icmp eq ptr %vtable, @vtable-address-point
br i1 %cond, label bb1, label bb2:
bb1:
call @callee
bb2:
%vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
%func = load ptr, ptr %vfn
call %func
```
Key changes:
1. Find out virtual calls and the vtables they come from.
- The ICP relies on type intrinsic `llvm.type.test` to find out virtual
calls and the
compatible vtables, and relies on type metadata to find the address
point for comparison.
2. ICP pass does cost-benefit analysis and compares vtable only when the
number of vtables for a function candidate is within (option specified)
threshold.
3. Sink the function addressing and vtable load instruction to indirect
fallback.
- The sink helper functions are simplified versions of
`InstCombinerImpl::tryToSinkInstruction`. Currently debug intrinsics are
not handled. Ideally `InstCombinerImpl::tryToSinkInstructionDbgValues`
and `InstCombinerImpl::tryToSinkInstructionDbgVariableRecords` could be
moved into Transforms/Utils/Local.cpp (or another util cpp file) to
handle debug intrinsics when moving instructions across basic blocks.
4. Keep value profiles updated
1) Update vtable value profiles after inline
2) For either function-based comparison or vtable-based comparison,
update both vtable and indirect call value profiles.
104 lines
4.3 KiB
C++
104 lines
4.3 KiB
C++
//===-- IndirectCallPromotionAnalysis.cpp - Find promotion candidates ===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Helper methods for identifying profitable indirect call promotion
|
|
// candidates for an instruction when the indirect-call value profile metadata
|
|
// is available.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
|
|
#include "llvm/IR/Instruction.h"
|
|
#include "llvm/ProfileData/InstrProf.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Support/Debug.h"
|
|
#include <memory>
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "pgo-icall-prom-analysis"
|
|
|
|
// The percent threshold for the direct-call target (this call site vs the
|
|
// remaining call count) for it to be considered as the promotion target.
|
|
static cl::opt<unsigned> ICPRemainingPercentThreshold(
|
|
"icp-remaining-percent-threshold", cl::init(30), cl::Hidden,
|
|
cl::desc("The percentage threshold against remaining unpromoted indirect "
|
|
"call count for the promotion"));
|
|
|
|
// The percent threshold for the direct-call target (this call site vs the
|
|
// total call count) for it to be considered as the promotion target.
|
|
static cl::opt<unsigned>
|
|
ICPTotalPercentThreshold("icp-total-percent-threshold", cl::init(5),
|
|
cl::Hidden,
|
|
cl::desc("The percentage threshold against total "
|
|
"count for the promotion"));
|
|
|
|
// Set the maximum number of targets to promote for a single indirect-call
|
|
// callsite.
|
|
static cl::opt<unsigned>
|
|
MaxNumPromotions("icp-max-prom", cl::init(3), cl::Hidden,
|
|
cl::desc("Max number of promotions for a single indirect "
|
|
"call callsite"));
|
|
|
|
cl::opt<unsigned> MaxNumVTableAnnotations(
|
|
"icp-max-num-vtables", cl::init(6), cl::Hidden,
|
|
cl::desc("Max number of vtables annotated for a vtable load instruction."));
|
|
|
|
ICallPromotionAnalysis::ICallPromotionAnalysis() {
|
|
ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumPromotions);
|
|
}
|
|
|
|
bool ICallPromotionAnalysis::isPromotionProfitable(uint64_t Count,
|
|
uint64_t TotalCount,
|
|
uint64_t RemainingCount) {
|
|
return Count * 100 >= ICPRemainingPercentThreshold * RemainingCount &&
|
|
Count * 100 >= ICPTotalPercentThreshold * TotalCount;
|
|
}
|
|
|
|
// Indirect-call promotion heuristic. The direct targets are sorted based on
|
|
// the count. Stop at the first target that is not promoted. Returns the
|
|
// number of candidates deemed profitable.
|
|
uint32_t ICallPromotionAnalysis::getProfitablePromotionCandidates(
|
|
const Instruction *Inst, uint32_t NumVals, uint64_t TotalCount) {
|
|
ArrayRef<InstrProfValueData> ValueDataRef(ValueDataArray.get(), NumVals);
|
|
|
|
LLVM_DEBUG(dbgs() << " \nWork on callsite " << *Inst
|
|
<< " Num_targets: " << NumVals << "\n");
|
|
|
|
uint32_t I = 0;
|
|
uint64_t RemainingCount = TotalCount;
|
|
for (; I < MaxNumPromotions && I < NumVals; I++) {
|
|
uint64_t Count = ValueDataRef[I].Count;
|
|
assert(Count <= RemainingCount);
|
|
LLVM_DEBUG(dbgs() << " Candidate " << I << " Count=" << Count
|
|
<< " Target_func: " << ValueDataRef[I].Value << "\n");
|
|
|
|
if (!isPromotionProfitable(Count, TotalCount, RemainingCount)) {
|
|
LLVM_DEBUG(dbgs() << " Not promote: Cold target.\n");
|
|
return I;
|
|
}
|
|
RemainingCount -= Count;
|
|
}
|
|
return I;
|
|
}
|
|
|
|
MutableArrayRef<InstrProfValueData>
|
|
ICallPromotionAnalysis::getPromotionCandidatesForInstruction(
|
|
const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates) {
|
|
uint32_t NumVals;
|
|
auto Res = getValueProfDataFromInst(*I, IPVK_IndirectCallTarget,
|
|
MaxNumPromotions, NumVals, TotalCount);
|
|
if (!Res) {
|
|
NumCandidates = 0;
|
|
return MutableArrayRef<InstrProfValueData>();
|
|
}
|
|
ValueDataArray = std::move(Res);
|
|
NumCandidates = getProfitablePromotionCandidates(I, NumVals, TotalCount);
|
|
return MutableArrayRef<InstrProfValueData>(ValueDataArray.get(), NumVals);
|
|
}
|