dd191d3d4f
Mechanism to keep the compiler-rt and llvm view of `FunctionData` in sync. Since CtxInstrContextNode.h is exactly the same on both sides (there's an existing test, `compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test`, checking that), we capture the structure in a macro that is then generated as `struct` fields on the compiler-rt side, and as `Type` objects on the llvm side. The macro needs to be told how to render a few kinds of fields. If we add more fields to FunctionData that can be described by the current known types of fields, then the llvm side would automatically be updated. If we need to add more kinds of fields, which we do by adding parameters to the macro, the llvm side (if not updated) would trigger a compilation error.
383 lines
16 KiB
C++
383 lines
16 KiB
C++
//===- PGOCtxProfLowering.cpp - Contextual PGO Instr. Lowering ------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
|
|
#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
|
|
#include "llvm/Analysis/CtxProfAnalysis.h"
|
|
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
|
|
#include "llvm/IR/Analysis.h"
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
|
#include "llvm/IR/GlobalValue.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/Instructions.h"
|
|
#include "llvm/IR/IntrinsicInst.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/IR/PassManager.h"
|
|
#include "llvm/ProfileData/CtxInstrContextNode.h"
|
|
#include "llvm/ProfileData/InstrProf.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include <utility>
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "ctx-instr-lower"
|
|
|
|
static cl::list<std::string> ContextRoots(
|
|
"profile-context-root", cl::Hidden,
|
|
cl::desc(
|
|
"A function name, assumed to be global, which will be treated as the "
|
|
"root of an interesting graph, which will be profiled independently "
|
|
"from other similar graphs."));
|
|
|
|
bool PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled() {
|
|
return !ContextRoots.empty();
|
|
}
|
|
|
|
// the names of symbols we expect in compiler-rt. Using a namespace for
|
|
// readability.
|
|
namespace CompilerRtAPINames {
|
|
static auto StartCtx = "__llvm_ctx_profile_start_context";
|
|
static auto ReleaseCtx = "__llvm_ctx_profile_release_context";
|
|
static auto GetCtx = "__llvm_ctx_profile_get_context";
|
|
static auto ExpectedCalleeTLS = "__llvm_ctx_profile_expected_callee";
|
|
static auto CallsiteTLS = "__llvm_ctx_profile_callsite";
|
|
} // namespace CompilerRtAPINames
|
|
|
|
namespace {
|
|
// The lowering logic and state.
|
|
class CtxInstrumentationLowerer final {
|
|
Module &M;
|
|
ModuleAnalysisManager &MAM;
|
|
Type *ContextNodeTy = nullptr;
|
|
Type *FunctionDataTy = nullptr;
|
|
|
|
DenseSet<const Function *> ContextRootSet;
|
|
Function *StartCtx = nullptr;
|
|
Function *GetCtx = nullptr;
|
|
Function *ReleaseCtx = nullptr;
|
|
GlobalVariable *ExpectedCalleeTLS = nullptr;
|
|
GlobalVariable *CallsiteInfoTLS = nullptr;
|
|
|
|
public:
|
|
CtxInstrumentationLowerer(Module &M, ModuleAnalysisManager &MAM);
|
|
// return true if lowering happened (i.e. a change was made)
|
|
bool lowerFunction(Function &F);
|
|
};
|
|
|
|
// llvm.instrprof.increment[.step] captures the total number of counters as one
|
|
// of its parameters, and llvm.instrprof.callsite captures the total number of
|
|
// callsites. Those values are the same for instances of those intrinsics in
|
|
// this function. Find the first instance of each and return them.
|
|
std::pair<uint32_t, uint32_t> getNumCountersAndCallsites(const Function &F) {
|
|
uint32_t NumCounters = 0;
|
|
uint32_t NumCallsites = 0;
|
|
for (const auto &BB : F) {
|
|
for (const auto &I : BB) {
|
|
if (const auto *Incr = dyn_cast<InstrProfIncrementInst>(&I)) {
|
|
uint32_t V =
|
|
static_cast<uint32_t>(Incr->getNumCounters()->getZExtValue());
|
|
assert((!NumCounters || V == NumCounters) &&
|
|
"expected all llvm.instrprof.increment[.step] intrinsics to "
|
|
"have the same total nr of counters parameter");
|
|
NumCounters = V;
|
|
} else if (const auto *CSIntr = dyn_cast<InstrProfCallsite>(&I)) {
|
|
uint32_t V =
|
|
static_cast<uint32_t>(CSIntr->getNumCounters()->getZExtValue());
|
|
assert((!NumCallsites || V == NumCallsites) &&
|
|
"expected all llvm.instrprof.callsite intrinsics to have the "
|
|
"same total nr of callsites parameter");
|
|
NumCallsites = V;
|
|
}
|
|
#if NDEBUG
|
|
if (NumCounters && NumCallsites)
|
|
return std::make_pair(NumCounters, NumCallsites);
|
|
#endif
|
|
}
|
|
}
|
|
return {NumCounters, NumCallsites};
|
|
}
|
|
} // namespace
|
|
|
|
// set up tie-in with compiler-rt.
|
|
// NOTE!!!
|
|
// These have to match compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
|
|
CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,
|
|
ModuleAnalysisManager &MAM)
|
|
: M(M), MAM(MAM) {
|
|
auto *PointerTy = PointerType::get(M.getContext(), 0);
|
|
auto *SanitizerMutexType = Type::getInt8Ty(M.getContext());
|
|
auto *I32Ty = Type::getInt32Ty(M.getContext());
|
|
auto *I64Ty = Type::getInt64Ty(M.getContext());
|
|
|
|
#define _PTRDECL(_, __) PointerTy,
|
|
#define _VOLATILE_PTRDECL(_, __) PointerTy,
|
|
#define _MUTEXDECL(_) SanitizerMutexType,
|
|
|
|
FunctionDataTy = StructType::get(
|
|
M.getContext(),
|
|
{CTXPROF_FUNCTION_DATA(_PTRDECL, _VOLATILE_PTRDECL, _MUTEXDECL)});
|
|
#undef _PTRDECL
|
|
#undef _VOLATILE_PTRDECL
|
|
#undef _MUTEXDECL
|
|
|
|
// The Context header.
|
|
ContextNodeTy = StructType::get(M.getContext(), {
|
|
I64Ty, /*Guid*/
|
|
PointerTy, /*Next*/
|
|
I32Ty, /*NumCounters*/
|
|
I32Ty, /*NumCallsites*/
|
|
});
|
|
|
|
// Define a global for each entrypoint. We'll reuse the entrypoint's name as
|
|
// prefix. We assume the entrypoint names to be unique.
|
|
for (const auto &Fname : ContextRoots) {
|
|
if (const auto *F = M.getFunction(Fname)) {
|
|
if (F->isDeclaration())
|
|
continue;
|
|
ContextRootSet.insert(F);
|
|
for (const auto &BB : *F)
|
|
for (const auto &I : BB)
|
|
if (const auto *CB = dyn_cast<CallBase>(&I))
|
|
if (CB->isMustTailCall()) {
|
|
M.getContext().emitError(
|
|
"The function " + Fname +
|
|
" was indicated as a context root, but it features musttail "
|
|
"calls, which is not supported.");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Declare the functions we will call.
|
|
StartCtx = cast<Function>(
|
|
M.getOrInsertFunction(
|
|
CompilerRtAPINames::StartCtx,
|
|
FunctionType::get(PointerTy,
|
|
{PointerTy, /*FunctionData*/
|
|
I64Ty, /*Guid*/ I32Ty,
|
|
/*NumCounters*/ I32Ty /*NumCallsites*/},
|
|
false))
|
|
.getCallee());
|
|
GetCtx = cast<Function>(
|
|
M.getOrInsertFunction(CompilerRtAPINames::GetCtx,
|
|
FunctionType::get(PointerTy,
|
|
{PointerTy, /*FunctionData*/
|
|
PointerTy, /*Callee*/
|
|
I64Ty, /*Guid*/
|
|
I32Ty, /*NumCounters*/
|
|
I32Ty}, /*NumCallsites*/
|
|
false))
|
|
.getCallee());
|
|
ReleaseCtx = cast<Function>(
|
|
M.getOrInsertFunction(CompilerRtAPINames::ReleaseCtx,
|
|
FunctionType::get(Type::getVoidTy(M.getContext()),
|
|
{
|
|
PointerTy, /*FunctionData*/
|
|
},
|
|
false))
|
|
.getCallee());
|
|
|
|
// Declare the TLSes we will need to use.
|
|
CallsiteInfoTLS =
|
|
new GlobalVariable(M, PointerTy, false, GlobalValue::ExternalLinkage,
|
|
nullptr, CompilerRtAPINames::CallsiteTLS);
|
|
CallsiteInfoTLS->setThreadLocal(true);
|
|
CallsiteInfoTLS->setVisibility(llvm::GlobalValue::HiddenVisibility);
|
|
ExpectedCalleeTLS =
|
|
new GlobalVariable(M, PointerTy, false, GlobalValue::ExternalLinkage,
|
|
nullptr, CompilerRtAPINames::ExpectedCalleeTLS);
|
|
ExpectedCalleeTLS->setThreadLocal(true);
|
|
ExpectedCalleeTLS->setVisibility(llvm::GlobalValue::HiddenVisibility);
|
|
}
|
|
|
|
PreservedAnalyses PGOCtxProfLoweringPass::run(Module &M,
|
|
ModuleAnalysisManager &MAM) {
|
|
CtxInstrumentationLowerer Lowerer(M, MAM);
|
|
bool Changed = false;
|
|
for (auto &F : M)
|
|
Changed |= Lowerer.lowerFunction(F);
|
|
return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
|
|
}
|
|
|
|
bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
|
|
if (F.isDeclaration())
|
|
return false;
|
|
auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
|
|
auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
|
|
|
|
Value *Guid = nullptr;
|
|
auto [NumCounters, NumCallsites] = getNumCountersAndCallsites(F);
|
|
|
|
Value *Context = nullptr;
|
|
Value *RealContext = nullptr;
|
|
|
|
StructType *ThisContextType = nullptr;
|
|
Value *TheRootFuctionData = nullptr;
|
|
Value *ExpectedCalleeTLSAddr = nullptr;
|
|
Value *CallsiteInfoTLSAddr = nullptr;
|
|
|
|
auto &Head = F.getEntryBlock();
|
|
for (auto &I : Head) {
|
|
// Find the increment intrinsic in the entry basic block.
|
|
if (auto *Mark = dyn_cast<InstrProfIncrementInst>(&I)) {
|
|
assert(Mark->getIndex()->isZero());
|
|
|
|
IRBuilder<> Builder(Mark);
|
|
Guid = Builder.getInt64(
|
|
AssignGUIDPass::getGUID(cast<Function>(*Mark->getNameValue())));
|
|
// The type of the context of this function is now knowable since we have
|
|
// NumCallsites and NumCounters. We delcare it here because it's more
|
|
// convenient - we have the Builder.
|
|
ThisContextType = StructType::get(
|
|
F.getContext(),
|
|
{ContextNodeTy, ArrayType::get(Builder.getInt64Ty(), NumCounters),
|
|
ArrayType::get(Builder.getPtrTy(), NumCallsites)});
|
|
// Figure out which way we obtain the context object for this function -
|
|
// if it's an entrypoint, then we call StartCtx, otherwise GetCtx. In the
|
|
// former case, we also set TheRootFuctionData since we need to release it
|
|
// at the end (plus it can be used to know if we have an entrypoint or a
|
|
// regular function)
|
|
// Don't set a name, they end up taking a lot of space and we don't need
|
|
// them.
|
|
auto *FData = new GlobalVariable(M, FunctionDataTy, false,
|
|
GlobalVariable::InternalLinkage,
|
|
Constant::getNullValue(FunctionDataTy));
|
|
|
|
if (ContextRootSet.contains(&F)) {
|
|
Context = Builder.CreateCall(
|
|
StartCtx, {FData, Guid, Builder.getInt32(NumCounters),
|
|
Builder.getInt32(NumCallsites)});
|
|
TheRootFuctionData = FData;
|
|
ORE.emit(
|
|
[&] { return OptimizationRemark(DEBUG_TYPE, "Entrypoint", &F); });
|
|
} else {
|
|
Context = Builder.CreateCall(GetCtx, {FData, &F, Guid,
|
|
Builder.getInt32(NumCounters),
|
|
Builder.getInt32(NumCallsites)});
|
|
ORE.emit([&] {
|
|
return OptimizationRemark(DEBUG_TYPE, "RegularFunction", &F);
|
|
});
|
|
}
|
|
// The context could be scratch.
|
|
auto *CtxAsInt = Builder.CreatePtrToInt(Context, Builder.getInt64Ty());
|
|
if (NumCallsites > 0) {
|
|
// Figure out which index of the TLS 2-element buffers to use.
|
|
// Scratch context => we use index == 1. Real contexts => index == 0.
|
|
auto *Index = Builder.CreateAnd(CtxAsInt, Builder.getInt64(1));
|
|
// The GEPs corresponding to that index, in the respective TLS.
|
|
ExpectedCalleeTLSAddr = Builder.CreateGEP(
|
|
PointerType::getUnqual(F.getContext()),
|
|
Builder.CreateThreadLocalAddress(ExpectedCalleeTLS), {Index});
|
|
CallsiteInfoTLSAddr = Builder.CreateGEP(
|
|
Builder.getInt32Ty(),
|
|
Builder.CreateThreadLocalAddress(CallsiteInfoTLS), {Index});
|
|
}
|
|
// Because the context pointer may have LSB set (to indicate scratch),
|
|
// clear it for the value we use as base address for the counter vector.
|
|
// This way, if later we want to have "real" (not clobbered) buffers
|
|
// acting as scratch, the lowering (at least this part of it that deals
|
|
// with counters) stays the same.
|
|
RealContext = Builder.CreateIntToPtr(
|
|
Builder.CreateAnd(CtxAsInt, Builder.getInt64(-2)),
|
|
PointerType::getUnqual(F.getContext()));
|
|
I.eraseFromParent();
|
|
break;
|
|
}
|
|
}
|
|
if (!Context) {
|
|
ORE.emit([&] {
|
|
return OptimizationRemarkMissed(DEBUG_TYPE, "Skip", &F)
|
|
<< "Function doesn't have instrumentation, skipping";
|
|
});
|
|
return false;
|
|
}
|
|
|
|
bool ContextWasReleased = false;
|
|
for (auto &BB : F) {
|
|
for (auto &I : llvm::make_early_inc_range(BB)) {
|
|
if (auto *Instr = dyn_cast<InstrProfCntrInstBase>(&I)) {
|
|
IRBuilder<> Builder(Instr);
|
|
switch (Instr->getIntrinsicID()) {
|
|
case llvm::Intrinsic::instrprof_increment:
|
|
case llvm::Intrinsic::instrprof_increment_step: {
|
|
// Increments (or increment-steps) are just a typical load - increment
|
|
// - store in the RealContext.
|
|
auto *AsStep = cast<InstrProfIncrementInst>(Instr);
|
|
auto *GEP = Builder.CreateGEP(
|
|
ThisContextType, RealContext,
|
|
{Builder.getInt32(0), Builder.getInt32(1), AsStep->getIndex()});
|
|
Builder.CreateStore(
|
|
Builder.CreateAdd(Builder.CreateLoad(Builder.getInt64Ty(), GEP),
|
|
AsStep->getStep()),
|
|
GEP);
|
|
} break;
|
|
case llvm::Intrinsic::instrprof_callsite:
|
|
// callsite lowering: write the called value in the expected callee
|
|
// TLS we treat the TLS as volatile because of signal handlers and to
|
|
// avoid these being moved away from the callsite they decorate.
|
|
auto *CSIntrinsic = dyn_cast<InstrProfCallsite>(Instr);
|
|
Builder.CreateStore(CSIntrinsic->getCallee(), ExpectedCalleeTLSAddr,
|
|
true);
|
|
// write the GEP of the slot in the sub-contexts portion of the
|
|
// context in TLS. Now, here, we use the actual Context value - as
|
|
// returned from compiler-rt - which may have the LSB set if the
|
|
// Context was scratch. Since the header of the context object and
|
|
// then the values are all 8-aligned (or, really, insofar as we care,
|
|
// they are even) - if the context is scratch (meaning, an odd value),
|
|
// so will the GEP. This is important because this is then visible to
|
|
// compiler-rt which will produce scratch contexts for callers that
|
|
// have a scratch context.
|
|
Builder.CreateStore(
|
|
Builder.CreateGEP(ThisContextType, Context,
|
|
{Builder.getInt32(0), Builder.getInt32(2),
|
|
CSIntrinsic->getIndex()}),
|
|
CallsiteInfoTLSAddr, true);
|
|
break;
|
|
}
|
|
I.eraseFromParent();
|
|
} else if (TheRootFuctionData && isa<ReturnInst>(I)) {
|
|
// Remember to release the context if we are an entrypoint.
|
|
IRBuilder<> Builder(&I);
|
|
Builder.CreateCall(ReleaseCtx, {TheRootFuctionData});
|
|
ContextWasReleased = true;
|
|
}
|
|
}
|
|
}
|
|
// FIXME: This would happen if the entrypoint tailcalls. A way to fix would be
|
|
// to disallow this, (so this then stays as an error), another is to detect
|
|
// that and then do a wrapper or disallow the tail call. This only affects
|
|
// instrumentation, when we want to detect the call graph.
|
|
if (TheRootFuctionData && !ContextWasReleased)
|
|
F.getContext().emitError(
|
|
"[ctx_prof] An entrypoint was instrumented but it has no `ret` "
|
|
"instructions above which to release the context: " +
|
|
F.getName());
|
|
return true;
|
|
}
|
|
|
|
PreservedAnalyses NoinlineNonPrevailing::run(Module &M,
|
|
ModuleAnalysisManager &MAM) {
|
|
bool Changed = false;
|
|
for (auto &F : M) {
|
|
if (F.isDeclaration())
|
|
continue;
|
|
if (F.hasFnAttribute(Attribute::NoInline))
|
|
continue;
|
|
if (!F.isWeakForLinker())
|
|
continue;
|
|
|
|
if (F.hasFnAttribute(Attribute::AlwaysInline))
|
|
F.removeFnAttr(Attribute::AlwaysInline);
|
|
|
|
F.addFnAttr(Attribute::NoInline);
|
|
Changed = true;
|
|
}
|
|
if (Changed)
|
|
return PreservedAnalyses::none();
|
|
return PreservedAnalyses::all();
|
|
}
|