The following bpf linux kernel selftest failed with latest llvm: $ ./test_progs -n 7/10 ... The sequence of 8193 jumps is too complex. verification time 126272 usec stack depth 320 processed 114799 insns (limit 1000000) ... libbpf: failed to load object 'pyperf600_nounroll.o' test_bpf_verif_scale:FAIL:110 #7/10 pyperf600_nounroll.o:FAIL #7 bpf_verif_scale:FAIL After some investigation, I found the following llvm patch https://reviews.llvm.org/D84108 is responsible. The patch disabled hoisting common instructions in SimplifyCFG by default. Later on, the code changes and a SimplifyCFG phase with hoisting on cannot do the work any more. A test is provided to demonstrate the problem. The IR before simplifyCFG looks like: for.cond: %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] %cmp = icmp ult i32 %i.0, 6 br i1 %cmp, label %for.body, label %for.cond.cleanup for.cond.cleanup: %2 = load i8*, i8** %frame_ptr, align 8, !tbaa !2 %cmp2 = icmp eq i8* %2, null %conv = zext i1 %cmp2 to i32 call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %1) #3 call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0) #3 ret i32 %conv for.body: %3 = load i8*, i8** %frame_ptr, align 8, !tbaa !2 %tobool.not = icmp eq i8* %3, null br i1 %tobool.not, label %for.inc, label %land.lhs.true The first two insns of `for.cond.cleanup` and `for.body`, load and icmp, can be hoisted to `for.cond` block. With Patch D84108, the optimization is delayed. But unfortunately, later on loop rotation added addition phi nodes to `for.body` and hoisting cannot be done any more. Note such a hoisting is beneficial to bpf programs as bpf verifier does path sensitive analysis and verification. The hoisting preverts reloading from stack which will assume conservative value and increase exploited insns. In this case, it caused verifier failure. To fix this problem, I added an IR pass from bpf target to performance additional simplifycfg with hoisting common inst enabled. Differential Revision: https://reviews.llvm.org/D85434
145 lines
4.9 KiB
C++
145 lines
4.9 KiB
C++
//===-- BPFTargetMachine.cpp - Define TargetMachine for BPF ---------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Implements the info about BPF target spec.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BPFTargetMachine.h"
|
|
#include "BPF.h"
|
|
#include "MCTargetDesc/BPFMCAsmInfo.h"
|
|
#include "TargetInfo/BPFTargetInfo.h"
|
|
#include "llvm/CodeGen/Passes.h"
|
|
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
|
|
#include "llvm/CodeGen/TargetPassConfig.h"
|
|
#include "llvm/IR/LegacyPassManager.h"
|
|
#include "llvm/Support/FormattedStream.h"
|
|
#include "llvm/Support/TargetRegistry.h"
|
|
#include "llvm/Target/TargetOptions.h"
|
|
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
|
|
#include "llvm/Transforms/Scalar.h"
|
|
#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
|
|
using namespace llvm;
|
|
|
|
static cl::
|
|
opt<bool> DisableMIPeephole("disable-bpf-peephole", cl::Hidden,
|
|
cl::desc("Disable machine peepholes for BPF"));
|
|
|
|
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() {
|
|
// Register the target.
|
|
RegisterTargetMachine<BPFTargetMachine> X(getTheBPFleTarget());
|
|
RegisterTargetMachine<BPFTargetMachine> Y(getTheBPFbeTarget());
|
|
RegisterTargetMachine<BPFTargetMachine> Z(getTheBPFTarget());
|
|
|
|
PassRegistry &PR = *PassRegistry::getPassRegistry();
|
|
initializeBPFAbstractMemberAccessPass(PR);
|
|
initializeBPFPreserveDITypePass(PR);
|
|
initializeBPFMIPeepholePass(PR);
|
|
initializeBPFMIPeepholeTruncElimPass(PR);
|
|
}
|
|
|
|
// DataLayout: little or big endian
|
|
static std::string computeDataLayout(const Triple &TT) {
|
|
if (TT.getArch() == Triple::bpfeb)
|
|
return "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
|
|
else
|
|
return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
|
|
}
|
|
|
|
static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
|
|
if (!RM.hasValue())
|
|
return Reloc::PIC_;
|
|
return *RM;
|
|
}
|
|
|
|
BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
|
|
StringRef CPU, StringRef FS,
|
|
const TargetOptions &Options,
|
|
Optional<Reloc::Model> RM,
|
|
Optional<CodeModel::Model> CM,
|
|
CodeGenOpt::Level OL, bool JIT)
|
|
: LLVMTargetMachine(T, computeDataLayout(TT), TT, CPU, FS, Options,
|
|
getEffectiveRelocModel(RM),
|
|
getEffectiveCodeModel(CM, CodeModel::Small), OL),
|
|
TLOF(std::make_unique<TargetLoweringObjectFileELF>()),
|
|
Subtarget(TT, std::string(CPU), std::string(FS), *this) {
|
|
initAsmInfo();
|
|
|
|
BPFMCAsmInfo *MAI =
|
|
static_cast<BPFMCAsmInfo *>(const_cast<MCAsmInfo *>(AsmInfo.get()));
|
|
MAI->setDwarfUsesRelocationsAcrossSections(!Subtarget.getUseDwarfRIS());
|
|
}
|
|
|
|
namespace {
|
|
// BPF Code Generator Pass Configuration Options.
|
|
class BPFPassConfig : public TargetPassConfig {
|
|
public:
|
|
BPFPassConfig(BPFTargetMachine &TM, PassManagerBase &PM)
|
|
: TargetPassConfig(TM, PM) {}
|
|
|
|
BPFTargetMachine &getBPFTargetMachine() const {
|
|
return getTM<BPFTargetMachine>();
|
|
}
|
|
|
|
void addIRPasses() override;
|
|
bool addInstSelector() override;
|
|
void addMachineSSAOptimization() override;
|
|
void addPreEmitPass() override;
|
|
};
|
|
}
|
|
|
|
TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) {
|
|
return new BPFPassConfig(*this, PM);
|
|
}
|
|
|
|
void BPFTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
|
|
Builder.addExtension(
|
|
PassManagerBuilder::EP_Peephole,
|
|
[&](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
|
|
PM.add(createCFGSimplificationPass(
|
|
SimplifyCFGOptions().hoistCommonInsts(true)));
|
|
});
|
|
}
|
|
|
|
void BPFPassConfig::addIRPasses() {
|
|
addPass(createBPFAbstractMemberAccess(&getBPFTargetMachine()));
|
|
addPass(createBPFPreserveDIType());
|
|
|
|
TargetPassConfig::addIRPasses();
|
|
}
|
|
|
|
// Install an instruction selector pass using
|
|
// the ISelDag to gen BPF code.
|
|
bool BPFPassConfig::addInstSelector() {
|
|
addPass(createBPFISelDag(getBPFTargetMachine()));
|
|
|
|
return false;
|
|
}
|
|
|
|
void BPFPassConfig::addMachineSSAOptimization() {
|
|
addPass(createBPFMISimplifyPatchablePass());
|
|
|
|
// The default implementation must be called first as we want eBPF
|
|
// Peephole ran at last.
|
|
TargetPassConfig::addMachineSSAOptimization();
|
|
|
|
const BPFSubtarget *Subtarget = getBPFTargetMachine().getSubtargetImpl();
|
|
if (!DisableMIPeephole) {
|
|
if (Subtarget->getHasAlu32())
|
|
addPass(createBPFMIPeepholePass());
|
|
addPass(createBPFMIPeepholeTruncElimPass());
|
|
}
|
|
}
|
|
|
|
void BPFPassConfig::addPreEmitPass() {
|
|
addPass(createBPFMIPreEmitCheckingPass());
|
|
if (getOptLevel() != CodeGenOpt::None)
|
|
if (!DisableMIPeephole)
|
|
addPass(createBPFMIPreEmitPeepholePass());
|
|
}
|