d896b1f5a6
The string pooling pass was incorrectly pooling global varables that were part of llvm.used or llvm.compiler.used. This patch fixes the pass to prevent that by checking each candidate to make sure that it is not in either of those lists.
366 lines
13 KiB
C++
366 lines
13 KiB
C++
//===-- PPCMergeStringPool.cpp -------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This transformation tries to merge the strings in the module into one pool
|
|
// of strings. The idea is to reduce the number of TOC entries in the module so
|
|
// that instead of having one TOC entry for each string there is only one global
|
|
// TOC entry and all of the strings are referenced off of that one entry plus
|
|
// an offset.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "PPC.h"
|
|
#include "llvm/ADT/Statistic.h"
|
|
#include "llvm/Analysis/DomTreeUpdater.h"
|
|
#include "llvm/Analysis/LoopInfo.h"
|
|
#include "llvm/Analysis/LoopIterator.h"
|
|
#include "llvm/Analysis/ScalarEvolution.h"
|
|
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
|
|
#include "llvm/IR/Constants.h"
|
|
#include "llvm/IR/Instructions.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/IR/ValueSymbolTable.h"
|
|
#include "llvm/Pass.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
|
|
#define DEBUG_TYPE "ppc-merge-strings"
|
|
|
|
STATISTIC(NumPooledStrings, "Number of Strings Pooled");
|
|
|
|
using namespace llvm;
|
|
|
|
static cl::opt<unsigned>
|
|
MaxStringsPooled("ppc-max-strings-pooled", cl::Hidden, cl::init(-1),
|
|
cl::desc("Maximum Number of Strings to Pool."));
|
|
|
|
static cl::opt<unsigned>
|
|
MinStringsBeforePool("ppc-min-strings-before-pool", cl::Hidden, cl::init(2),
|
|
cl::desc("Minimum number of string candidates before "
|
|
"pooling is considered."));
|
|
|
|
namespace {
|
|
struct {
|
|
bool operator()(const GlobalVariable *LHS, const GlobalVariable *RHS) const {
|
|
// First priority is alignment.
|
|
// If elements are sorted in terms of alignment then there won't be an
|
|
// issue with incorrect alignment that would require padding.
|
|
Align LHSAlign = LHS->getAlign().valueOrOne();
|
|
Align RHSAlign = RHS->getAlign().valueOrOne();
|
|
if (LHSAlign > RHSAlign)
|
|
return true;
|
|
else if (LHSAlign < RHSAlign)
|
|
return false;
|
|
|
|
// Next priority is the number of uses.
|
|
// Smaller offsets are easier to materialize because materializing a large
|
|
// offset may require more than one instruction. (ie addis, addi).
|
|
if (LHS->getNumUses() > RHS->getNumUses())
|
|
return true;
|
|
else if (LHS->getNumUses() < RHS->getNumUses())
|
|
return false;
|
|
|
|
const Constant *ConstLHS = LHS->getInitializer();
|
|
const ConstantDataSequential *ConstDataLHS =
|
|
dyn_cast<ConstantDataSequential>(ConstLHS);
|
|
unsigned LHSSize =
|
|
ConstDataLHS->getNumElements() * ConstDataLHS->getElementByteSize();
|
|
const Constant *ConstRHS = RHS->getInitializer();
|
|
const ConstantDataSequential *ConstDataRHS =
|
|
dyn_cast<ConstantDataSequential>(ConstRHS);
|
|
unsigned RHSSize =
|
|
ConstDataRHS->getNumElements() * ConstDataRHS->getElementByteSize();
|
|
|
|
// Finally smaller constants should go first. This is, again, trying to
|
|
// minimize the offsets into the final struct.
|
|
return LHSSize < RHSSize;
|
|
}
|
|
} CompareConstants;
|
|
|
|
class PPCMergeStringPool : public ModulePass {
|
|
public:
|
|
static char ID;
|
|
PPCMergeStringPool() : ModulePass(ID) {}
|
|
|
|
bool runOnModule(Module &M) override { return mergeModuleStringPool(M); }
|
|
|
|
StringRef getPassName() const override { return "PPC Merge String Pool"; }
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
|
AU.addPreserved<LoopInfoWrapperPass>();
|
|
AU.addPreserved<ScalarEvolutionWrapperPass>();
|
|
AU.addPreserved<SCEVAAWrapperPass>();
|
|
}
|
|
|
|
private:
|
|
// Globals in a Module are already unique so a set is not required and a
|
|
// vector will do.
|
|
std::vector<GlobalVariable *> MergeableStrings;
|
|
Align MaxAlignment;
|
|
Type *PooledStructType;
|
|
LLVMContext *Context;
|
|
void collectCandidateConstants(Module &M);
|
|
bool mergeModuleStringPool(Module &M);
|
|
void replaceUsesWithGEP(GlobalVariable *GlobalToReplace, GlobalVariable *GPool,
|
|
unsigned ElementIndex);
|
|
};
|
|
|
|
|
|
// In order for a constant to be pooled we need to be able to replace all of
|
|
// the uses for that constant. This function checks all of the uses to make
|
|
// sure that they can be replaced.
|
|
static bool hasReplaceableUsers(GlobalVariable &GV) {
|
|
for (User *CurrentUser : GV.users()) {
|
|
// Instruction users are always valid.
|
|
if (isa<Instruction>(CurrentUser))
|
|
continue;
|
|
|
|
// We cannot replace GlobalValue users because they are not just nodes
|
|
// in IR. To replace a user like this we would need to create a new
|
|
// GlobalValue with the replacement and then try to delete the original
|
|
// GlobalValue. Deleting the original would only happen if it has no other
|
|
// uses.
|
|
if (isa<GlobalValue>(CurrentUser))
|
|
return false;
|
|
|
|
// We only support Instruction and Constant users.
|
|
if (!isa<Constant>(CurrentUser))
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Run through all of the constants in the module and determine if they are
|
|
// valid candidates to be merged into the string pool. Valid candidates will
|
|
// be added to MergeableStrings.
|
|
void PPCMergeStringPool::collectCandidateConstants(Module &M) {
|
|
SmallVector<GlobalValue *, 4> UsedV;
|
|
collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/false);
|
|
SmallVector<GlobalValue *, 4> UsedVCompiler;
|
|
collectUsedGlobalVariables(M, UsedVCompiler, /*CompilerUsed=*/true);
|
|
// Combine all of the Global Variables marked as used into a SmallPtrSet for
|
|
// faster lookup inside the loop.
|
|
SmallPtrSet<GlobalValue *, 8> AllUsedGlobals;
|
|
AllUsedGlobals.insert(UsedV.begin(), UsedV.end());
|
|
AllUsedGlobals.insert(UsedVCompiler.begin(), UsedVCompiler.end());
|
|
|
|
for (GlobalVariable &Global : M.globals()) {
|
|
LLVM_DEBUG(dbgs() << "Looking at global:");
|
|
LLVM_DEBUG(Global.dump());
|
|
LLVM_DEBUG(dbgs() << "isConstant() " << Global.isConstant() << "\n");
|
|
LLVM_DEBUG(dbgs() << "hasInitializer() " << Global.hasInitializer()
|
|
<< "\n");
|
|
|
|
// We can only pool constants.
|
|
if (!Global.isConstant() || !Global.hasInitializer())
|
|
continue;
|
|
|
|
// If a global constant has a section we do not try to pool it because
|
|
// there is no guarantee that other constants will also be in the same
|
|
// section. Trying to pool constants from different sections (or no
|
|
// section) means that the pool has to be in multiple sections at the same
|
|
// time.
|
|
if (Global.hasSection())
|
|
continue;
|
|
|
|
// Do not pool constants with metadata because we should not add metadata
|
|
// to the pool when that metadata refers to a single constant in the pool.
|
|
if (Global.hasMetadata())
|
|
continue;
|
|
|
|
ConstantDataSequential *ConstData =
|
|
dyn_cast<ConstantDataSequential>(Global.getInitializer());
|
|
|
|
// If the constant is undef then ConstData will be null.
|
|
if (!ConstData)
|
|
continue;
|
|
|
|
// Do not pool globals that are part of llvm.used or llvm.compiler.end.
|
|
if (AllUsedGlobals.contains(&Global))
|
|
continue;
|
|
|
|
if (!hasReplaceableUsers(Global))
|
|
continue;
|
|
|
|
Align AlignOfGlobal = Global.getAlign().valueOrOne();
|
|
|
|
// TODO: At this point do not allow over-aligned types. Adding a type
|
|
// with larger alignment may lose the larger alignment once it is
|
|
// added to the struct.
|
|
// Fix this in a future patch.
|
|
if (AlignOfGlobal.value() > ConstData->getElementByteSize())
|
|
continue;
|
|
|
|
// Make sure that the global is only visible inside the compilation unit.
|
|
if (Global.getLinkage() != GlobalValue::PrivateLinkage &&
|
|
Global.getLinkage() != GlobalValue::InternalLinkage)
|
|
continue;
|
|
|
|
LLVM_DEBUG(dbgs() << "Constant data of Global: ");
|
|
LLVM_DEBUG(ConstData->dump());
|
|
LLVM_DEBUG(dbgs() << "\n\n");
|
|
|
|
MergeableStrings.push_back(&Global);
|
|
if (MaxAlignment < AlignOfGlobal)
|
|
MaxAlignment = AlignOfGlobal;
|
|
|
|
// If we have already reached the maximum number of pooled strings then
|
|
// there is no point in looking for more.
|
|
if (MergeableStrings.size() >= MaxStringsPooled)
|
|
break;
|
|
}
|
|
}
|
|
|
|
bool PPCMergeStringPool::mergeModuleStringPool(Module &M) {
|
|
|
|
LLVM_DEBUG(dbgs() << "Merging string pool for module: " << M.getName()
|
|
<< "\n");
|
|
LLVM_DEBUG(dbgs() << "Number of globals is: " << M.global_size() << "\n");
|
|
|
|
collectCandidateConstants(M);
|
|
|
|
// If we have too few constants in the module that are merge candidates we
|
|
// will skip doing the merging.
|
|
if (MergeableStrings.size() < MinStringsBeforePool)
|
|
return false;
|
|
|
|
// Sort the global constants to make access more efficient.
|
|
std::sort(MergeableStrings.begin(), MergeableStrings.end(), CompareConstants);
|
|
|
|
SmallVector<Constant *> ConstantsInStruct;
|
|
for (GlobalVariable *GV : MergeableStrings)
|
|
ConstantsInStruct.push_back(GV->getInitializer());
|
|
|
|
// Use an anonymous struct to pool the strings.
|
|
// TODO: This pass uses a single anonymous struct for all of the pooled
|
|
// entries. This may cause a performance issue in the situation where
|
|
// computing the offset requires two instructions (addis, addi). For the
|
|
// future we may want to split this into multiple structs.
|
|
Constant *ConstantPool = ConstantStruct::getAnon(ConstantsInStruct);
|
|
PooledStructType = ConstantPool->getType();
|
|
|
|
// The GlobalVariable constructor calls
|
|
// MM->insertGlobalVariable(PooledGlobal).
|
|
GlobalVariable *PooledGlobal =
|
|
new GlobalVariable(M, PooledStructType,
|
|
/* isConstant */ true, GlobalValue::PrivateLinkage,
|
|
ConstantPool, "__ModuleStringPool");
|
|
PooledGlobal->setAlignment(MaxAlignment);
|
|
|
|
LLVM_DEBUG(dbgs() << "Constructing global variable for string pool: ");
|
|
LLVM_DEBUG(PooledGlobal->dump());
|
|
|
|
Context = &M.getContext();
|
|
size_t ElementIndex = 0;
|
|
for (GlobalVariable *GV : MergeableStrings) {
|
|
|
|
LLVM_DEBUG(dbgs() << "The global:\n");
|
|
LLVM_DEBUG(GV->dump());
|
|
LLVM_DEBUG(dbgs() << "Has " << GV->getNumUses() << " uses.\n");
|
|
|
|
// Access to the pooled constant strings require an offset. Add a GEP
|
|
// before every use in order to compute this offset.
|
|
replaceUsesWithGEP(GV, PooledGlobal, ElementIndex);
|
|
|
|
// This GV has no more uses so we can erase it.
|
|
if (GV->use_empty())
|
|
GV->eraseFromParent();
|
|
|
|
NumPooledStrings++;
|
|
ElementIndex++;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool userHasOperand(User *TheUser, GlobalVariable *GVOperand) {
|
|
for (Value *Op : TheUser->operands())
|
|
if (Op == GVOperand)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
// For pooled strings we need to add the offset into the pool for each string.
|
|
// This is done by adding a Get Element Pointer (GEP) before each user. This
|
|
// function adds the GEP.
|
|
void PPCMergeStringPool::replaceUsesWithGEP(GlobalVariable *GlobalToReplace,
|
|
GlobalVariable *GPool,
|
|
unsigned ElementIndex) {
|
|
SmallVector<Value *, 2> Indices;
|
|
Indices.push_back(ConstantInt::get(Type::getInt32Ty(*Context), 0));
|
|
Indices.push_back(ConstantInt::get(Type::getInt32Ty(*Context), ElementIndex));
|
|
|
|
// Need to save a temporary copy of each user list because we remove uses
|
|
// as we replace them.
|
|
SmallVector<User *> Users;
|
|
for (User *CurrentUser : GlobalToReplace->users())
|
|
Users.push_back(CurrentUser);
|
|
|
|
for (User *CurrentUser : Users) {
|
|
Instruction *UserInstruction = dyn_cast<Instruction>(CurrentUser);
|
|
Constant *UserConstant = dyn_cast<Constant>(CurrentUser);
|
|
|
|
// At this point we expect that the user is either an instruction or a
|
|
// constant.
|
|
assert((UserConstant || UserInstruction) &&
|
|
"Expected the user to be an instruction or a constant.");
|
|
|
|
// The user was not found so it must have been replaced earlier.
|
|
if (!userHasOperand(CurrentUser, GlobalToReplace))
|
|
continue;
|
|
|
|
// We cannot replace operands in globals so we ignore those.
|
|
if (isa<GlobalValue>(CurrentUser))
|
|
continue;
|
|
|
|
if (!UserInstruction) {
|
|
// User is a constant type.
|
|
Constant *ConstGEP = ConstantExpr::getInBoundsGetElementPtr(
|
|
PooledStructType, GPool, Indices);
|
|
UserConstant->handleOperandChange(GlobalToReplace, ConstGEP);
|
|
continue;
|
|
}
|
|
|
|
if (PHINode *UserPHI = dyn_cast<PHINode>(UserInstruction)) {
|
|
// GEP instructions cannot be added before PHI nodes.
|
|
// With getInBoundsGetElementPtr we create the GEP and then replace it
|
|
// inline into the PHI.
|
|
Constant *ConstGEP = ConstantExpr::getInBoundsGetElementPtr(
|
|
PooledStructType, GPool, Indices);
|
|
UserPHI->replaceUsesOfWith(GlobalToReplace, ConstGEP);
|
|
continue;
|
|
}
|
|
// The user is a valid instruction that is not a PHINode.
|
|
GetElementPtrInst *GEPInst =
|
|
GetElementPtrInst::Create(PooledStructType, GPool, Indices);
|
|
GEPInst->insertBefore(UserInstruction);
|
|
|
|
LLVM_DEBUG(dbgs() << "Inserting GEP before:\n");
|
|
LLVM_DEBUG(UserInstruction->dump());
|
|
|
|
LLVM_DEBUG(dbgs() << "Replacing this global:\n");
|
|
LLVM_DEBUG(GlobalToReplace->dump());
|
|
LLVM_DEBUG(dbgs() << "with this:\n");
|
|
LLVM_DEBUG(GEPInst->dump());
|
|
|
|
// After the GEP is inserted the GV can be replaced.
|
|
CurrentUser->replaceUsesOfWith(GlobalToReplace, GEPInst);
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
|
|
char PPCMergeStringPool::ID = 0;
|
|
|
|
INITIALIZE_PASS(PPCMergeStringPool, DEBUG_TYPE, "PPC Merge String Pool", false,
|
|
false)
|
|
|
|
ModulePass *llvm::createPPCMergeStringPoolPass() {
|
|
return new PPCMergeStringPool();
|
|
}
|