diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h index 111fa9f9ec01..710c43182d64 100644 --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -32,7 +32,6 @@ class BinaryFunction; class BinaryBasicBlock { public: - /// Profile execution information for a given edge in CFG. /// /// If MispredictedCount equals COUNT_INFERRED, then we have a profile @@ -55,7 +54,7 @@ public: }; static constexpr uint32_t INVALID_OFFSET = - std::numeric_limits::max(); + std::numeric_limits::max(); using BranchInfoType = SmallVector; @@ -148,11 +147,9 @@ private: BinaryBasicBlock &operator=(const BinaryBasicBlock &) = delete; BinaryBasicBlock &operator=(const BinaryBasicBlock &&) = delete; - explicit BinaryBasicBlock( - BinaryFunction *Function, - MCSymbol *Label, - uint32_t Offset = INVALID_OFFSET) - : Function(Function), Label(Label) { + explicit BinaryBasicBlock(BinaryFunction *Function, MCSymbol *Label, + uint32_t Offset = INVALID_OFFSET) + : Function(Function), Label(Label) { assert(Function && "Function must be non-null"); InputRange.first = Offset; } @@ -163,9 +160,7 @@ private: const BinaryBasicBlock &RHS); /// Assign new label to the basic block. - void setLabel(MCSymbol *Symbol) { - Label = Symbol; - } + void setLabel(MCSymbol *Symbol) { Label = Symbol; } public: static constexpr uint64_t COUNT_INFERRED = @@ -324,12 +319,12 @@ public: using branch_info_iterator = BranchInfoType::iterator; using const_branch_info_iterator = BranchInfoType::const_iterator; using branch_info_reverse_iterator = - std::reverse_iterator; + std::reverse_iterator; using const_branch_info_reverse_iterator = - std::reverse_iterator; + std::reverse_iterator; branch_info_iterator branch_info_begin() { return BranchInfo.begin(); } - branch_info_iterator branch_info_end() { return BranchInfo.end(); } + branch_info_iterator branch_info_end() { return BranchInfo.end(); } const_branch_info_iterator branch_info_begin() const { return BranchInfo.begin(); } @@ -339,9 +334,7 @@ public: branch_info_reverse_iterator branch_info_rbegin() { return BranchInfo.rbegin(); } - branch_info_reverse_iterator branch_info_rend() { - return BranchInfo.rend(); - } + branch_info_reverse_iterator branch_info_rend() { return BranchInfo.rend(); } const_branch_info_reverse_iterator branch_info_rbegin() const { return BranchInfo.rbegin(); } @@ -349,16 +342,16 @@ public: return BranchInfo.rend(); } - size_t branch_info_size() const { return BranchInfo.size(); } + size_t branch_info_size() const { return BranchInfo.size(); } bool branch_info_empty() const { return BranchInfo.empty(); } inline iterator_range branch_info() { - return iterator_range( - BranchInfo.begin(), BranchInfo.end()); + return iterator_range(BranchInfo.begin(), + BranchInfo.end()); } inline iterator_range branch_info() const { - return iterator_range( - BranchInfo.begin(), BranchInfo.end()); + return iterator_range(BranchInfo.begin(), + BranchInfo.end()); } /// Get instruction at given index. @@ -369,14 +362,10 @@ public: } /// Return symbol marking the start of this basic block. - MCSymbol *getLabel() { - return Label; - } + MCSymbol *getLabel() { return Label; } /// Return symbol marking the start of this basic block (const version). - const MCSymbol *getLabel() const { - return Label; - } + const MCSymbol *getLabel() const { return Label; } /// Get successor with given \p Label if \p Label != nullptr. /// Returns nullptr if no such successor is found. @@ -399,13 +388,13 @@ public: } const BinaryBasicBlock *getConditionalSuccessor(bool Condition) const { - return - const_cast(this)->getConditionalSuccessor(Condition); + return const_cast(this)->getConditionalSuccessor( + Condition); } /// Find the fallthrough successor for a block, or nullptr if there is /// none. - BinaryBasicBlock* getFallthrough() { + BinaryBasicBlock *getFallthrough() { if (succ_size() == 2) return getConditionalSuccessor(false); else @@ -438,8 +427,7 @@ public: BinaryBranchInfo &getBranchInfo(const MCSymbol *Label); /// Set branch information for the outgoing edge to block \p Succ. - void setSuccessorBranchInfo(const BinaryBasicBlock &Succ, - uint64_t Count, + void setSuccessorBranchInfo(const BinaryBasicBlock &Succ, uint64_t Count, uint64_t MispredictedCount) { BinaryBranchInfo &BI = getBranchInfo(Succ); BI.Count = Count; @@ -473,9 +461,7 @@ public: BinaryBasicBlock *getLandingPad(const MCSymbol *Label) const; /// Return local name for the block. - StringRef getName() const { - return Label->getName(); - } + StringRef getName() const { return Label->getName(); } /// Add instruction at the end of this basic block. /// Returns iterator pointing to the inserted instruction. @@ -494,23 +480,20 @@ public: } /// Add a range of instructions to the end of this basic block. - template - void addInstructions(Itr Begin, Itr End) { + template void addInstructions(Itr Begin, Itr End) { while (Begin != End) { addInstruction(*Begin++); } } /// Add a range of instructions to the end of this basic block. - template - void addInstructions(RangeTy R) { + template void addInstructions(RangeTy R) { for (auto &I : R) addInstruction(I); } /// Add instruction before Pos in this basic block. - template - Itr insertPseudoInstr(Itr Pos, MCInst &Instr) { + template Itr insertPseudoInstr(Itr Pos, MCInst &Instr) { ++NumPseudos; return Instructions.insert(Pos, Instr); } @@ -519,9 +502,7 @@ public: uint32_t getNumPseudos() const; /// Return the number of emitted instructions for this basic block. - uint32_t getNumNonPseudos() const { - return size() - getNumPseudos(); - } + uint32_t getNumNonPseudos() const { return size() - getNumPseudos(); } /// Return iterator to the first non-pseudo instruction or end() /// if no such instruction was found. @@ -573,35 +554,24 @@ public: /// Calculate and return CFI state after execution of this basic block. /// The state depends on CFI state at entry and CFI instructions inside the /// basic block. - int32_t getCFIStateAtExit() const { - return getCFIStateAtInstr(nullptr); - } + int32_t getCFIStateAtExit() const { return getCFIStateAtInstr(nullptr); } /// Set minimum alignment for the basic block. - void setAlignment(uint32_t Align) { - Alignment = Align; - } + void setAlignment(uint32_t Align) { Alignment = Align; } /// Return required alignment for the block. - uint32_t getAlignment() const { - return Alignment; - } + uint32_t getAlignment() const { return Alignment; } /// Set the maximum number of bytes to use for the block alignment. - void setAlignmentMaxBytes(uint32_t Value) { - AlignmentMaxBytes = Value; - } + void setAlignmentMaxBytes(uint32_t Value) { AlignmentMaxBytes = Value; } /// Return the maximum number of bytes to use for the block alignment. - uint32_t getAlignmentMaxBytes() const { - return AlignmentMaxBytes; - } + uint32_t getAlignmentMaxBytes() const { return AlignmentMaxBytes; } /// Adds block to successor list, and also updates predecessor list for /// successor block. /// Set branch info for this path. - void addSuccessor(BinaryBasicBlock *Succ, - uint64_t Count = 0, + void addSuccessor(BinaryBasicBlock *Succ, uint64_t Count = 0, uint64_t MispredictedCount = 0); void addSuccessor(BinaryBasicBlock *Succ, const BinaryBranchInfo &BI) { @@ -609,8 +579,7 @@ public: } /// Add a range of successors. - template - void addSuccessors(Itr Begin, Itr End) { + template void addSuccessors(Itr Begin, Itr End) { while (Begin != End) { addSuccessor(*Begin++); } @@ -627,19 +596,15 @@ public: /// Replace Succ with NewSucc. This routine is helpful for preserving /// the order of conditional successors when editing the CFG. - void replaceSuccessor(BinaryBasicBlock *Succ, - BinaryBasicBlock *NewSucc, - uint64_t Count = 0, - uint64_t MispredictedCount = 0); + void replaceSuccessor(BinaryBasicBlock *Succ, BinaryBasicBlock *NewSucc, + uint64_t Count = 0, uint64_t MispredictedCount = 0); /// Move all of this block's successors to a new block, and set the /// execution count of this new block with our execution count. This is /// useful when splitting a block in two. void moveAllSuccessorsTo(BinaryBasicBlock *New) { - New->addSuccessors(successors().begin(), - successors().end(), - branch_info_begin(), - branch_info_end()); + New->addSuccessors(successors().begin(), successors().end(), + branch_info_begin(), branch_info_end()); removeAllSuccessors(); // Update the execution count on the new block. @@ -672,17 +637,13 @@ public: } /// Test if this BB has a valid execution count. - bool hasProfile() const { - return ExecutionCount != COUNT_NO_PROFILE; - } + bool hasProfile() const { return ExecutionCount != COUNT_NO_PROFILE; } /// Return the information about the number of times this basic block was /// executed. /// /// Return COUNT_NO_PROFILE if there's no profile info. - uint64_t getExecutionCount() const { - return ExecutionCount; - } + uint64_t getExecutionCount() const { return ExecutionCount; } /// Return the execution count for blocks with known profile. /// Return 0 if the block has no profile. @@ -691,9 +652,7 @@ public: } /// Set the execution count for this block. - void setExecutionCount(uint64_t Count) { - ExecutionCount = Count; - } + void setExecutionCount(uint64_t Count) { ExecutionCount = Count; } /// Apply a given \p Ratio to the profile information of this basic block. void adjustExecutionCount(double Ratio); @@ -702,33 +661,21 @@ public: /// (either primary or secondary). bool isEntryPoint() const; - bool isValid() const { - return IsValid; - } + bool isValid() const { return IsValid; } - void markValid(const bool Valid) { - IsValid = Valid; - } + void markValid(const bool Valid) { IsValid = Valid; } - bool isCold() const { - return IsCold; - } + bool isCold() const { return IsCold; } - void setIsCold(const bool Flag) { - IsCold = Flag; - } + void setIsCold(const bool Flag) { IsCold = Flag; } /// Return true if the block can be outlined. At the moment we disallow /// outlining of blocks that can potentially throw exceptions or are /// the beginning of a landing pad. The entry basic block also can /// never be outlined. - bool canOutline() const { - return CanOutline; - } + bool canOutline() const { return CanOutline; } - void setCanOutline(const bool Flag) { - CanOutline = Flag; - } + void setCanOutline(const bool Flag) { CanOutline = Flag; } /// Erase pseudo instruction at a given iterator. /// Return iterator following the removed instruction. @@ -815,7 +762,7 @@ public: InstructionListType SplitInst; assert(!Instructions.empty()); - while(&Instructions.back() != Inst) { + while (&Instructions.back() != Inst) { SplitInst.push_back(Instructions.back()); Instructions.pop_back(); } @@ -867,9 +814,10 @@ public: /// Return offset translation table for the basic block. OffsetTranslationTableTy &getOffsetTranslationTable() { - return OffsetTranslationTable ? - *OffsetTranslationTable : - *(OffsetTranslationTable = std::make_unique()); + return OffsetTranslationTable + ? *OffsetTranslationTable + : *(OffsetTranslationTable = + std::make_unique()); } /// Return offset translation table for the basic block. @@ -882,16 +830,12 @@ public: return OutputAddressRange.second - OutputAddressRange.first; } - BinaryFunction *getFunction() const { - return Function; - } + BinaryFunction *getFunction() const { return Function; } /// Analyze and interpret the terminators of this basic block. TBB must be /// initialized with the original fall-through for this BB. - bool analyzeBranch(const MCSymbol *&TBB, - const MCSymbol *&FBB, - MCInst *&CondBranch, - MCInst *&UncondBranch); + bool analyzeBranch(const MCSymbol *&TBB, const MCSymbol *&FBB, + MCInst *&CondBranch, MCInst *&UncondBranch); /// Return true if iterator \p I is pointing to the first instruction in /// a pair that could be macro-fused. @@ -917,15 +861,11 @@ public: bool validateSuccessorInvariants(); /// Return offset of the basic block from the function start on input. - uint32_t getInputOffset() const { - return InputRange.first; - } + uint32_t getInputOffset() const { return InputRange.first; } /// Return offset from the function start to location immediately past /// the end of the basic block. - uint32_t getEndOffset() const { - return InputRange.second; - } + uint32_t getEndOffset() const { return InputRange.second; } /// Return size of the basic block on input. uint32_t getOriginalSize() const { @@ -947,14 +887,10 @@ public: } /// Set layout index. To be used by BinaryFunction. - void setLayoutIndex(unsigned Index) const { - LayoutIndex = Index; - } + void setLayoutIndex(unsigned Index) const { LayoutIndex = Index; } /// Needed by graph traits. - BinaryFunction *getParent() const { - return getFunction(); - } + BinaryFunction *getParent() const { return getFunction(); } /// Return true if the containing function is in CFG state. bool hasCFG() const; @@ -963,9 +899,7 @@ public: bool hasInstructions() const; /// Return offset of the basic block from the function start. - uint32_t getOffset() const { - return InputRange.first; - } + uint32_t getOffset() const { return InputRange.first; } /// Get the index of this basic block. unsigned getIndex() const { @@ -978,8 +912,7 @@ public: private: void adjustNumPseudos(const MCInst &Inst, int Sign); - template - void adjustNumPseudos(Itr Begin, Itr End, int Sign) { + template void adjustNumPseudos(Itr Begin, Itr End, int Sign) { while (Begin != End) { adjustNumPseudos(*Begin++, Sign); } @@ -993,19 +926,15 @@ private: /// If \p Multiple is set to true, it will remove all predecessors that /// are equal to \p Pred. Otherwise, the first instance of \p Pred found /// will be removed. This only matters in awkward, redundant CFGs. - void removePredecessor(BinaryBasicBlock *Pred, bool Multiple=true); + void removePredecessor(BinaryBasicBlock *Pred, bool Multiple = true); /// Set end offset of this basic block. - void setEndOffset(uint32_t Offset) { - InputRange.second = Offset; - } + void setEndOffset(uint32_t Offset) { InputRange.second = Offset; } /// Set the index of this basic block. - void setIndex(unsigned I) { - Index = I; - } + void setIndex(unsigned I) { Index = I; } - template void clearList(T& List) { + template void clearList(T &List) { T TempList; TempList.swap(List); } @@ -1031,7 +960,6 @@ bool operator<(const BinaryBasicBlock &LHS, const BinaryBasicBlock &RHS); } // namespace bolt - // GraphTraits specializations for basic block graphs (CFGs) template <> struct GraphTraits { using NodeRef = bolt::BinaryBasicBlock *; @@ -1041,24 +969,18 @@ template <> struct GraphTraits { static inline ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); } - static inline ChildIteratorType child_end(NodeRef N) { - return N->succ_end(); - } + static inline ChildIteratorType child_end(NodeRef N) { return N->succ_end(); } }; template <> struct GraphTraits { using NodeRef = const bolt::BinaryBasicBlock *; using ChildIteratorType = bolt::BinaryBasicBlock::const_succ_iterator; - static NodeRef getEntryNode(const bolt::BinaryBasicBlock *BB) { - return BB; - } + static NodeRef getEntryNode(const bolt::BinaryBasicBlock *BB) { return BB; } static inline ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); } - static inline ChildIteratorType child_end(NodeRef N) { - return N->succ_end(); - } + static inline ChildIteratorType child_end(NodeRef N) { return N->succ_end(); } }; template <> struct GraphTraits> { @@ -1070,9 +992,7 @@ template <> struct GraphTraits> { static inline ChildIteratorType child_begin(NodeRef N) { return N->pred_begin(); } - static inline ChildIteratorType child_end(NodeRef N) { - return N->pred_end(); - } + static inline ChildIteratorType child_end(NodeRef N) { return N->pred_end(); } }; template <> struct GraphTraits> { @@ -1084,9 +1004,7 @@ template <> struct GraphTraits> { static inline ChildIteratorType child_begin(NodeRef N) { return N->pred_begin(); } - static inline ChildIteratorType child_end(NodeRef N) { - return N->pred_end(); - } + static inline ChildIteratorType child_end(NodeRef N) { return N->pred_end(); } }; } // namespace llvm diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h index 684f5edafd9e..23e3071fc007 100644 --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -84,22 +84,22 @@ inline raw_ostream &operator<<(raw_ostream &OS, const SegmentInfo &SegInfo) { } enum class MemoryContentsType : char { - UNKNOWN = 0, /// Unknown contents. - POSSIBLE_JUMP_TABLE, /// Possibly a non-PIC jump table. - POSSIBLE_PIC_JUMP_TABLE, /// Possibly a PIC jump table. + UNKNOWN = 0, /// Unknown contents. + POSSIBLE_JUMP_TABLE, /// Possibly a non-PIC jump table. + POSSIBLE_PIC_JUMP_TABLE, /// Possibly a PIC jump table. }; /// Helper function to truncate a \p Value to given size in \p Bytes. inline int64_t truncateToSize(int64_t Value, unsigned Bytes) { - return Value & ((uint64_t) (int64_t) -1 >> (64 - Bytes * 8)); + return Value & ((uint64_t)(int64_t)-1 >> (64 - Bytes * 8)); } /// Filter iterator. template > + typename PredType = std::function> class FilterIterator - : public std::iterator::value_type> { + : public std::iterator::value_type> { using Iterator = FilterIterator; using T = typename std::iterator_traits::reference; using PointerT = typename std::iterator_traits::pointer; @@ -119,21 +119,18 @@ class FilterIterator while (Itr != End && !Pred(Itr)) ++Itr; } + public: Iterator &operator++() { next(); return *this; } Iterator &operator--() { prev(); return *this; } Iterator operator++(int) { auto Tmp(Itr); next(); return Tmp; } Iterator operator--(int) { auto Tmp(Itr); prev(); return Tmp; } - bool operator==(const Iterator& Other) const { - return Itr == Other.Itr; - } - bool operator!=(const Iterator& Other) const { - return !operator==(Other); - } + bool operator==(const Iterator &Other) const { return Itr == Other.Itr; } + bool operator!=(const Iterator &Other) const { return !operator==(Other); } T operator*() { return *Itr; } PointerT operator->() { return &operator*(); } FilterIterator(PredType Pred, ItrType Itr, ItrType End) - : Pred(Pred), Itr(Itr), End(End) { + : Pred(Pred), Itr(Itr), End(End) { nextMatching(); } }; @@ -273,7 +270,7 @@ public: BinaryDataMapType BinaryDataMap; using FilteredBinaryDataConstIterator = - FilterIterator; + FilterIterator; using FilteredBinaryDataIterator = FilterIterator; /// Memory manager for sections and segments. Used to communicate with ORC @@ -292,12 +289,8 @@ public: } void setFileBuildID(StringRef ID) { FileBuildID = std::string(ID); } - bool hasSymbolsWithFileName() const { - return HasSymbolsWithFileName; - } - void setHasSymbolsWithFileName(bool Value) { - HasSymbolsWithFileName = true; - } + bool hasSymbolsWithFileName() const { return HasSymbolsWithFileName; } + void setHasSymbolsWithFileName(bool Value) { HasSymbolsWithFileName = true; } /// Return true if relocations against symbol with a given name /// must be created. @@ -306,9 +299,7 @@ public: uint64_t getNumUnusedProfiledObjects() const { return NumUnusedProfiledObjects; } - void setNumUnusedProfiledObjects(uint64_t N) { - NumUnusedProfiledObjects = N; - } + void setNumUnusedProfiledObjects(uint64_t N) { NumUnusedProfiledObjects = N; } RuntimeLibrary *getRuntimeLibrary() { return RtLibrary.get(); } void setRuntimeLibrary(std::unique_ptr Lib) { @@ -341,8 +332,8 @@ public: BinaryFunction *getBinaryFunctionAtAddress(uint64_t Address); const BinaryFunction *getBinaryFunctionAtAddress(uint64_t Address) const { - return const_cast(this)-> - getBinaryFunctionAtAddress(Address); + return const_cast(this)->getBinaryFunctionAtAddress( + Address); } /// Return size of an entry for the given jump table \p Type. @@ -365,7 +356,8 @@ public: unsigned getDWARFEncodingSize(unsigned Encoding) { switch (Encoding & 0x0f) { - default: llvm_unreachable("unknown encoding"); + default: + llvm_unreachable("unknown encoding"); case dwarf::DW_EH_PE_absptr: case dwarf::DW_EH_PE_signed: return AsmInfo->getCodePointerSize(); @@ -385,8 +377,7 @@ public: /// /// As we fold identical functions, multiple symbols can point /// to the same BinaryFunction. - std::unordered_map SymbolToFunctionMap; + std::unordered_map SymbolToFunctionMap; /// A mutex that is used to control parallel accesses to SymbolToFunctionMap mutable std::shared_timed_mutex SymbolToFunctionMapMutex; @@ -426,10 +417,8 @@ public: /// Construct BinaryFunction object and add it to internal maps. BinaryFunction *createBinaryFunction(const std::string &Name, - BinarySection &Section, - uint64_t Address, - uint64_t Size, - uint64_t SymbolSize = 0, + BinarySection &Section, uint64_t Address, + uint64_t Size, uint64_t SymbolSize = 0, uint16_t Alignment = 0); /// Return all functions for this rewrite instance. @@ -474,8 +463,7 @@ public: /// Optionally, populate \p Offsets with jump table entries. The entries /// could be partially populated if the jump table detection fails. bool analyzeJumpTable(const uint64_t Address, - const JumpTable::JumpTableType Type, - BinaryFunction &BF, + const JumpTable::JumpTableType Type, BinaryFunction &BF, const uint64_t NextJTAddress = 0, JumpTable::OffsetsType *Offsets = nullptr); @@ -635,10 +623,8 @@ public: BinaryContext(std::unique_ptr Ctx, std::unique_ptr DwCtx, - std::unique_ptr TheTriple, - const Target *TheTarget, - std::string TripleName, - std::unique_ptr MCE, + std::unique_ptr TheTriple, const Target *TheTarget, + std::string TripleName, std::unique_ptr MCE, std::unique_ptr MOFI, std::unique_ptr AsmInfo, std::unique_ptr MII, @@ -653,13 +639,9 @@ public: std::unique_ptr createObjectWriter(raw_pwrite_stream &OS); - bool isELF() const { - return TheTriple->isOSBinFormatELF(); - } + bool isELF() const { return TheTriple->isOSBinFormatELF(); } - bool isMachO() const { - return TheTriple->isOSBinFormatMachO(); - } + bool isMachO() const { return TheTriple->isOSBinFormatMachO(); } bool isAArch64() const { return TheTriple->getArch() == llvm::Triple::aarch64; @@ -688,10 +670,9 @@ public: --Begin; } auto End = BinaryDataMap.upper_bound(Section.getEndAddress()); - auto pred = - [&Section](const binary_data_const_iterator &Itr) -> bool { - return Itr->second->getSection() == Section; - }; + auto pred = [&Section](const binary_data_const_iterator &Itr) -> bool { + return Itr->second->getSection() == Section; + }; return make_range(FilteredBinaryDataConstIterator(pred, Begin, End), FilteredBinaryDataConstIterator(pred, End, End)); } @@ -726,9 +707,8 @@ public: /// Process \p Address reference from code in function \BF. /// \p IsPCRel indicates if the reference is PC-relative. /// Return pair corresponding to the \p Address. - std::pair handleAddressRef(uint64_t Address, - BinaryFunction &BF, - bool IsPCRel); + std::pair + handleAddressRef(uint64_t Address, BinaryFunction &BF, bool IsPCRel); /// Analyze memory contents at the given \p Address and return the type of /// memory contents (such as a possible jump table). @@ -747,10 +727,8 @@ public: /// If no symbol exists, create one with unique name using \p Prefix. /// If there are multiple symbols registered at the \p Address, then /// return the first one. - MCSymbol *getOrCreateGlobalSymbol(uint64_t Address, - Twine Prefix, - uint64_t Size = 0, - uint16_t Alignment = 0, + MCSymbol *getOrCreateGlobalSymbol(uint64_t Address, Twine Prefix, + uint64_t Size = 0, uint16_t Alignment = 0, unsigned Flags = 0); /// Create a global symbol without registering an address. @@ -759,10 +737,8 @@ public: /// Register a symbol with \p Name at a given \p Address using \p Size, /// \p Alignment, and \p Flags. See llvm::SymbolRef::Flags for the definition /// of \p Flags. - MCSymbol *registerNameAtAddress(StringRef Name, - uint64_t Address, - uint64_t Size, - uint16_t Alignment, + MCSymbol *registerNameAtAddress(StringRef Name, uint64_t Address, + uint64_t Size, uint16_t Alignment, unsigned Flags = 0); /// Return BinaryData registered at a given \p Address or nullptr if no @@ -780,14 +756,13 @@ public: /// Look up the symbol entry that contains the given \p Address (based on /// the start address and size for each symbol). Returns a pointer to /// the BinaryData for that symbol. If no data is found, nullptr is returned. - const BinaryData * - getBinaryDataContainingAddress(uint64_t Address) const { + const BinaryData *getBinaryDataContainingAddress(uint64_t Address) const { return getBinaryDataContainingAddressImpl(Address); } BinaryData *getBinaryDataContainingAddress(uint64_t Address) { - return - const_cast(getBinaryDataContainingAddressImpl(Address)); + return const_cast( + getBinaryDataContainingAddressImpl(Address)); } /// Return BinaryData for the given \p Name or nullptr if no @@ -805,8 +780,7 @@ public: /// Return true if \p SymbolName was generated internally and was not present /// in the input binary. bool isInternalSymbolName(const StringRef Name) { - return Name.startswith("SYMBOLat") || - Name.startswith("DATAat") || + return Name.startswith("SYMBOLat") || Name.startswith("DATAat") || Name.startswith("HOLEat"); } @@ -818,9 +792,7 @@ public: return Ctx->getOrCreateSymbol("__hot_end"); } - MCSection *getTextSection() const { - return MOFI->getTextSection(); - } + MCSection *getTextSection() const { return MOFI->getTextSection(); } /// Return code section with a given name. MCSection *getCodeSection(StringRef SectionName) const { @@ -841,21 +813,13 @@ public: /// \name Pre-assigned Section Names /// @{ - const char *getMainCodeSectionName() const { - return ".text"; - } + const char *getMainCodeSectionName() const { return ".text"; } - const char *getColdCodeSectionName() const { - return ".text.cold"; - } + const char *getColdCodeSectionName() const { return ".text.cold"; } - const char *getHotTextMoverSectionName() const { - return ".text.mover"; - } + const char *getHotTextMoverSectionName() const { return ".text.mover"; } - const char *getInjectedCodeSectionName() const { - return ".text.injected"; - } + const char *getInjectedCodeSectionName() const { return ".text.injected"; } const char *getInjectedColdCodeSectionName() const { return ".text.injected.cold"; @@ -892,7 +856,7 @@ public: bool setBinaryDataSize(uint64_t Address, uint64_t Size); /// Print the global symbol table. - void printGlobalSymbols(raw_ostream& OS) const; + void printGlobalSymbols(raw_ostream &OS) const; /// Register information about the given \p Section so we can look up /// sections by address. @@ -905,8 +869,7 @@ public: /// Register or update the information for the section with the given /// /p Name. If the section already exists, the information in the /// section will be updated with the new data. - BinarySection ®isterOrUpdateSection(StringRef Name, - unsigned ELFType, + BinarySection ®isterOrUpdateSection(StringRef Name, unsigned ELFType, unsigned ELFFlags, uint8_t *Data = nullptr, uint64_t Size = 0, @@ -916,15 +879,13 @@ public: /// with the given /p Name. If the section already exists, the /// information in the section will be updated with the new data. BinarySection & - registerOrUpdateNoteSection(StringRef Name, - uint8_t *Data = nullptr, - uint64_t Size = 0, - unsigned Alignment = 1, + registerOrUpdateNoteSection(StringRef Name, uint8_t *Data = nullptr, + uint64_t Size = 0, unsigned Alignment = 1, bool IsReadOnly = true, unsigned ELFType = ELF::SHT_PROGBITS) { return registerOrUpdateSection(Name, ELFType, - BinarySection::getFlags(IsReadOnly), - Data, Size, Alignment); + BinarySection::getFlags(IsReadOnly), Data, + Size, Alignment); } /// Remove the given /p Section from the set of all sections. Return @@ -933,15 +894,10 @@ public: /// Iterate over all registered sections. iterator_range sections() { - auto notNull = [](const SectionIterator &Itr) { - return (bool)*Itr; - }; - return make_range(FilteredSectionIterator(notNull, - Sections.begin(), - Sections.end()), - FilteredSectionIterator(notNull, - Sections.end(), - Sections.end())); + auto notNull = [](const SectionIterator &Itr) { return (bool)*Itr; }; + return make_range( + FilteredSectionIterator(notNull, Sections.begin(), Sections.end()), + FilteredSectionIterator(notNull, Sections.end(), Sections.end())); } /// Iterate over all registered sections. @@ -954,12 +910,10 @@ public: auto isAllocatable = [](const SectionIterator &Itr) { return *Itr && Itr->isAllocatable(); }; - return make_range(FilteredSectionIterator(isAllocatable, - Sections.begin(), - Sections.end()), - FilteredSectionIterator(isAllocatable, - Sections.end(), - Sections.end())); + return make_range( + FilteredSectionIterator(isAllocatable, Sections.begin(), + Sections.end()), + FilteredSectionIterator(isAllocatable, Sections.end(), Sections.end())); } /// Iterate over all registered code sections. @@ -967,12 +921,9 @@ public: auto isText = [](const SectionIterator &Itr) { return *Itr && Itr->isAllocatable() && Itr->isText(); }; - return make_range(FilteredSectionIterator(isText, - Sections.begin(), - Sections.end()), - FilteredSectionIterator(isText, - Sections.end(), - Sections.end())); + return make_range( + FilteredSectionIterator(isText, Sections.begin(), Sections.end()), + FilteredSectionIterator(isText, Sections.end(), Sections.end())); } /// Iterate over all registered allocatable sections. @@ -985,12 +936,9 @@ public: auto notAllocated = [](const SectionIterator &Itr) { return *Itr && !Itr->isAllocatable(); }; - return make_range(FilteredSectionIterator(notAllocated, - Sections.begin(), - Sections.end()), - FilteredSectionIterator(notAllocated, - Sections.end(), - Sections.end())); + return make_range( + FilteredSectionIterator(notAllocated, Sections.begin(), Sections.end()), + FilteredSectionIterator(notAllocated, Sections.end(), Sections.end())); } /// Iterate over all registered non-allocatable sections. @@ -1004,10 +952,8 @@ public: return *Itr && Itr->isAllocatable() && Itr->isRela(); }; return make_range(FilteredSectionIterator(isAllocatableRela, - Sections.begin(), - Sections.end()), - FilteredSectionIterator(isAllocatableRela, - Sections.end(), + Sections.begin(), Sections.end()), + FilteredSectionIterator(isAllocatableRela, Sections.end(), Sections.end())); } @@ -1020,7 +966,7 @@ public: ErrorOr getSectionNameForAddress(uint64_t Address) const; /// Print all sections. - void printSections(raw_ostream& OS) const; + void printSections(raw_ostream &OS) const; /// Return largest section containing the given \p Address. These /// functions only work for allocatable sections, i.e. ones with non-zero @@ -1106,10 +1052,11 @@ public: BinaryFunction *getFunctionForSymbol(const MCSymbol *Symbol, uint64_t *EntryDesc = nullptr); - const BinaryFunction *getFunctionForSymbol( - const MCSymbol *Symbol, uint64_t *EntryDesc = nullptr) const { - return const_cast(this)-> - getFunctionForSymbol(Symbol, EntryDesc); + const BinaryFunction * + getFunctionForSymbol(const MCSymbol *Symbol, + uint64_t *EntryDesc = nullptr) const { + return const_cast(this)->getFunctionForSymbol(Symbol, + EntryDesc); } /// Associate the symbol \p Sym with the function \p BF for lookups with @@ -1123,8 +1070,7 @@ public: /// Add a filename entry from SrcCUID to DestCUID. unsigned addDebugFilenameToUnit(const uint32_t DestCUID, - const uint32_t SrcCUID, - unsigned FileIndex); + const uint32_t SrcCUID, unsigned FileIndex); /// Return functions in output layout order std::vector getSortedFunctions(); @@ -1136,8 +1082,8 @@ public: /// /// Return the pair where the first size is for the main part, and the second /// size is for the cold one. - std::pair - calculateEmittedSize(BinaryFunction &BF, bool FixBranches = true); + std::pair calculateEmittedSize(BinaryFunction &BF, + bool FixBranches = true); /// Calculate the size of the instruction \p Inst optionally using a /// user-supplied emitter for lock-free multi-thread work. MCCodeEmitter is @@ -1202,27 +1148,22 @@ public: /// info. /// If printMCInst is true, the instruction is also printed in the /// architecture independent format. - void printInstruction(raw_ostream &OS, - const MCInst &Instruction, + void printInstruction(raw_ostream &OS, const MCInst &Instruction, uint64_t Offset = 0, const BinaryFunction *Function = nullptr, - bool PrintMCInst = false, - bool PrintMemData = false, + bool PrintMCInst = false, bool PrintMemData = false, bool PrintRelocations = false) const; /// Print a range of instructions. template - uint64_t printInstructions(raw_ostream &OS, - Itr Begin, - Itr End, - uint64_t Offset = 0, - const BinaryFunction *Function = nullptr, - bool PrintMCInst = false, - bool PrintMemData = false, - bool PrintRelocations = false) const { + uint64_t + printInstructions(raw_ostream &OS, Itr Begin, Itr End, uint64_t Offset = 0, + const BinaryFunction *Function = nullptr, + bool PrintMCInst = false, bool PrintMemData = false, + bool PrintRelocations = false) const { while (Begin != End) { - printInstruction(OS, *Begin, Offset, Function, PrintMCInst, - PrintMemData, PrintRelocations); + printInstruction(OS, *Begin, Offset, Function, PrintMCInst, PrintMemData, + PrintRelocations); Offset += computeCodeSize(Begin, Begin + 1); ++Begin; } @@ -1274,10 +1215,8 @@ public: } }; -template > -inline raw_ostream &operator<<(raw_ostream &OS, - const ArrayRef &ByteArray) { +template > +inline raw_ostream &operator<<(raw_ostream &OS, const ArrayRef &ByteArray) { const char *Sep = ""; for (const auto Byte : ByteArray) { OS << Sep << format("%.2x", Byte); diff --git a/bolt/include/bolt/Core/BinaryData.h b/bolt/include/bolt/Core/BinaryData.h index b8c4ce6e689e..817b1564c7cd 100644 --- a/bolt/include/bolt/Core/BinaryData.h +++ b/bolt/include/bolt/Core/BinaryData.h @@ -75,13 +75,9 @@ protected: public: BinaryData(BinaryData &&) = default; - BinaryData(MCSymbol &Symbol, - uint64_t Address, - uint64_t Size, - uint16_t Alignment, - BinarySection &Section, - unsigned Flags = 0); - virtual ~BinaryData() { } + BinaryData(MCSymbol &Symbol, uint64_t Address, uint64_t Size, + uint16_t Alignment, BinarySection &Section, unsigned Flags = 0); + virtual ~BinaryData() {} virtual bool isJumpTable() const { return false; } virtual bool isObject() const { return !isJumpTable(); } @@ -94,9 +90,7 @@ public: // BinaryData that is considered atomic and potentially moveable. All // MemInfo data and relocations should be wrt. to atomic data. - bool isAtomic() const { - return isTopLevelJumpTable() || !Parent; - } + bool isAtomic() const { return isTopLevelJumpTable() || !Parent; } iterator_range::const_iterator> symbols() const { return make_range(Symbols.begin(), Symbols.end()); @@ -147,9 +141,7 @@ public: return containsAddress(Address) && Address + Size <= getEndAddress(); } - const BinaryData *getParent() const { - return Parent; - } + const BinaryData *getParent() const { return Parent; } const BinaryData *getRootData() const { const BinaryData *BD = this; @@ -202,8 +194,7 @@ struct AddressAccess { uint64_t Offset; /// Offset within the object or absolute address uint64_t Count; /// Number of accesses bool operator==(const AddressAccess &Other) const { - return MemoryObject == Other.MemoryObject && - Offset == Other.Offset && + return MemoryObject == Other.MemoryObject && Offset == Other.Offset && Count == Other.Count; } }; diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 43b4a08ffe92..6ba7d34f51db 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -48,11 +48,7 @@ namespace bolt { using InputOffsetToAddressMapTy = std::unordered_multimap; /// Types of macro-fusion alignment corrections. -enum MacroFusionType { - MFT_NONE, - MFT_HOT, - MFT_ALL -}; +enum MacroFusionType { MFT_NONE, MFT_HOT, MFT_ALL }; enum IndirectCallPromotionType : char { ICP_NONE, /// Don't perform ICP. @@ -68,13 +64,12 @@ struct IndirectCallProfile { uint64_t Count; uint64_t Mispreds; - IndirectCallProfile(MCSymbol *Symbol, uint64_t Count, - uint64_t Mispreds, uint32_t Offset = 0) - : Symbol(Symbol), Offset(Offset), Count(Count), Mispreds(Mispreds) {} + IndirectCallProfile(MCSymbol *Symbol, uint64_t Count, uint64_t Mispreds, + uint32_t Offset = 0) + : Symbol(Symbol), Offset(Offset), Count(Count), Mispreds(Mispreds) {} bool operator==(const IndirectCallProfile &Other) const { - return Symbol == Other.Symbol && - Offset == Other.Offset; + return Symbol == Other.Symbol && Offset == Other.Offset; } }; @@ -111,20 +106,20 @@ inline raw_ostream &operator<<(raw_ostream &OS, class BinaryFunction { public: enum class State : char { - Empty = 0, /// Function body is empty. - Disassembled, /// Function have been disassembled. - CFG, /// Control flow graph has been built. - CFG_Finalized, /// CFG is finalized. No optimizations allowed. - EmittedCFG, /// Instructions have been emitted to output. - Emitted, /// Same as above plus CFG is destroyed. + Empty = 0, /// Function body is empty. + Disassembled, /// Function have been disassembled. + CFG, /// Control flow graph has been built. + CFG_Finalized, /// CFG is finalized. No optimizations allowed. + EmittedCFG, /// Instructions have been emitted to output. + Emitted, /// Same as above plus CFG is destroyed. }; /// Types of profile the function can use. Could be a combination. enum { - PF_NONE = 0, /// No profile. - PF_LBR = 1, /// Profile is based on last branch records. - PF_SAMPLE = 2, /// Non-LBR sample-based profile. - PF_MEMEVENT = 4, /// Profile has mem events. + PF_NONE = 0, /// No profile. + PF_LBR = 1, /// Profile is based on last branch records. + PF_SAMPLE = 2, /// Non-LBR sample-based profile. + PF_MEMEVENT = 4, /// Profile has mem events. }; /// Struct for tracking exception handling ranges. @@ -169,7 +164,7 @@ public: }; static constexpr uint64_t COUNT_NO_PROFILE = - BinaryBasicBlock::COUNT_NO_PROFILE; + BinaryBasicBlock::COUNT_NO_PROFILE; /// We have to use at least 2-byte alignment for functions because of C++ ABI. static constexpr unsigned MinAlign = 2; @@ -412,8 +407,8 @@ private: BinaryBasicBlock *getBasicBlockContainingOffset(uint64_t Offset); const BinaryBasicBlock *getBasicBlockContainingOffset(uint64_t Offset) const { - return const_cast(this) - ->getBasicBlockContainingOffset(Offset); + return const_cast(this)->getBasicBlockContainingOffset( + Offset); } /// Return basic block that started at offset \p Offset. @@ -423,7 +418,7 @@ private: } /// Release memory taken by the list. - template BinaryFunction &clearList(T& List) { + template BinaryFunction &clearList(T &List) { T TempList; TempList.swap(List); return *this; @@ -453,8 +448,8 @@ private: DenseMap LabelToBB; using BranchListType = SmallVector, 0>; - BranchListType TakenBranches; /// All local taken branches. - BranchListType IgnoredBranches; /// Branches ignored by CFG purposes. + BranchListType TakenBranches; /// All local taken branches. + BranchListType IgnoredBranches; /// Branches ignored by CFG purposes. /// Map offset in the function to a label. /// Labels are used for building CFG for simple functions. For non-simple @@ -494,7 +489,7 @@ private: /// A map of restore state CFI instructions to their equivalent CFI /// instructions that produce the same state, in order to eliminate /// remember-restore CFI instructions when rewriting CFI. - DenseMap> FrameRestoreEquivalents; + DenseMap> FrameRestoreEquivalents; // For tracking exception handling ranges. CallSitesType CallSites; @@ -573,7 +568,7 @@ private: mutable MCSymbol *FunctionColdEndLabel{nullptr}; /// Unique number associated with the function. - uint64_t FunctionNumber; + uint64_t FunctionNumber; /// Count the number of functions created. static uint64_t Count; @@ -654,8 +649,7 @@ private: /// Analyze and process indirect branch \p Instruction before it is /// added to Instructions list. - IndirectBranchType processIndirectBranch(MCInst &Instruction, - unsigned Size, + IndirectBranchType processIndirectBranch(MCInst &Instruction, unsigned Size, uint64_t Offset, uint64_t &TargetAddress); @@ -802,15 +796,13 @@ public: } inline iterator_range rlayout() { - return - iterator_range(BasicBlocksLayout.rbegin(), - BasicBlocksLayout.rend()); + return iterator_range(BasicBlocksLayout.rbegin(), + BasicBlocksLayout.rend()); } inline iterator_range rlayout() const { - return - iterator_range(BasicBlocksLayout.rbegin(), - BasicBlocksLayout.rend()); + return iterator_range( + BasicBlocksLayout.rbegin(), BasicBlocksLayout.rend()); } cfi_iterator cie_begin() { return CIEFrameInstructions.begin(); } @@ -870,13 +862,9 @@ public: void calculateMacroOpFusionStats(); /// Returns if loop detection has been run for this function. - bool hasLoopInfo() const { - return BLI != nullptr; - } + bool hasLoopInfo() const { return BLI != nullptr; } - const BinaryLoopInfo &getLoopInfo() { - return *BLI.get(); - } + const BinaryLoopInfo &getLoopInfo() { return *BLI.get(); } bool isLoopFree() { if (!hasLoopInfo()) { @@ -892,7 +880,7 @@ public: void viewGraph() const; /// Dump CFG in graphviz format - void dumpGraph(raw_ostream& OS) const; + void dumpGraph(raw_ostream &OS) const; /// Dump CFG in graphviz format to file. void dumpGraphToFile(std::string Filename) const; @@ -903,14 +891,10 @@ public: void dumpGraphForPass(std::string Annotation = "") const; /// Return BinaryContext for the function. - const BinaryContext &getBinaryContext() const { - return BC; - } + const BinaryContext &getBinaryContext() const { return BC; } /// Return BinaryContext for the function. - BinaryContext &getBinaryContext() { - return BC; - } + BinaryContext &getBinaryContext() { return BC; } /// Attempt to validate CFG invariants. bool validateCFG() const; @@ -929,8 +913,8 @@ public: /// nullptr the last basic block is given. const BinaryBasicBlock *getBasicBlockAfter(const BinaryBasicBlock *BB, bool IgnoreSplits = true) const { - return - const_cast(this)->getBasicBlockAfter(BB, IgnoreSplits); + return const_cast(this)->getBasicBlockAfter(BB, + IgnoreSplits); } BinaryBasicBlock *getBasicBlockAfter(const BinaryBasicBlock *BB, @@ -938,8 +922,8 @@ public: for (auto I = layout_begin(), E = layout_end(); I != E; ++I) { auto Next = std::next(I); if (*I == BB && Next != E) { - return (IgnoreSplits || (*I)->isCold() == (*Next)->isCold()) - ? *Next : nullptr; + return (IgnoreSplits || (*I)->isCold() == (*Next)->isCold()) ? *Next + : nullptr; } } return nullptr; @@ -953,7 +937,7 @@ public: const Optional LP = BC.MIB->getEHInfo(InvokeInst); if (LP && LP->first) { BinaryBasicBlock *LBB = BB.getLandingPad(LP->first); - assert (LBB && "Landing pad should be defined"); + assert(LBB && "Landing pad should be defined"); return LBB; } return nullptr; @@ -981,8 +965,8 @@ public: } const JumpTable *getJumpTableContainingAddress(uint64_t Address) const { - return const_cast(this)-> - getJumpTableContainingAddress(Address); + return const_cast(this)->getJumpTableContainingAddress( + Address); } /// Return the name of the function if the function has just one name. @@ -999,8 +983,8 @@ public: std::string getPrintName() const { const size_t NumNames = Symbols.size() + Aliases.size(); return NumNames == 1 - ? getOneName().str() - : (getOneName().str() + "(*" + std::to_string(NumNames) + ")"); + ? getOneName().str() + : (getOneName().str() + "(*" + std::to_string(NumNames) + ")"); } /// The function may have many names. For that reason, we avoid having @@ -1008,9 +992,7 @@ public: /// interface, such as forEachName(), hasName(), hasNameRegex(), etc. /// In some cases though, we need just a name uniquely identifying /// the function, and that's what this method is for. - StringRef getOneName() const { - return Symbols[0]->getName(); - } + StringRef getOneName() const { return Symbols[0]->getName(); } /// Return the name of the function as getPrintName(), but also trying /// to demangle it. @@ -1035,9 +1017,8 @@ public: /// Check if (possibly one out of many) function name matches the given /// string. Use this member function instead of direct name comparison. bool hasName(const std::string &FunctionName) const { - auto Res = forEachName([&](StringRef Name) { - return Name == FunctionName; - }); + auto Res = + forEachName([&](StringRef Name) { return Name == FunctionName; }); return Res.hasValue(); } @@ -1051,9 +1032,9 @@ public: /// Return a vector of all possible names for the function. const std::vector getNames() const { std::vector AllNames; - forEachName([&AllNames] (StringRef Name) { - AllNames.push_back(Name); - return false; + forEachName([&AllNames](StringRef Name) { + AllNames.push_back(Name); + return false; }); return AllNames; @@ -1061,65 +1042,44 @@ public: /// Return a state the function is in (see BinaryFunction::State definition /// for description). - State getState() const { - return CurrentState; - } + State getState() const { return CurrentState; } /// Return true if function has a control flow graph available. bool hasCFG() const { - return getState() == State::CFG || - getState() == State::CFG_Finalized || + return getState() == State::CFG || getState() == State::CFG_Finalized || getState() == State::EmittedCFG; } /// Return true if the function state implies that it includes instructions. bool hasInstructions() const { - return getState() == State::Disassembled || - hasCFG(); + return getState() == State::Disassembled || hasCFG(); } bool isEmitted() const { - return getState() == State::EmittedCFG || - getState() == State::Emitted; + return getState() == State::EmittedCFG || getState() == State::Emitted; } /// Return the section in the input binary this function originated from or /// nullptr if the function did not originate from the file. - BinarySection *getOriginSection() const { - return OriginSection; - } + BinarySection *getOriginSection() const { return OriginSection; } - void setOriginSection(BinarySection *Section) { - OriginSection = Section; - } + void setOriginSection(BinarySection *Section) { OriginSection = Section; } /// Return true if the function did not originate from the primary input file. - bool isInjected() const { - return IsInjected; - } + bool isInjected() const { return IsInjected; } /// Return original address of the function (or offset from base for PIC). - uint64_t getAddress() const { - return Address; - } + uint64_t getAddress() const { return Address; } - uint64_t getOutputAddress() const { - return OutputAddress; - } + uint64_t getOutputAddress() const { return OutputAddress; } - uint64_t getOutputSize() const { - return OutputSize; - } + uint64_t getOutputSize() const { return OutputSize; } /// Does this function have a valid streaming order index? - bool hasValidIndex() const { - return Index != -1U; - } + bool hasValidIndex() const { return Index != -1U; } /// Get the streaming order index for this function. - uint32_t getIndex() const { - return Index; - } + uint32_t getIndex() const { return Index; } /// Set the streaming order index for this function. void setIndex(uint32_t Idx) { @@ -1133,19 +1093,13 @@ public: } /// Return offset of the function body in the binary file. - uint64_t getFileOffset() const { - return FileOffset; - } + uint64_t getFileOffset() const { return FileOffset; } /// Return (original) byte size of the function. - uint64_t getSize() const { - return Size; - } + uint64_t getSize() const { return Size; } /// Return the maximum size the body of the function could have. - uint64_t getMaxSize() const { - return MaxSize; - } + uint64_t getMaxSize() const { return MaxSize; } /// Return the number of emitted instructions for this function. uint32_t getNumNonPseudos() const { @@ -1158,15 +1112,11 @@ public: /// Return MC symbol associated with the function. /// All references to the function should use this symbol. - MCSymbol *getSymbol() { - return Symbols[0]; - } + MCSymbol *getSymbol() { return Symbols[0]; } /// Return MC symbol associated with the function (const version). /// All references to the function should use this symbol. - const MCSymbol *getSymbol() const { - return Symbols[0]; - } + const MCSymbol *getSymbol() const { return Symbols[0]; } /// Return a list of symbols associated with the main entry of the function. SymbolListTy &getSymbols() { return Symbols; } @@ -1197,7 +1147,6 @@ public: return getSecondaryEntryPointSymbol(BB); } - /// Return MC symbol corresponding to an enumerated entry for multiple-entry /// functions. MCSymbol *getSymbolForEntryID(uint64_t EntryNum); @@ -1205,7 +1154,7 @@ public: return const_cast(this)->getSymbolForEntryID(EntryNum); } - using EntryPointCallbackTy = function_ref; + using EntryPointCallbackTy = function_ref; /// Invoke \p Callback function for every entry point in the function starting /// with the main entry and using entries in the ascending address order. @@ -1270,15 +1219,11 @@ public: } /// Return true if this is a function representing a PLT entry. - bool isPLTFunction() const { - return PLTSymbol != nullptr; - } + bool isPLTFunction() const { return PLTSymbol != nullptr; } /// Return PLT function reference symbol for PLT functions and nullptr for /// non-PLT functions. - const MCSymbol *getPLTSymbol() const { - return PLTSymbol; - } + const MCSymbol *getPLTSymbol() const { return PLTSymbol; } /// Set function PLT reference symbol for PLT functions. void setPLTSymbol(const MCSymbol *Symbol) { @@ -1392,9 +1337,7 @@ public: } /// Return internal section name for this function. - StringRef getCodeSectionName() const { - return StringRef(CodeSectionName); - } + StringRef getCodeSectionName() const { return StringRef(CodeSectionName); } /// Assign a code section name to the function. void setCodeSectionName(StringRef Name) { @@ -1422,98 +1365,65 @@ public: } /// Return true iif the function will halt execution on entry. - bool trapsOnEntry() const { - return TrapsOnEntry; - } + bool trapsOnEntry() const { return TrapsOnEntry; } /// Make the function always trap on entry. Other than the trap instruction, /// the function body will be empty. void setTrapOnEntry(); /// Return true if the function could be correctly processed. - bool isSimple() const { - return IsSimple; - } + bool isSimple() const { return IsSimple; } /// Return true if the function should be ignored for optimization purposes. - bool isIgnored() const { - return IsIgnored; - } + bool isIgnored() const { return IsIgnored; } /// Return true if the function should not be disassembled, emitted, or /// otherwise processed. - bool isPseudo() const { - return IsPseudo; - } + bool isPseudo() const { return IsPseudo; } /// Return true if the function contains a jump table with entries pointing /// to split fragments. - bool hasSplitJumpTable() const { - return HasSplitJumpTable; - } + bool hasSplitJumpTable() const { return HasSplitJumpTable; } /// Return true if all CFG edges have local successors. - bool hasCanonicalCFG() const { - return HasCanonicalCFG; - } + bool hasCanonicalCFG() const { return HasCanonicalCFG; } /// Return true if the original function code has all necessary relocations /// to track addresses of functions emitted to new locations. - bool hasExternalRefRelocations() const { - return HasExternalRefRelocations; - } + bool hasExternalRefRelocations() const { return HasExternalRefRelocations; } /// Return true if the function has instruction(s) with unknown control flow. - bool hasUnknownControlFlow() const { - return HasUnknownControlFlow; - } + bool hasUnknownControlFlow() const { return HasUnknownControlFlow; } /// Return true if the function body is non-contiguous. bool isSplit() const { - return isSimple() && - layout_size() && + return isSimple() && layout_size() && layout_front()->isCold() != layout_back()->isCold(); } /// Return true if the function has exception handling tables. - bool hasEHRanges() const { - return HasEHRanges; - } + bool hasEHRanges() const { return HasEHRanges; } /// Return true if the function uses DW_CFA_GNU_args_size CFIs. - bool usesGnuArgsSize() const { - return UsesGnuArgsSize; - } + bool usesGnuArgsSize() const { return UsesGnuArgsSize; } /// Return true if the function has more than one entry point. - bool isMultiEntry() const { - return !SecondaryEntryPoints.empty(); - } + bool isMultiEntry() const { return !SecondaryEntryPoints.empty(); } /// Return true if the function might have a profile available externally, /// but not yet populated into the function. - bool hasProfileAvailable() const { - return HasProfileAvailable; - } + bool hasProfileAvailable() const { return HasProfileAvailable; } - bool hasMemoryProfile() const { - return HasMemoryProfile; - } + bool hasMemoryProfile() const { return HasMemoryProfile; } /// Return true if the body of the function was merged into another function. - bool isFolded() const { - return FoldedIntoFunction != nullptr; - } + bool isFolded() const { return FoldedIntoFunction != nullptr; } /// If this function was folded, return the function it was folded into. - BinaryFunction *getFoldedIntoFunction() const { - return FoldedIntoFunction; - } + BinaryFunction *getFoldedIntoFunction() const { return FoldedIntoFunction; } /// Return true if the function uses jump tables. - bool hasJumpTables() const { - return !JumpTables.empty(); - } + bool hasJumpTables() const { return !JumpTables.empty(); } /// Return true if the function has SDT marker bool hasSDTMarker() const { return HasSDTMarker; } @@ -1522,9 +1432,7 @@ public: bool hasPseudoProbe() const { return HasPseudoProbe; } /// Return true if the original entry point was patched. - bool isPatched() const { - return IsPatched; - } + bool isPatched() const { return IsPatched; } const JumpTable *getJumpTable(const MCInst &Inst) const { const uint64_t Address = BC.MIB->getJumpTable(Inst); @@ -1536,21 +1444,15 @@ public: return getJumpTableContainingAddress(Address); } - const MCSymbol *getPersonalityFunction() const { - return PersonalityFunction; - } + const MCSymbol *getPersonalityFunction() const { return PersonalityFunction; } - uint8_t getPersonalityEncoding() const { - return PersonalityEncoding; - } + uint8_t getPersonalityEncoding() const { return PersonalityEncoding; } const CallSitesType &getCallSites() const { return CallSites; } const CallSitesType &getColdCallSites() const { return ColdCallSites; } - const ArrayRef getLSDAActionTable() const { - return LSDAActionTable; - } + const ArrayRef getLSDAActionTable() const { return LSDAActionTable; } const LSDATypeTableTy &getLSDATypeTable() const { return LSDATypeTable; } @@ -1562,9 +1464,7 @@ public: return LSDATypeIndexTable; } - const LabelsMapType &getLabels() const { - return Labels; - } + const LabelsMapType &getLabels() const { return Labels; } IslandInfo &getIslandInfo() { assert(Islands && "function expected to have constant islands"); @@ -1582,9 +1482,7 @@ public: } /// Return unique number associated with the function. - uint64_t getFunctionNumber() const { - return FunctionNumber; - } + uint64_t getFunctionNumber() const { return FunctionNumber; } /// Return true if the given address \p PC is inside the function body. bool containsAddress(uint64_t PC, bool UseMaxSize = false) const { @@ -1600,8 +1498,7 @@ public: /// The new block is not inserted into the CFG. The client must /// use insertBasicBlocks to add any new blocks to the CFG. std::unique_ptr - createBasicBlock(uint64_t Offset, - MCSymbol *Label = nullptr, + createBasicBlock(uint64_t Offset, MCSymbol *Label = nullptr, bool DeriveAlignment = false) { assert(BC.Ctx && "cannot be called with empty context"); if (!Label) { @@ -1609,7 +1506,7 @@ public: Label = BC.Ctx->createNamedTempSymbol("BB"); } auto BB = std::unique_ptr( - new BinaryBasicBlock(this, Label, Offset)); + new BinaryBasicBlock(this, Label, Offset)); if (DeriveAlignment) { uint64_t DerivedAlignment = Offset & (1 + ~Offset); @@ -1651,8 +1548,7 @@ public: } assert(CurrentState == State::CFG || - (std::is_sorted(BasicBlockOffsets.begin(), - BasicBlockOffsets.end(), + (std::is_sorted(BasicBlockOffsets.begin(), BasicBlockOffsets.end(), CompareBasicBlockOffsets()) && std::is_sorted(begin(), end()))); @@ -1684,24 +1580,22 @@ public: /// BB offsets and BB indices. The new BBs are inserted after Start. /// This operation could affect fallthrough branches for Start. /// - void insertBasicBlocks( - BinaryBasicBlock *Start, - std::vector> &&NewBBs, - const bool UpdateLayout = true, - const bool UpdateCFIState = true, - const bool RecomputeLandingPads = true); + void + insertBasicBlocks(BinaryBasicBlock *Start, + std::vector> &&NewBBs, + const bool UpdateLayout = true, + const bool UpdateCFIState = true, + const bool RecomputeLandingPads = true); iterator insertBasicBlocks( - iterator StartBB, - std::vector> &&NewBBs, - const bool UpdateLayout = true, - const bool UpdateCFIState = true, - const bool RecomputeLandingPads = true); + iterator StartBB, std::vector> &&NewBBs, + const bool UpdateLayout = true, const bool UpdateCFIState = true, + const bool RecomputeLandingPads = true); /// Update the basic block layout for this function. The BBs from /// [Start->Index, Start->Index + NumNewBlocks) are inserted into the /// layout after the BB indicated by Start. - void updateLayout(BinaryBasicBlock* Start, const unsigned NumNewBlocks); + void updateLayout(BinaryBasicBlock *Start, const unsigned NumNewBlocks); /// Make sure basic blocks' indices match the current layout. void updateLayoutIndices() const { @@ -1732,8 +1626,7 @@ public: /// Change \p OrigDest to \p NewDest in the jump table used at the end of /// \p BB. Returns false if \p OrigDest couldn't be find as a valid target /// and no replacement took place. - bool replaceJumpTableEntryIn(BinaryBasicBlock *BB, - BinaryBasicBlock *OldDest, + bool replaceJumpTableEntryIn(BinaryBasicBlock *BB, BinaryBasicBlock *OldDest, BinaryBasicBlock *NewDest); /// Split the CFG edge by inserting an intermediate basic block. @@ -1778,14 +1671,11 @@ public: /// Return true if function has a profile, even if the profile does not /// match CFG 100%. - bool hasProfile() const { - return ExecutionCount != COUNT_NO_PROFILE; - } + bool hasProfile() const { return ExecutionCount != COUNT_NO_PROFILE; } /// Return true if function profile is present and accurate. bool hasValidProfile() const { - return ExecutionCount != COUNT_NO_PROFILE && - ProfileMatchRatio == 1.0f; + return ExecutionCount != COUNT_NO_PROFILE && ProfileMatchRatio == 1.0f; } /// Mark this function as having a valid profile. @@ -1797,9 +1687,7 @@ public: } /// Return flags describing a profile for this function. - uint16_t getProfileFlags() const { - return ProfileFlags; - } + uint16_t getProfileFlags() const { return ProfileFlags; } void addCFIInstruction(uint64_t Offset, MCCFIInstruction &&Inst) { assert(!Instructions.empty()); @@ -1904,9 +1792,7 @@ public: return *this; } - void setPseudo(bool Pseudo) { - IsPseudo = Pseudo; - } + void setPseudo(bool Pseudo) { IsPseudo = Pseudo; } BinaryFunction &setUsesGnuArgsSize(bool Uses = true) { UsesGnuArgsSize = Uses; @@ -1921,21 +1807,13 @@ public: /// Mark function that should not be emitted. void setIgnored(); - void setIsPatched(bool V) { - IsPatched = V; - } + void setIsPatched(bool V) { IsPatched = V; } - void setHasSplitJumpTable(bool V) { - HasSplitJumpTable = V; - } + void setHasSplitJumpTable(bool V) { HasSplitJumpTable = V; } - void setHasCanonicalCFG(bool V) { - HasCanonicalCFG = V; - } + void setHasCanonicalCFG(bool V) { HasCanonicalCFG = V; } - void setFolded(BinaryFunction *BF) { - FoldedIntoFunction = BF; - } + void setFolded(BinaryFunction *BF) { FoldedIntoFunction = BF; } BinaryFunction &setPersonalityFunction(uint64_t Addr) { assert(!PersonalityFunction && "can't set personality function twice"); @@ -1953,27 +1831,21 @@ public: return *this; } - uint16_t getAlignment() const { - return Alignment; - } + uint16_t getAlignment() const { return Alignment; } BinaryFunction &setMaxAlignmentBytes(uint16_t MaxAlignBytes) { MaxAlignmentBytes = MaxAlignBytes; return *this; } - uint16_t getMaxAlignmentBytes() const { - return MaxAlignmentBytes; - } + uint16_t getMaxAlignmentBytes() const { return MaxAlignmentBytes; } BinaryFunction &setMaxColdAlignmentBytes(uint16_t MaxAlignBytes) { MaxColdAlignmentBytes = MaxAlignBytes; return *this; } - uint16_t getMaxColdAlignmentBytes() const { - return MaxColdAlignmentBytes; - } + uint16_t getMaxColdAlignmentBytes() const { return MaxColdAlignmentBytes; } BinaryFunction &setImageAddress(uint64_t Address) { ImageAddress = Address; @@ -1981,9 +1853,7 @@ public: } /// Return the address of this function' image in memory. - uint64_t getImageAddress() const { - return ImageAddress; - } + uint64_t getImageAddress() const { return ImageAddress; } BinaryFunction &setImageSize(uint64_t Size) { ImageSize = Size; @@ -1991,14 +1861,10 @@ public: } /// Return the size of this function' image in memory. - uint64_t getImageSize() const { - return ImageSize; - } + uint64_t getImageSize() const { return ImageSize; } /// Return true if the function is a secondary fragment of another function. - bool isFragment() const { - return IsFragment; - } + bool isFragment() const { return IsFragment; } /// Returns if the given function is a parent fragment of this function. bool isParentFragment(BinaryFunction *Parent) const { @@ -2034,9 +1900,7 @@ public: /// the function was executed. /// /// Return COUNT_NO_PROFILE if there's no profile info. - uint64_t getExecutionCount() const { - return ExecutionCount; - } + uint64_t getExecutionCount() const { return ExecutionCount; } /// Return the raw profile information about the number of branch /// executions corresponding to this function. @@ -2049,9 +1913,7 @@ public: } /// Return original LSDA address for the function or NULL. - uint64_t getLSDAAddress() const { - return LSDAAddress; - } + uint64_t getLSDAAddress() const { return LSDAAddress; } /// Return symbol pointing to function's LSDA. MCSymbol *getLSDASymbol() { @@ -2060,9 +1922,8 @@ public: if (CallSites.empty()) return nullptr; - LSDASymbol = - BC.Ctx->getOrCreateSymbol(Twine("GCC_except_table") + - Twine::utohexstr(getFunctionNumber())); + LSDASymbol = BC.Ctx->getOrCreateSymbol( + Twine("GCC_except_table") + Twine::utohexstr(getFunctionNumber())); return LSDASymbol; } @@ -2074,9 +1935,8 @@ public: if (ColdCallSites.empty()) return nullptr; - ColdLSDASymbol = - BC.Ctx->getOrCreateSymbol(Twine("GCC_cold_except_table") + - Twine::utohexstr(getFunctionNumber())); + ColdLSDASymbol = BC.Ctx->getOrCreateSymbol( + Twine("GCC_cold_except_table") + Twine::utohexstr(getFunctionNumber())); return ColdLSDASymbol; } @@ -2086,21 +1946,15 @@ public: bool isDataMarker(const SymbolRef &Symbol, uint64_t SymbolSize) const; bool isCodeMarker(const SymbolRef &Symbol, uint64_t SymbolSize) const; - void setOutputDataAddress(uint64_t Address) { - OutputDataOffset = Address; - } + void setOutputDataAddress(uint64_t Address) { OutputDataOffset = Address; } - uint64_t getOutputDataAddress() const { - return OutputDataOffset; - } + uint64_t getOutputDataAddress() const { return OutputDataOffset; } void setOutputColdDataAddress(uint64_t Address) { OutputColdDataOffset = Address; } - uint64_t getOutputColdDataAddress() const { - return OutputColdDataOffset; - } + uint64_t getOutputColdDataAddress() const { return OutputColdDataOffset; } /// If \p Address represents an access to a constant island managed by this /// function, return a symbol so code can safely refer to it. Otherwise, @@ -2135,17 +1989,16 @@ public: /// island symbols of this function. We create a proxy for it, so we emit /// separate symbols when emitting our constant island on behalf of this other /// function. - MCSymbol * - getOrCreateProxyIslandAccess(uint64_t Address, BinaryFunction &Referrer) { + MCSymbol *getOrCreateProxyIslandAccess(uint64_t Address, + BinaryFunction &Referrer) { MCSymbol *Symbol = getOrCreateIslandAccess(Address); if (!Symbol) return nullptr; MCSymbol *Proxy; if (!Islands->Proxies[&Referrer].count(Symbol)) { - Proxy = - BC.Ctx->getOrCreateSymbol(Symbol->getName() + - ".proxy.for." + Referrer.getPrintName()); + Proxy = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".proxy.for." + + Referrer.getPrintName()); Islands->Proxies[&Referrer][Symbol] = Proxy; Islands->Proxies[&Referrer][Proxy] = Symbol; } @@ -2224,9 +2077,7 @@ public: return Size; } - bool hasIslandsInfo() const { - return !!Islands; - } + bool hasIslandsInfo() const { return !!Islands; } bool hasConstantIsland() const { return Islands && !Islands->DataOffsets.empty(); @@ -2301,9 +2152,7 @@ public: bool postProcessIndirectBranches(MCPlusBuilder::AllocatorIdTy AllocId); /// Return all call site profile info for this function. - IndirectCallSiteProfile &getAllCallSites() { - return AllCallSites; - } + IndirectCallSiteProfile &getAllCallSites() { return AllCallSites; } const IndirectCallSiteProfile &getAllCallSites() const { return AllCallSites; @@ -2343,9 +2192,7 @@ public: /// Get the number of instructions within this function. uint64_t getInstructionCount() const; - const CFIInstrMapType &getFDEProgram() const { - return FrameInstructions; - } + const CFIInstrMapType &getFDEProgram() const { return FrameInstructions; } void moveRememberRestorePair(BinaryBasicBlock *BB); @@ -2387,9 +2234,7 @@ public: void fixBranches(); /// Mark function as finalized. No further optimizations are permitted. - void setFinalized() { - CurrentState = State::CFG_Finalized; - } + void setFinalized() { CurrentState = State::CFG_Finalized; } void setEmitted(bool KeepCFG = false) { CurrentState = State::EmittedCFG; @@ -2416,12 +2261,10 @@ public: void mergeProfileDataInto(BinaryFunction &BF) const; /// Returns the last computed hash value of the function. - size_t getHash() const { - return Hash; - } + size_t getHash() const { return Hash; } using OperandHashFuncTy = - function_ref; + function_ref; /// Compute the hash value of the function based on its contents. /// @@ -2431,18 +2274,16 @@ public: /// By default, instruction operands are ignored while calculating the hash. /// The caller can change this via passing \p OperandHashFunc function. /// The return result of this function will be mixed with internal hash. - size_t computeHash(bool UseDFS = false, - OperandHashFuncTy OperandHashFunc = - [](const MCOperand&) { return std::string(); }) const; + size_t computeHash( + bool UseDFS = false, + OperandHashFuncTy OperandHashFunc = [](const MCOperand &) { + return std::string(); + }) const; - void setDWARFUnit(DWARFUnit *Unit) { - DwarfUnit = Unit; - } + void setDWARFUnit(DWARFUnit *Unit) { DwarfUnit = Unit; } /// Return DWARF compile unit for this function. - DWARFUnit *getDWARFUnit() const { - return DwarfUnit; - } + DWARFUnit *getDWARFUnit() const { return DwarfUnit; } /// Return line info table for this function. const DWARFDebugLine::LineTable *getDWARFLineTable() const { @@ -2514,8 +2355,8 @@ public: /// Similar to translateInputToOutputRanges() but operates on location lists /// and moves associated data to output location lists. - DebugLocationsVector translateInputToOutputLocationList( - const DebugLocationsVector &InputLL) const; + DebugLocationsVector + translateInputToOutputLocationList(const DebugLocationsVector &InputLL) const; /// Return true if the function is an AArch64 linker inserted veneer bool isAArch64Veneer() const; @@ -2529,6 +2370,7 @@ public: uint64_t ImageAddress{0}; uint64_t ImageSize{0}; uint64_t FileOffset{0}; + public: uint64_t getAddress() const { return Address; } uint64_t getImageAddress() const { return ImageAddress; } @@ -2555,26 +2397,12 @@ inline raw_ostream &operator<<(raw_ostream &OS, return OS; } -inline raw_ostream &operator<<(raw_ostream &OS, - const BinaryFunction::State State) { - switch (State) { - case BinaryFunction::State::Empty: OS << "empty"; break; - case BinaryFunction::State::Disassembled: OS << "disassembled"; break; - case BinaryFunction::State::CFG: OS << "CFG constructed"; break; - case BinaryFunction::State::CFG_Finalized:OS << "CFG finalized"; break; - case BinaryFunction::State::EmittedCFG: OS << "emitted with CFG"; break; - case BinaryFunction::State::Emitted: OS << "emitted"; break; - } - - return OS; -} - } // namespace bolt - // GraphTraits specializations for function basic block graphs (CFGs) -template <> struct GraphTraits : - public GraphTraits { +template <> +struct GraphTraits + : public GraphTraits { static NodeRef getEntryNode(bolt::BinaryFunction *F) { return *F->layout_begin(); } @@ -2589,13 +2417,12 @@ template <> struct GraphTraits : llvm_unreachable("Not implemented"); return nodes_iterator(F->end()); } - static size_t size(bolt::BinaryFunction *F) { - return F->size(); - } + static size_t size(bolt::BinaryFunction *F) { return F->size(); } }; -template <> struct GraphTraits : - public GraphTraits { +template <> +struct GraphTraits + : public GraphTraits { static NodeRef getEntryNode(const bolt::BinaryFunction *F) { return *F->layout_begin(); } @@ -2610,20 +2437,20 @@ template <> struct GraphTraits : llvm_unreachable("Not implemented"); return nodes_iterator(F->end()); } - static size_t size(const bolt::BinaryFunction *F) { - return F->size(); - } + static size_t size(const bolt::BinaryFunction *F) { return F->size(); } }; -template <> struct GraphTraits> : - public GraphTraits> { +template <> +struct GraphTraits> + : public GraphTraits> { static NodeRef getEntryNode(Inverse G) { return *G.Graph->layout_begin(); } }; -template <> struct GraphTraits> : - public GraphTraits> { +template <> +struct GraphTraits> + : public GraphTraits> { static NodeRef getEntryNode(Inverse G) { return *G.Graph->layout_begin(); } diff --git a/bolt/include/bolt/Core/BinaryLoop.h b/bolt/include/bolt/Core/BinaryLoop.h index d9fdf09f2fed..ae810dfd6ef7 100644 --- a/bolt/include/bolt/Core/BinaryLoop.h +++ b/bolt/include/bolt/Core/BinaryLoop.h @@ -25,11 +25,11 @@ namespace bolt { class BinaryBasicBlock; using BinaryDomTreeNode = DomTreeNodeBase; -using BinaryDominatorTree = DomTreeBase; +using BinaryDominatorTree = DomTreeBase; class BinaryLoop : public LoopBase { public: - BinaryLoop() : LoopBase() { } + BinaryLoop() : LoopBase() {} // The total count of all the back edges of this loop. uint64_t TotalBackEdgeCount{0}; @@ -44,13 +44,13 @@ public: protected: friend class LoopInfoBase; - explicit BinaryLoop(BinaryBasicBlock *BB) : - LoopBase(BB) { } + explicit BinaryLoop(BinaryBasicBlock *BB) + : LoopBase(BB) {} }; class BinaryLoopInfo : public LoopInfoBase { public: - BinaryLoopInfo() { } + BinaryLoopInfo() {} unsigned OuterLoops{0}; unsigned TotalLoops{0}; @@ -65,16 +65,19 @@ public: namespace llvm { // BinaryDominatorTree GraphTraits specializations. -template <> struct GraphTraits - : public DomTreeGraphTraitsBase {}; +template <> +struct GraphTraits + : public DomTreeGraphTraitsBase {}; -template <> struct GraphTraits - : public DomTreeGraphTraitsBase {}; +template <> +struct GraphTraits + : public DomTreeGraphTraitsBase {}; -template <> struct GraphTraits - : public GraphTraits { +template <> +struct GraphTraits + : public GraphTraits { static NodeRef getEntryNode(bolt::BinaryDominatorTree *DT) { return DT->getRootNode(); } @@ -88,6 +91,6 @@ template <> struct GraphTraits } }; -} // namescpae llvm +} // namespace llvm #endif diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h index 8d0c973c7a5e..4914f871bdf4 100644 --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -39,16 +39,16 @@ class BinaryData; class BinarySection { friend class BinaryContext; - BinaryContext &BC; // Owning BinaryContext - std::string Name; // Section name - const SectionRef Section; // SectionRef (may be null) - StringRef Contents; // Input section contents - const uint64_t Address; // Address of section in input binary (may be 0) - const uint64_t Size; // Input section size - uint64_t InputFileOffset{0};// Offset in the input binary - unsigned Alignment; // alignment in bytes (must be > 0) - unsigned ELFType; // ELF section type - unsigned ELFFlags; // ELF section flags + BinaryContext &BC; // Owning BinaryContext + std::string Name; // Section name + const SectionRef Section; // SectionRef (may be null) + StringRef Contents; // Input section contents + const uint64_t Address; // Address of section in input binary (may be 0) + const uint64_t Size; // Input section size + uint64_t InputFileOffset{0}; // Offset in the input binary + unsigned Alignment; // alignment in bytes (must be > 0) + unsigned ELFType; // ELF section type + unsigned ELFFlags; // ELF section flags // Relocations associated with this section. Relocation offsets are // wrt. to the original section address and size. @@ -67,7 +67,7 @@ class BinarySection { SmallString<8> Bytes; BinaryPatch(uint64_t Offset, const SmallVectorImpl &Bytes) - : Offset(Offset), Bytes(Bytes.begin(), Bytes.end()) {} + : Offset(Offset), Bytes(Bytes.begin(), Bytes.end()) {} }; std::vector Patches; /// Patcher used to apply simple changes to sections of the input binary. @@ -122,49 +122,34 @@ class BinarySection { RelocationSetType reorderRelocations(bool Inplace) const; /// Set output info for this section. - void update(uint8_t *NewData, - uint64_t NewSize, - unsigned NewAlignment, - unsigned NewELFType, - unsigned NewELFFlags) { + void update(uint8_t *NewData, uint64_t NewSize, unsigned NewAlignment, + unsigned NewELFType, unsigned NewELFFlags) { assert(NewAlignment > 0 && "section alignment must be > 0"); Alignment = NewAlignment; ELFType = NewELFType; ELFFlags = NewELFFlags; OutputSize = NewSize; - OutputContents = StringRef(reinterpret_cast(NewData), + OutputContents = StringRef(reinterpret_cast(NewData), NewData ? NewSize : 0); IsFinalized = true; } + public: /// Copy a section. - explicit BinarySection(BinaryContext &BC, - StringRef Name, + explicit BinarySection(BinaryContext &BC, StringRef Name, const BinarySection &Section) - : BC(BC), - Name(Name), - Section(Section.getSectionRef()), - Contents(Section.getContents()), - Address(Section.getAddress()), - Size(Section.getSize()), - Alignment(Section.getAlignment()), - ELFType(Section.getELFType()), - ELFFlags(Section.getELFFlags()), - Relocations(Section.Relocations), - PendingRelocations(Section.PendingRelocations), - OutputName(Name) { - } + : BC(BC), Name(Name), Section(Section.getSectionRef()), + Contents(Section.getContents()), Address(Section.getAddress()), + Size(Section.getSize()), Alignment(Section.getAlignment()), + ELFType(Section.getELFType()), ELFFlags(Section.getELFFlags()), + Relocations(Section.Relocations), + PendingRelocations(Section.PendingRelocations), OutputName(Name) {} - BinarySection(BinaryContext &BC, - SectionRef Section) - : BC(BC), - Name(getName(Section)), - Section(Section), - Contents(getContents(Section)), - Address(Section.getAddress()), - Size(Section.getSize()), - Alignment(Section.getAlignment()), - OutputName(Name) { + BinarySection(BinaryContext &BC, SectionRef Section) + : BC(BC), Name(getName(Section)), Section(Section), + Contents(getContents(Section)), Address(Section.getAddress()), + Size(Section.getSize()), Alignment(Section.getAlignment()), + OutputName(Name) { if (isELF()) { ELFType = ELFSectionRef(Section).getType(); ELFFlags = ELFSectionRef(Section).getFlags(); @@ -178,33 +163,20 @@ public: } // TODO: pass Data as StringRef/ArrayRef? use StringRef::copy method. - BinarySection(BinaryContext &BC, - StringRef Name, - uint8_t *Data, - uint64_t Size, - unsigned Alignment, - unsigned ELFType, - unsigned ELFFlags) - : BC(BC), - Name(Name), - Contents(reinterpret_cast(Data), Data ? Size : 0), - Address(0), - Size(Size), - Alignment(Alignment), - ELFType(ELFType), - ELFFlags(ELFFlags), - IsFinalized(true), - OutputName(Name), - OutputSize(Size), - OutputContents(Contents) { + BinarySection(BinaryContext &BC, StringRef Name, uint8_t *Data, uint64_t Size, + unsigned Alignment, unsigned ELFType, unsigned ELFFlags) + : BC(BC), Name(Name), + Contents(reinterpret_cast(Data), Data ? Size : 0), + Address(0), Size(Size), Alignment(Alignment), ELFType(ELFType), + ELFFlags(ELFFlags), IsFinalized(true), OutputName(Name), + OutputSize(Size), OutputContents(Contents) { assert(Alignment > 0 && "section alignment must be > 0"); } ~BinarySection(); /// Helper function to generate the proper ELF flags from section properties. - static unsigned getFlags(bool IsReadOnly = true, - bool IsText = false, + static unsigned getFlags(bool IsReadOnly = true, bool IsText = false, bool IsAllocatable = false) { unsigned Flags = 0; if (IsAllocatable) @@ -216,17 +188,12 @@ public: return Flags; } - operator bool() const { - return ELFType != ELF::SHT_NULL; - } + operator bool() const { return ELFType != ELF::SHT_NULL; } bool operator==(const BinarySection &Other) const { - return (Name == Other.Name && - Address == Other.Address && - Size == Other.Size && - getData() == Other.getData() && - Alignment == Other.Alignment && - ELFType == Other.ELFType && + return (Name == Other.Name && Address == Other.Address && + Size == Other.Size && getData() == Other.getData() && + Alignment == Other.Alignment && ELFType == Other.ELFType && ELFFlags == Other.ELFFlags); } @@ -239,8 +206,7 @@ public: return (getAddress() < Other.getAddress() || (getAddress() == Other.getAddress() && (getSize() < Other.getSize() || - (getSize() == Other.getSize() && - getName() < Other.getName())))); + (getSize() == Other.getSize() && getName() < Other.getName())))); } /// @@ -270,17 +236,12 @@ public: return (ELFType == ELF::SHT_NOBITS && (ELFFlags & (ELF::SHF_ALLOC | ELF::SHF_WRITE))); } - bool isTLS() const { - return (ELFFlags & ELF::SHF_TLS); - } - bool isTBSS() const { - return isBSS() && isTLS(); - } + bool isTLS() const { return (ELFFlags & ELF::SHF_TLS); } + bool isTBSS() const { return isBSS() && isTLS(); } bool isVirtual() const { return ELFType == ELF::SHT_NOBITS; } bool isRela() const { return ELFType == ELF::SHT_RELA; } bool isReadOnly() const { - return ((ELFFlags & ELF::SHF_ALLOC) && - !(ELFFlags & ELF::SHF_WRITE) && + return ((ELFFlags & ELF::SHF_ALLOC) && !(ELFFlags & ELF::SHF_WRITE) && ELFType == ELF::SHT_PROGBITS); } bool isAllocatable() const { @@ -297,7 +258,8 @@ public: unsigned getELFFlags() const { return ELFFlags; } uint8_t *getData() { - return reinterpret_cast(const_cast(getContents().data())); + return reinterpret_cast( + const_cast(getContents().data())); } const uint8_t *getData() const { return reinterpret_cast(getContents().data()); @@ -331,14 +293,10 @@ public: } /// Does this section have any non-pending relocations? - bool hasRelocations() const { - return !Relocations.empty(); - } + bool hasRelocations() const { return !Relocations.empty(); } /// Does this section have any pending relocations? - bool hasPendingRelocations() const { - return !PendingRelocations.empty(); - } + bool hasPendingRelocations() const { return !PendingRelocations.empty(); } /// Remove non-pending relocation with the given /p Offset. bool removeRelocationAt(uint64_t Offset) { @@ -353,11 +311,8 @@ public: void clearRelocations(); /// Add a new relocation at the given /p Offset. - void addRelocation(uint64_t Offset, - MCSymbol *Symbol, - uint64_t Type, - uint64_t Addend, - uint64_t Value = 0, + void addRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, + uint64_t Addend, uint64_t Value = 0, bool Pending = false) { assert(Offset < getSize() && "offset not within section bounds"); if (!Pending) { @@ -369,11 +324,8 @@ public: } /// Add a dynamic relocation at the given /p Offset. - void addDynamicRelocation(uint64_t Offset, - MCSymbol *Symbol, - uint64_t Type, - uint64_t Addend, - uint64_t Value = 0) { + void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, + uint64_t Addend, uint64_t Value = 0) { assert(Offset < getSize() && "offset not within section bounds"); DynamicRelocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); } @@ -423,7 +375,8 @@ public: StringRef getOutputName() const { return OutputName; } uint64_t getOutputSize() const { return OutputSize; } uint8_t *getOutputData() { - return reinterpret_cast(const_cast(getOutputContents().data())); + return reinterpret_cast( + const_cast(getOutputContents().data())); } const uint8_t *getOutputData() const { return reinterpret_cast(getOutputContents().data()); @@ -438,33 +391,19 @@ public: assert(hasValidSectionID() && "trying to use uninitialized section id"); return SectionID; } - bool hasValidSectionID() const { - return SectionID != -1u; - } - uint32_t getIndex() const { - return Index; - } + bool hasValidSectionID() const { return SectionID != -1u; } + uint32_t getIndex() const { return Index; } // mutation - void setOutputAddress(uint64_t Address) { - OutputAddress = Address; - } - void setOutputFileOffset(uint64_t Offset) { - OutputFileOffset = Offset; - } + void setOutputAddress(uint64_t Address) { OutputAddress = Address; } + void setOutputFileOffset(uint64_t Offset) { OutputFileOffset = Offset; } void setSectionID(unsigned ID) { assert(!hasValidSectionID() && "trying to set section id twice"); SectionID = ID; } - void setIndex(uint32_t I) { - Index = I; - } - void setOutputName(StringRef Name) { - OutputName = std::string(Name); - } - void setAnonymous(bool Flag) { - IsAnonymous = Flag; - } + void setIndex(uint32_t I) { Index = I; } + void setOutputName(StringRef Name) { OutputName = std::string(Name); } + void setAnonymous(bool Flag) { IsAnonymous = Flag; } /// Emit the section as data, possibly with relocations. Use name \p NewName // for the section during emission if non-empty. @@ -491,10 +430,7 @@ public: uint32_t Type); /// Code for ELF notes written by producer 'BOLT' - enum { - NT_BOLT_BAT = 1, - NT_BOLT_INSTRUMENTATION_TABLES = 2 - }; + enum { NT_BOLT_BAT = 1, NT_BOLT_INSTRUMENTATION_TABLES = 2 }; }; inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) { @@ -504,12 +440,12 @@ inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) { } inline uint8_t *copyByteArray(StringRef Buffer) { - return copyByteArray(reinterpret_cast(Buffer.data()), + return copyByteArray(reinterpret_cast(Buffer.data()), Buffer.size()); } inline uint8_t *copyByteArray(ArrayRef Buffer) { - return copyByteArray(reinterpret_cast(Buffer.data()), + return copyByteArray(reinterpret_cast(Buffer.data()), Buffer.size()); } diff --git a/bolt/include/bolt/Core/DebugData.h b/bolt/include/bolt/Core/DebugData.h index 35b7cd014978..e802491befa7 100644 --- a/bolt/include/bolt/Core/DebugData.h +++ b/bolt/include/bolt/Core/DebugData.h @@ -80,7 +80,7 @@ struct DebugLineTableRowRef { bool operator==(const DebugLineTableRowRef &Rhs) const { return DwCompileUnitIndex == Rhs.DwCompileUnitIndex && - RowIndex == Rhs.RowIndex; + RowIndex == Rhs.RowIndex; } bool operator!=(const DebugLineTableRowRef &Rhs) const { @@ -158,9 +158,7 @@ public: void writeARangesSection(raw_svector_ostream &RangesStream) const; /// Resets the writer to a clear state. - void reset() { - CUAddressRanges.clear(); - } + void reset() { CUAddressRanges.clear(); } /// Map DWARFCompileUnit index to ranges. using CUAddressRangesType = std::map; diff --git a/bolt/include/bolt/Core/DynoStats.h b/bolt/include/bolt/Core/DynoStats.h index 8250447740ab..c531eeefc464 100644 --- a/bolt/include/bolt/Core/DynoStats.h +++ b/bolt/include/bolt/Core/DynoStats.h @@ -58,9 +58,8 @@ public: enum Category : uint8_t { DYNO_STATS }; #undef D - private: - uint64_t Stats[LAST_DYNO_STAT+1]; + uint64_t Stats[LAST_DYNO_STAT + 1]; bool PrintAArch64Stats; #define D(name, desc, ...) desc, @@ -75,8 +74,7 @@ public: } uint64_t &operator[](size_t I) { - assert(I > FIRST_DYNO_STAT && I < LAST_DYNO_STAT && - "index out of bounds"); + assert(I > FIRST_DYNO_STAT && I < LAST_DYNO_STAT && "index out of bounds"); return Stats[I]; } @@ -114,9 +112,7 @@ public: bool operator!=(const DynoStats &Other) const { return !operator==(Other); } bool lessThan(const DynoStats &Other, ArrayRef Keys) const; - static const char* Description(const Category C) { - return Desc[C]; - } + static const char *Description(const Category C) { return Desc[C]; } /// Maps instruction opcodes to: /// 1. Accumulated executed instruction counts. @@ -161,11 +157,8 @@ inline DynoStats getDynoStats(const FuncsType &Funcs) { /// Call a function with optional before and after dynostats printing. template -inline void -callWithDynoStats(FnType &&Func, - const FuncsType &Funcs, - StringRef Phase, - const bool Flag) { +inline void callWithDynoStats(FnType &&Func, const FuncsType &Funcs, + StringRef Phase, const bool Flag) { bool IsAArch64 = Funcs.begin()->second.getBinaryContext().isAArch64(); DynoStats DynoStatsBefore(IsAArch64); if (Flag) { @@ -177,8 +170,8 @@ callWithDynoStats(FnType &&Func, if (Flag) { const DynoStats DynoStatsAfter = getDynoStats(Funcs); const bool Changed = (DynoStatsAfter != DynoStatsBefore); - outs() << "BOLT-INFO: program-wide dynostats after running " - << Phase << (Changed ? "" : " (no change)") << ":\n\n" + outs() << "BOLT-INFO: program-wide dynostats after running " << Phase + << (Changed ? "" : " (no change)") << ":\n\n" << DynoStatsBefore << '\n'; if (Changed) { DynoStatsAfter.print(outs(), &DynoStatsBefore); diff --git a/bolt/include/bolt/Core/Exceptions.h b/bolt/include/bolt/Core/Exceptions.h index ca64c2aea3c0..c4b4819d9d91 100644 --- a/bolt/include/bolt/Core/Exceptions.h +++ b/bolt/include/bolt/Core/Exceptions.h @@ -44,17 +44,15 @@ public: /// /// \p EHFrameHeaderAddress specifies location of .eh_frame_hdr, /// and is required for relative addressing used in the section. - std::vector generateEHFrameHeader( - const DWARFDebugFrame &OldEHFrame, - const DWARFDebugFrame &NewEHFrame, - uint64_t EHFrameHeaderAddress, - std::vector &FailedAddresses) const; + std::vector + generateEHFrameHeader(const DWARFDebugFrame &OldEHFrame, + const DWARFDebugFrame &NewEHFrame, + uint64_t EHFrameHeaderAddress, + std::vector &FailedAddresses) const; using FDEsMap = std::map; - const FDEsMap &getFDEs() const { - return FDEs; - } + const FDEsMap &getFDEs() const { return FDEs; } private: FDEsMap FDEs; diff --git a/bolt/include/bolt/Core/JumpTable.h b/bolt/include/bolt/Core/JumpTable.h index 9bb83fe96fe9..f47e7c15d049 100644 --- a/bolt/include/bolt/Core/JumpTable.h +++ b/bolt/include/bolt/Core/JumpTable.h @@ -88,12 +88,8 @@ public: private: /// Constructor should only be called by a BinaryContext. - JumpTable(MCSymbol &Symbol, - uint64_t Address, - size_t EntrySize, - JumpTableType Type, - LabelMapType &&Labels, - BinaryFunction &BF, + JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize, + JumpTableType Type, LabelMapType &&Labels, BinaryFunction &BF, BinarySection &Section); public: diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h index 15dd0bd751d7..b82061c0c89e 100644 --- a/bolt/include/bolt/Core/MCPlus.h +++ b/bolt/include/bolt/Core/MCPlus.h @@ -68,8 +68,10 @@ public: virtual void print(raw_ostream &OS) const = 0; virtual bool equals(const MCAnnotation &) const = 0; virtual ~MCAnnotation() {} + protected: MCAnnotation() {} + private: // noncopyable MCAnnotation(const MCAnnotation &Other) = delete; @@ -82,19 +84,15 @@ private: /// only be freed if the annotation is removed with the /// MCPlusBuilder::removeAnnotation method. This is because all /// annotations are arena allocated. -template -class MCSimpleAnnotation : public MCAnnotation { +template class MCSimpleAnnotation : public MCAnnotation { public: ValueType &getValue() { return Value; } bool equals(const MCAnnotation &Other) const override { return Value == static_cast(Other).Value; } - explicit MCSimpleAnnotation(const ValueType &Val) - : Value(Val) {} + explicit MCSimpleAnnotation(const ValueType &Val) : Value(Val) {} - void print(raw_ostream &OS) const override { - OS << Value; - } + void print(raw_ostream &OS) const override { OS << Value; } private: ValueType Value; @@ -104,8 +102,8 @@ private: /// annotations. inline unsigned getNumPrimeOperands(const MCInst &Inst) { if (Inst.getNumOperands() > 0 && std::prev(Inst.end())->isInst()) { - assert(std::prev(Inst.end())-> - getInst()->getOpcode() == TargetOpcode::ANNOTATION_LABEL); + assert(std::prev(Inst.end())->getInst()->getOpcode() == + TargetOpcode::ANNOTATION_LABEL); return Inst.getNumOperands() - 1; } return Inst.getNumOperands(); diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 6197ce2d297f..1b4ff50bb739 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -130,8 +130,8 @@ private: AnnotationInst->addOperand(MCOperand::createImm(AnnotationValue)); } - Optional - getAnnotationOpValue(const MCInst &Inst, unsigned Index) const { + Optional getAnnotationOpValue(const MCInst &Inst, + unsigned Index) const { const MCInst *AnnotationInst = getAnnotationInst(Inst); if (!AnnotationInst) return NoneType(); @@ -163,7 +163,7 @@ protected: public: class InstructionIterator - : public std::iterator { + : public std::iterator { public: class Impl { public: @@ -172,36 +172,29 @@ public: virtual void Prev() = 0; virtual MCInst &Deref() = 0; virtual bool Compare(const Impl &Other) const = 0; - virtual ~Impl() { } + virtual ~Impl() {} }; - template - class SeqImpl : public Impl { + template class SeqImpl : public Impl { public: - virtual Impl *Copy() const override { - return new SeqImpl(Itr); - } + virtual Impl *Copy() const override { return new SeqImpl(Itr); } virtual void Next() override { ++Itr; } virtual void Prev() override { --Itr; } - virtual MCInst &Deref() override { - return const_cast(*Itr); - } + virtual MCInst &Deref() override { return const_cast(*Itr); } virtual bool Compare(const Impl &Other) const override { // assumes that Other is same underlying type - return Itr == static_cast&>(Other).Itr; + return Itr == static_cast &>(Other).Itr; } - explicit SeqImpl(T &&Itr) : Itr(std::move(Itr)) { } - explicit SeqImpl(const T &Itr) : Itr(Itr) { } + explicit SeqImpl(T &&Itr) : Itr(std::move(Itr)) {} + explicit SeqImpl(const T &Itr) : Itr(Itr) {} + private: T Itr; }; - template - class MapImpl : public Impl { + template class MapImpl : public Impl { public: - virtual Impl *Copy() const override { - return new MapImpl(Itr); - } + virtual Impl *Copy() const override { return new MapImpl(Itr); } virtual void Next() override { ++Itr; } virtual void Prev() override { --Itr; } virtual MCInst &Deref() override { @@ -209,10 +202,11 @@ public: } virtual bool Compare(const Impl &Other) const override { // assumes that Other is same underlying type - return Itr == static_cast&>(Other).Itr; + return Itr == static_cast &>(Other).Itr; } - explicit MapImpl(T &&Itr) : Itr(std::move(Itr)) { } - explicit MapImpl(const T &Itr) : Itr(Itr) { } + explicit MapImpl(T &&Itr) : Itr(std::move(Itr)) {} + explicit MapImpl(const T &Itr) : Itr(Itr) {} + private: T Itr; }; @@ -235,14 +229,14 @@ public: Itr->Prev(); return InstructionIterator(std::move(Tmp)); } - bool operator==(const InstructionIterator& Other) const { + bool operator==(const InstructionIterator &Other) const { return Itr->Compare(*Other.Itr); } - bool operator!=(const InstructionIterator& Other) const { + bool operator!=(const InstructionIterator &Other) const { return !Itr->Compare(*Other.Itr); } - MCInst& operator*() { return Itr->Deref(); } - MCInst* operator->() { return &Itr->Deref(); } + MCInst &operator*() { return Itr->Deref(); } + MCInst *operator->() { return &Itr->Deref(); } InstructionIterator &operator=(InstructionIterator &&Other) { Itr = std::move(Other.Itr); @@ -318,9 +312,7 @@ public: Allocator.ValueAllocator.Reset(); } - virtual ~MCPlusBuilder() { - freeAnnotations(); - } + virtual ~MCPlusBuilder() { freeAnnotations(); } /// Free all memory allocated for annotations void freeAnnotations() { @@ -339,8 +331,7 @@ public: bool equals(const MCInst &A, const MCInst &B, CompFuncTy Comp) const; - bool equals(const MCOperand &A, const MCOperand &B, - CompFuncTy Comp) const; + bool equals(const MCOperand &A, const MCOperand &B, CompFuncTy Comp) const; bool equals(const MCExpr &A, const MCExpr &B, CompFuncTy Comp) const; @@ -382,9 +373,7 @@ public: } /// Check whether we support inverting this branch - virtual bool isUnsupportedBranch(unsigned Opcode) const { - return false; - } + virtual bool isUnsupportedBranch(unsigned Opcode) const { return false; } /// Return true of the instruction is of pseudo kind. bool isPseudo(const MCInst &Inst) const { @@ -424,9 +413,7 @@ public: return false; } - virtual MCPhysReg getX86R11() const { - llvm_unreachable("not implemented"); - } + virtual MCPhysReg getX86R11() const { llvm_unreachable("not implemented"); } /// Create increment contents of target by 1 for Instrumentation virtual void createInstrIncMemory(InstructionListType &Instrs, @@ -644,7 +631,7 @@ public: return true; if (static_cast(OpNum) >= - MCPlus::getNumPrimeOperands(*CurInst)) + MCPlus::getNumPrimeOperands(*CurInst)) return false; const MCOperand &Op = CurInst->getOperand(OpNum); @@ -689,13 +676,12 @@ public: if (I == InInstrWindow.begin()) return false; --I; - if (OpNum < 0 || static_cast(OpNum) >= - MCPlus::getNumPrimeOperands(*I)) + if (OpNum < 0 || + static_cast(OpNum) >= MCPlus::getNumPrimeOperands(*I)) return false; Op = I->getOperand(OpNum); return true; } - }; /// Matches operands that are immediates @@ -746,8 +732,8 @@ public: if (I == InInstrWindow.begin()) return false; --I; - if (OpNum < 0 || static_cast(OpNum) >= - MCPlus::getNumPrimeOperands(*I)) + if (OpNum < 0 || + static_cast(OpNum) >= MCPlus::getNumPrimeOperands(*I)) return false; const MCOperand &Op = I->getOperand(OpNum); if (!Op.isReg()) @@ -789,7 +775,7 @@ public: } std::unique_ptr matchSymbol() const { - static const MCSymbol* Unused; + static const MCSymbol *Unused; return std::unique_ptr(new SymbolMatcher(Unused)); } @@ -832,9 +818,8 @@ public: /// \brief Given a branch instruction try to get the address the branch /// targets. Return true on success, and the address in Target. - virtual bool - evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, - uint64_t &Target) const; + virtual bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, + uint64_t &Target) const; /// Return true if one of the operands of the \p Inst instruction uses /// PC-relative addressing. @@ -872,14 +857,11 @@ public: /// /// Since a Displacement field could be either an immediate or an expression, /// the function sets either \p DispImm or \p DispExpr value. - virtual bool evaluateX86MemoryOperand(const MCInst &Inst, - unsigned *BaseRegNum, - int64_t *ScaleImm, - unsigned *IndexRegNum, - int64_t *DispImm, - unsigned *SegmentRegNum, - const MCExpr **DispExpr = nullptr) const - { + virtual bool + evaluateX86MemoryOperand(const MCInst &Inst, unsigned *BaseRegNum, + int64_t *ScaleImm, unsigned *IndexRegNum, + int64_t *DispImm, unsigned *SegmentRegNum, + const MCExpr **DispExpr = nullptr) const { llvm_unreachable("not implemented"); return false; } @@ -890,8 +872,7 @@ public: /// /// For RIP-relative addressing the caller is required to pass instruction /// \p Address and \p Size. - virtual bool evaluateMemOperandTarget(const MCInst &Inst, - uint64_t &Target, + virtual bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, uint64_t Address = 0, uint64_t Size = 0) const { llvm_unreachable("not implemented"); @@ -1273,9 +1254,7 @@ public: /// Perform any additional actions to transform a (conditional) tail call /// into a (conditional) jump. Assume the target was already replaced with /// a local one, so the default is to do nothing more. - virtual bool convertTailCallToJmp(MCInst &Inst) { - return true; - } + virtual bool convertTailCallToJmp(MCInst &Inst) { return true; } /// Replace instruction opcode to be a regural call instead of tail call. virtual bool convertTailCallToCall(MCInst &Inst) { @@ -1313,12 +1292,9 @@ public: /// Receives a list of MCInst of the basic block to analyze and interpret the /// terminators of this basic block. TBB must be initialized with the original /// fall-through for this BB. - virtual bool analyzeBranch(InstructionIterator Begin, - InstructionIterator End, - const MCSymbol *&TBB, - const MCSymbol *&FBB, - MCInst *&CondBranch, - MCInst *&UncondBranch) const { + virtual bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, + const MCSymbol *&TBB, const MCSymbol *&FBB, + MCInst *&CondBranch, MCInst *&UncondBranch) const { llvm_unreachable("not implemented"); return false; } @@ -1330,29 +1306,22 @@ public: /// will be set to the different components of the branch. \p MemLocInstr /// is the instruction that loads up the indirect function pointer. It may /// or may not be same as \p Instruction. - virtual IndirectBranchType analyzeIndirectBranch( - MCInst &Instruction, - InstructionIterator Begin, - InstructionIterator End, - const unsigned PtrSize, - MCInst *&MemLocInstr, - unsigned &BaseRegNum, - unsigned &IndexRegNum, - int64_t &DispValue, - const MCExpr *&DispExpr, - MCInst *&PCRelBaseOut - ) const { + virtual IndirectBranchType + analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin, + InstructionIterator End, const unsigned PtrSize, + MCInst *&MemLocInstr, unsigned &BaseRegNum, + unsigned &IndexRegNum, int64_t &DispValue, + const MCExpr *&DispExpr, MCInst *&PCRelBaseOut) const { llvm_unreachable("not implemented"); return IndirectBranchType::UNKNOWN; } - virtual bool - analyzeVirtualMethodCall(InstructionIterator Begin, - InstructionIterator End, - std::vector &MethodFetchInsns, - unsigned &VtableRegNum, - unsigned &BaseRegNum, - uint64_t &MethodOffset) const { + virtual bool analyzeVirtualMethodCall(InstructionIterator Begin, + InstructionIterator End, + std::vector &MethodFetchInsns, + unsigned &VtableRegNum, + unsigned &BaseRegNum, + uint64_t &MethodOffset) const { llvm_unreachable("not implemented"); return false; } @@ -1378,10 +1347,8 @@ public: /// TgtHiBits does not have an immediate operand, but an expression, then /// this expression is put in TgtHiSym and Tgt only contains the lower bits. virtual bool matchLinkerVeneer(InstructionIterator Begin, - InstructionIterator End, - uint64_t Address, - const MCInst &CurInst, - MCInst *&TargetHiBits, + InstructionIterator End, uint64_t Address, + const MCInst &CurInst, MCInst *&TargetHiBits, MCInst *&TargetLowBits, uint64_t &Target) const { llvm_unreachable("not implemented"); @@ -1714,8 +1681,8 @@ public: ValueType &getAnnotationAs(const MCInst &Inst, unsigned Index) const { Optional Value = getAnnotationOpValue(Inst, Index); assert(Value && "annotation should exist"); - return reinterpret_cast *> - (*Value)->getValue(); + return reinterpret_cast *>(*Value) + ->getValue(); } /// Get an annotation as a specific value. Assumes that the annotation exists. @@ -1729,7 +1696,8 @@ public: /// Get an annotation as a specific value. If the annotation does not exist, /// return the \p DefaultValue. - template const ValueType & + template + const ValueType & getAnnotationWithDefault(const MCInst &Inst, unsigned Index, const ValueType &DefaultValue = ValueType()) { if (!hasAnnotation(Inst, Index)) @@ -1739,7 +1707,8 @@ public: /// Get an annotation as a specific value. If the annotation does not exist, /// return the \p DefaultValue. - template const ValueType & + template + const ValueType & getAnnotationWithDefault(const MCInst &Inst, StringRef Name, const ValueType &DefaultValue = ValueType()) { const unsigned Index = getOrCreateAnnotationIndex(Name); @@ -1896,27 +1865,23 @@ public: std::vector>; struct MultiBlocksCode { BlocksVectorTy Blocks; - std::vector Successors; + std::vector Successors; }; virtual BlocksVectorTy indirectCallPromotion( - const MCInst &CallInst, - const std::vector> &Targets, - const std::vector> &VtableSyms, - const std::vector &MethodFetchInsns, - const bool MinimizeCodeSize, - MCContext *Ctx - ) { + const MCInst &CallInst, + const std::vector> &Targets, + const std::vector> &VtableSyms, + const std::vector &MethodFetchInsns, + const bool MinimizeCodeSize, MCContext *Ctx) { llvm_unreachable("not implemented"); return BlocksVectorTy(); } virtual BlocksVectorTy jumpTablePromotion( - const MCInst &IJmpInst, - const std::vector>& Targets, - const std::vector &TargetFetchInsns, - MCContext *Ctx - ) const { + const MCInst &IJmpInst, + const std::vector> &Targets, + const std::vector &TargetFetchInsns, MCContext *Ctx) const { llvm_unreachable("not implemented"); return BlocksVectorTy(); } diff --git a/bolt/include/bolt/Core/ParallelUtilities.h b/bolt/include/bolt/Core/ParallelUtilities.h index 26f4f5dbae29..d3358521c90f 100644 --- a/bolt/include/bolt/Core/ParallelUtilities.h +++ b/bolt/include/bolt/Core/ParallelUtilities.h @@ -24,7 +24,7 @@ namespace opts { extern cl::opt ThreadCount; extern cl::opt NoThreads; extern cl::opt TaskCount; -} +} // namespace opts namespace llvm { class ThreadPool; diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h index 22681e76ae2a..4e79ef9c64f1 100644 --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -20,10 +20,8 @@ class raw_ostream; namespace ELF { /// Relocation type mask that was accidentally output by bfd 2.30 linker. -enum { - R_X86_64_converted_reloc_bit = 0x80 -}; -} +enum { R_X86_64_converted_reloc_bit = 0x80 }; +} // namespace ELF namespace bolt { @@ -91,9 +89,7 @@ struct Relocation { static uint64_t getPC64(); /// Return true if this relocation is PC-relative. Return false otherwise. - bool isPCRelative() const { - return isPCRelative(Type); - } + bool isPCRelative() const { return isPCRelative(Type); } /// Emit relocation at a current \p Streamer' position. The caller is /// responsible for setting the position correctly. diff --git a/bolt/include/bolt/Passes/Aligner.h b/bolt/include/bolt/Passes/Aligner.h index 9f241635561c..5a87a6a06d5a 100644 --- a/bolt/include/bolt/Passes/Aligner.h +++ b/bolt/include/bolt/Passes/Aligner.h @@ -31,9 +31,7 @@ private: public: explicit AlignerPass() : BinaryFunctionPass(false) {} - const char *getName() const override { - return "aligner"; - } + const char *getName() const override { return "aligner"; } /// Pass entry point void runOnFunctions(BinaryContext &BC) override; @@ -42,5 +40,4 @@ public: } // namespace bolt } // namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/AllocCombiner.h b/bolt/include/bolt/Passes/AllocCombiner.h index b62a21d44a91..f291dc72669e 100644 --- a/bolt/include/bolt/Passes/AllocCombiner.h +++ b/bolt/include/bolt/Passes/AllocCombiner.h @@ -29,9 +29,7 @@ public: explicit AllocCombinerPass(const cl::opt &PrintPass) : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "alloc-combiner"; - } + const char *getName() const override { return "alloc-combiner"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && FuncsChanged.count(&BF) > 0; @@ -44,5 +42,4 @@ public: } // namespace bolt } // namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/BinaryFunctionCallGraph.h b/bolt/include/bolt/Passes/BinaryFunctionCallGraph.h index 093b41b571ab..76243c20fe39 100644 --- a/bolt/include/bolt/Passes/BinaryFunctionCallGraph.h +++ b/bolt/include/bolt/Passes/BinaryFunctionCallGraph.h @@ -51,7 +51,7 @@ private: std::vector Funcs; }; -using CgFilterFunction = std::function; +using CgFilterFunction = std::function; inline bool NoFilter(const BinaryFunction &) { return false; } /// Builds a call graph from the map of BinaryFunctions provided in BC. @@ -64,16 +64,13 @@ inline bool NoFilter(const BinaryFunction &) { return false; } /// filling in the Size attribute of new Nodes. /// UseEdgeCounts is used to control if the Weight attribute on Arcs is computed /// using the number of calls. -BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, - CgFilterFunction Filter = NoFilter, - bool CgFromPerfData = false, - bool IncludeColdCalls = true, - bool UseFunctionHotSize = false, - bool UseSplitHotSize = false, - bool UseEdgeCounts = false, - bool IgnoreRecursiveCalls = false); +BinaryFunctionCallGraph +buildCallGraph(BinaryContext &BC, CgFilterFunction Filter = NoFilter, + bool CgFromPerfData = false, bool IncludeColdCalls = true, + bool UseFunctionHotSize = false, bool UseSplitHotSize = false, + bool UseEdgeCounts = false, bool IgnoreRecursiveCalls = false); -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/BinaryPasses.h b/bolt/include/bolt/Passes/BinaryPasses.h index 87a0f1ae01d1..3e38eaed41b8 100644 --- a/bolt/include/bolt/Passes/BinaryPasses.h +++ b/bolt/include/bolt/Passes/BinaryPasses.h @@ -31,12 +31,12 @@ class BinaryFunctionPass { protected: bool PrintPass; - explicit BinaryFunctionPass(const bool PrintPass) - : PrintPass(PrintPass) { } + explicit BinaryFunctionPass(const bool PrintPass) : PrintPass(PrintPass) {} /// Control whether a specific function should be skipped during /// optimization. virtual bool shouldOptimize(const BinaryFunction &BF) const; + public: virtual ~BinaryFunctionPass() = default; @@ -62,24 +62,19 @@ protected: public: DynoStatsPrintPass(const DynoStats &PrevDynoStats, const char *Title) - : BinaryFunctionPass(false) - , PrevDynoStats(PrevDynoStats) - , Title(Title) { - } + : BinaryFunctionPass(false), PrevDynoStats(PrevDynoStats), Title(Title) {} const char *getName() const override { return "print dyno-stats after optimizations"; } - bool shouldPrint(const BinaryFunction &BF) const override { - return false; - } + bool shouldPrint(const BinaryFunction &BF) const override { return false; } void runOnFunctions(BinaryContext &BC) override { const DynoStats NewDynoStats = getDynoStats(BC.getBinaryFunctions()); const bool Changed = (NewDynoStats != PrevDynoStats); - outs() << "BOLT-INFO: program-wide dynostats " - << Title << (Changed ? "" : " (no change)") << ":\n\n" + outs() << "BOLT-INFO: program-wide dynostats " << Title + << (Changed ? "" : " (no change)") << ":\n\n" << PrevDynoStats; if (Changed) { outs() << '\n'; @@ -114,18 +109,17 @@ class EliminateUnreachableBlocks : public BinaryFunctionPass { std::unordered_set Modified; std::atomic DeletedBlocks{0}; std::atomic DeletedBytes{0}; - void runOnFunction(BinaryFunction& Function); - public: - EliminateUnreachableBlocks(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + void runOnFunction(BinaryFunction &Function); - const char *getName() const override { - return "eliminate-unreachable"; - } +public: + EliminateUnreachableBlocks(const cl::opt &PrintPass) + : BinaryFunctionPass(PrintPass) {} + + const char *getName() const override { return "eliminate-unreachable"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0; } - void runOnFunctions(BinaryContext&) override; + void runOnFunctions(BinaryContext &) override; }; // Reorder the basic blocks for each function based on hotness. @@ -156,44 +150,38 @@ public: }; private: - void modifyFunctionLayout(BinaryFunction &Function, - LayoutType Type, + void modifyFunctionLayout(BinaryFunction &Function, LayoutType Type, bool MinBranchClusters) const; + public: explicit ReorderBasicBlocks(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} bool shouldOptimize(const BinaryFunction &BF) const override; - const char *getName() const override { - return "reorder-blocks"; - } + const char *getName() const override { return "reorder-blocks"; } bool shouldPrint(const BinaryFunction &BF) const override; void runOnFunctions(BinaryContext &BC) override; }; /// Sync local branches with CFG. class FixupBranches : public BinaryFunctionPass { - public: +public: explicit FixupBranches(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "fix-branches"; - } + const char *getName() const override { return "fix-branches"; } void runOnFunctions(BinaryContext &BC) override; }; /// Fix the CFI state and exception handling information after all other /// passes have completed. class FinalizeFunctions : public BinaryFunctionPass { - public: +public: explicit FinalizeFunctions(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "finalize-functions"; - } + const char *getName() const override { return "finalize-functions"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -202,11 +190,9 @@ class FinalizeFunctions : public BinaryFunctionPass { class CheckLargeFunctions : public BinaryFunctionPass { public: explicit CheckLargeFunctions(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "check-large-functions"; - } + const char *getName() const override { return "check-large-functions"; } void runOnFunctions(BinaryContext &BC) override; @@ -215,13 +201,11 @@ public: /// Convert and remove all BOLT-related annotations before LLVM code emission. class LowerAnnotations : public BinaryFunctionPass { - public: +public: explicit LowerAnnotations(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "lower-annotations"; - } + const char *getName() const override { return "lower-annotations"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -279,15 +263,14 @@ class SimplifyConditionalTailCalls : public BinaryFunctionPass { std::set BeenOptimized; bool shouldRewriteBranch(const BinaryBasicBlock *PredBB, - const MCInst &CondBranch, - const BinaryBasicBlock *BB, + const MCInst &CondBranch, const BinaryBasicBlock *BB, const bool DirectionFlag); uint64_t fixTailCalls(BinaryFunction &BF); public: explicit SimplifyConditionalTailCalls(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} const char *getName() const override { return "simplify-conditional-tail-calls"; @@ -321,11 +304,9 @@ class Peepholes : public BinaryFunctionPass { public: explicit Peepholes(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "peepholes"; - } + const char *getName() const override { return "peepholes"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -351,11 +332,9 @@ class SimplifyRODataLoads : public BinaryFunctionPass { public: explicit SimplifyRODataLoads(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "simplify-read-only-loads"; - } + const char *getName() const override { return "simplify-read-only-loads"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0; } @@ -364,14 +343,10 @@ public: /// Assign output sections to all functions. class AssignSections : public BinaryFunctionPass { - public: - explicit AssignSections() - : BinaryFunctionPass(false) { - } +public: + explicit AssignSections() : BinaryFunctionPass(false) {} - const char *getName() const override { - return "assign-sections"; - } + const char *getName() const override { return "assign-sections"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -381,32 +356,24 @@ class AssignSections : public BinaryFunctionPass { /// for blocks of interest (excluding prologues, epilogues, and BB frequency /// lower than 100). class PrintProfileStats : public BinaryFunctionPass { - public: +public: explicit PrintProfileStats(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "profile-stats"; - } - bool shouldPrint(const BinaryFunction &) const override { - return false; - } + const char *getName() const override { return "profile-stats"; } + bool shouldPrint(const BinaryFunction &) const override { return false; } void runOnFunctions(BinaryContext &BC) override; }; /// Prints a list of the top 100 functions sorted by a set of /// dyno stats categories. class PrintProgramStats : public BinaryFunctionPass { - public: +public: explicit PrintProgramStats(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "print-stats"; - } - bool shouldPrint(const BinaryFunction &) const override { - return false; - } + const char *getName() const override { return "print-stats"; } + bool shouldPrint(const BinaryFunction &) const override { return false; } void runOnFunctions(BinaryContext &BC) override; }; @@ -415,11 +382,9 @@ class PrintProgramStats : public BinaryFunctionPass { class InstructionLowering : public BinaryFunctionPass { public: explicit InstructionLowering(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) {} + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "inst-lowering"; - } + const char *getName() const override { return "inst-lowering"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -428,11 +393,9 @@ public: class StripRepRet : public BinaryFunctionPass { public: explicit StripRepRet(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) {} + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "strip-rep-ret"; - } + const char *getName() const override { return "strip-rep-ret"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -441,11 +404,9 @@ public: class InlineMemcpy : public BinaryFunctionPass { public: explicit InlineMemcpy(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) {} + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "inline-memcpy"; - } + const char *getName() const override { return "inline-memcpy"; } void runOnFunctions(BinaryContext &BC) override; }; @@ -462,13 +423,11 @@ private: public: explicit SpecializeMemcpy1(const cl::opt &PrintPass, cl::list &Spec) - : BinaryFunctionPass(PrintPass), Spec(Spec) {} + : BinaryFunctionPass(PrintPass), Spec(Spec) {} bool shouldOptimize(const BinaryFunction &BF) const override; - const char *getName() const override { - return "specialize-memcpy"; - } + const char *getName() const override { return "specialize-memcpy"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/CacheMetrics.h b/bolt/include/bolt/Passes/CacheMetrics.h index ca543519604a..59a0d1bfc6e3 100644 --- a/bolt/include/bolt/Passes/CacheMetrics.h +++ b/bolt/include/bolt/Passes/CacheMetrics.h @@ -30,9 +30,7 @@ void printAll(const std::vector &BinaryFunctions); /// - SrcSize is the size of the source block; /// - DstAddr is the address of the destination block; /// - Count is the number of jumps between the pair of blocks. -double extTSPScore(uint64_t SrcAddr, - uint64_t SrcSize, - uint64_t DstAddr, +double extTSPScore(uint64_t SrcAddr, uint64_t SrcSize, uint64_t DstAddr, uint64_t Count); } // namespace CacheMetrics diff --git a/bolt/include/bolt/Passes/CallGraph.h b/bolt/include/bolt/Passes/CallGraph.h index a66fbb945588..5d7eb88dce54 100644 --- a/bolt/include/bolt/Passes/CallGraph.h +++ b/bolt/include/bolt/Passes/CallGraph.h @@ -32,8 +32,7 @@ public: using NodeId = size_t; static constexpr NodeId InvalidId = -1; - template - class iterator_range { + template class iterator_range { T Begin; T End; @@ -41,8 +40,7 @@ public: template iterator_range(Container &&c) : Begin(c.begin()), End(c.end()) {} iterator_range(T Begin, T End) - : Begin(std::move(Begin)), - End(std::move(End)) {} + : Begin(std::move(Begin)), End(std::move(End)) {} T begin() const { return Begin; } T end() const { return End; } @@ -54,12 +52,8 @@ public: int64_t operator()(const Arc &Arc) const; }; - Arc(NodeId S, NodeId D, double W = 0) - : Src(S) - , Dst(D) - , Weight(W) - {} - Arc(const Arc&) = delete; + Arc(NodeId S, NodeId D, double W = 0) : Src(S), Dst(D), Weight(W) {} + Arc(const Arc &) = delete; friend bool operator==(const Arc &Lhs, const Arc &Rhs) { return Lhs.Src == Rhs.Src && Lhs.Dst == Rhs.Dst; @@ -87,35 +81,27 @@ public: class Node { public: explicit Node(uint32_t Size, uint64_t Samples = 0) - : Size(Size), Samples(Samples) - {} + : Size(Size), Samples(Samples) {} uint32_t size() const { return Size; } uint64_t samples() const { return Samples; } - const std::vector &successors() const { - return Succs; - } - const std::vector &predecessors() const { - return Preds; - } + const std::vector &successors() const { return Succs; } + const std::vector &predecessors() const { return Preds; } private: friend class CallGraph; uint32_t Size; uint64_t Samples; - // preds and succs contain no duplicate elements and self arcs are not allowed + // preds and succs contain no duplicate elements and self arcs are not + // allowed std::vector Preds; std::vector Succs; }; - size_t numNodes() const { - return Nodes.size(); - } - size_t numArcs() const { - return Arcs.size(); - } + size_t numNodes() const { return Nodes.size(); } + size_t numArcs() const { return Arcs.size(); } const Node &getNode(const NodeId Id) const { assert(Id < Nodes.size()); return Nodes[Id]; @@ -149,11 +135,12 @@ public: return iterator_range(Arcs.begin(), Arcs.end()); } iterator_range::const_iterator> nodes() const { - return iterator_range::const_iterator>(Nodes.begin(), Nodes.end()); + return iterator_range::const_iterator>(Nodes.begin(), + Nodes.end()); } double density() const { - return double(Arcs.size()) / (Nodes.size()*Nodes.size()); + return double(Arcs.size()) / (Nodes.size() * Nodes.size()); } // Initialize NormalizedWeight field for every arc @@ -162,8 +149,7 @@ public: // samples for every node void adjustArcWeights(); - template - void printDot(char* fileName, L getLabel) const; + template void printDot(char *fileName, L getLabel) const; private: void setSamples(const NodeId Id, uint64_t Samples) { @@ -175,35 +161,28 @@ private: ArcsType Arcs; }; -template -void CallGraph::printDot(char* FileName, L GetLabel) const { - FILE* File = fopen(FileName, "wt"); - if (!File) return; +template void CallGraph::printDot(char *FileName, L GetLabel) const { + FILE *File = fopen(FileName, "wt"); + if (!File) + return; fprintf(File, "digraph g {\n"); for (NodeId F = 0; F < Nodes.size(); F++) { - if (Nodes[F].samples() == 0) continue; - fprintf( - File, - "f%lu [label=\"%s\\nsamples=%u\\nsize=%u\"];\n", - F, - GetLabel(F), - Nodes[F].samples(), - Nodes[F].size()); + if (Nodes[F].samples() == 0) + continue; + fprintf(File, "f%lu [label=\"%s\\nsamples=%u\\nsize=%u\"];\n", F, + GetLabel(F), Nodes[F].samples(), Nodes[F].size()); } for (NodeId F = 0; F < Nodes.size(); F++) { - if (Nodes[F].samples() == 0) continue; + if (Nodes[F].samples() == 0) + continue; for (NodeId Dst : Nodes[F].successors()) { ArcConstIterator Arc = findArc(F, Dst); fprintf( - File, - "f%lu -> f%u [label=\"normWgt=%.3lf,weight=%.0lf,callOffset=%.1lf\"];" - "\n", - F, - Dst, - Arc->normalizedWeight(), - Arc->weight(), - Arc->avgCallOffset()); + File, + "f%lu -> f%u [label=\"normWgt=%.3lf,weight=%.0lf,callOffset=%.1lf\"];" + "\n", + F, Dst, Arc->normalizedWeight(), Arc->weight(), Arc->avgCallOffset()); } } fprintf(File, "}\n"); diff --git a/bolt/include/bolt/Passes/CallGraphWalker.h b/bolt/include/bolt/Passes/CallGraphWalker.h index 8ba6d92a5464..bca9b6d7fdb4 100644 --- a/bolt/include/bolt/Passes/CallGraphWalker.h +++ b/bolt/include/bolt/Passes/CallGraphWalker.h @@ -31,7 +31,7 @@ class CallGraphWalker { std::deque TopologicalCGOrder; /// Stores all visitor functions to call when traversing the call graph - typedef std::function CallbackTy; + typedef std::function CallbackTy; std::vector Visitors; /// Do the bottom-up traversal @@ -46,15 +46,13 @@ public: /// whatever information it is keeping track of has changed. Function must /// converge with time, ie, it must eventually return false, otherwise the /// call graph walk will never finish. - void registerVisitor(CallbackTy Callback) { - Visitors.emplace_back(Callback); - } + void registerVisitor(CallbackTy Callback) { Visitors.emplace_back(Callback); } /// Build the call graph, establish a traversal order and traverse it. void walk(); }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/DataflowAnalysis.h b/bolt/include/bolt/Passes/DataflowAnalysis.h index a17c54cf7644..80880cb677b3 100644 --- a/bolt/include/bolt/Passes/DataflowAnalysis.h +++ b/bolt/include/bolt/Passes/DataflowAnalysis.h @@ -101,20 +101,18 @@ void doForAllSuccs(const BinaryBasicBlock &BB, std::function Task); /// Default printer for State data. -template -class StatePrinter { +template class StatePrinter { public: - void print(raw_ostream &OS, const StateTy &State) const { - OS << State; - } - explicit StatePrinter(const BinaryContext &) { } + void print(raw_ostream &OS, const StateTy &State) const { OS << State; } + explicit StatePrinter(const BinaryContext &) {} }; /// Printer for State data that is a BitVector of registers. class RegStatePrinter { public: void print(raw_ostream &OS, const BitVector &State) const; - explicit RegStatePrinter(const BinaryContext &BC) : BC(BC) { } + explicit RegStatePrinter(const BinaryContext &BC) : BC(BC) {} + private: const BinaryContext &BC; }; @@ -146,18 +144,14 @@ private: /// Confluence operator = union (if a reg is alive in any succ, it is alive /// in the current block). /// -template > class DataflowAnalysis { /// CRTP convenience methods - Derived &derived() { - return *static_cast(this); - } + Derived &derived() { return *static_cast(this); } const Derived &const_derived() const { - return *static_cast(this); + return *static_cast(this); } mutable Optional AnnotationIndex; @@ -180,9 +174,7 @@ protected: DenseMap PrevPoint; /// Perform any bookkeeping before dataflow starts - void preflight() { - llvm_unreachable("Unimplemented method"); - } + void preflight() { llvm_unreachable("Unimplemented method"); } /// Sets initial state for each BB StateTy getStartingStateAtBB(const BinaryBasicBlock &BB) { @@ -210,7 +202,7 @@ protected: /// instruction considering in sets for the first instructions of its /// landing pads. void doConfluenceWithLP(StateTy &StateOut, const StateTy &StateIn, - const MCInst &Invoke) { + const MCInst &Invoke) { return derived().doConfluence(StateOut, StateIn); } @@ -231,7 +223,7 @@ protected: if (AnnotationIndex) return *AnnotationIndex; AnnotationIndex = - BC.MIB->getOrCreateAnnotationIndex(const_derived().getAnnotationName()); + BC.MIB->getOrCreateAnnotationIndex(const_derived().getAnnotationName()); return *AnnotationIndex; } @@ -253,9 +245,7 @@ protected: public: /// Return the allocator id - unsigned getAllocatorId() { - return AllocatorId; - } + unsigned getAllocatorId() { return AllocatorId; } /// If the direction of the dataflow is forward, operates on the last /// instruction of all predecessors when performing an iteration of the @@ -274,9 +264,7 @@ public: MCPlusBuilder::AllocatorIdTy AllocatorId = 0) : BC(BF.getBinaryContext()), Func(BF), AllocatorId(AllocatorId) {} - virtual ~DataflowAnalysis() { - cleanAnnotations(); - } + virtual ~DataflowAnalysis() { cleanAnnotations(); } /// Track the state at basic block start (end) if direction of the dataflow /// is forward (backward). @@ -400,7 +388,7 @@ public: else LAST = &*BB->begin(); - auto doNext = [&] (MCInst &Inst, const BinaryBasicBlock &BB) { + auto doNext = [&](MCInst &Inst, const BinaryBasicBlock &BB) { StateTy CurState = derived().computeNext(Inst, *PrevState); if (Backward && BC.MIB->isInvoke(Inst)) { @@ -488,27 +476,22 @@ public: : BV(BV), Expressions(Exprs) { Idx = BV->find_first(); } - ExprIterator(const BitVector *BV, const std::vector &Exprs, - int Idx) + ExprIterator(const BitVector *BV, const std::vector &Exprs, int Idx) : BV(BV), Expressions(Exprs), Idx(Idx) {} - int getBitVectorIndex() const { - return Idx; - } + int getBitVectorIndex() const { return Idx; } }; /// Specialization of DataflowAnalysis whose state specifically stores /// a set of instructions. -template > class InstrsDataflowAnalysis : public DataflowAnalysis { public: /// These iterator functions offer access to the set of pointers to /// instructions in a given program point - template - ExprIterator expr_begin(const T &Point) const { + template ExprIterator expr_begin(const T &Point) const { if (auto State = this->getStateAt(Point)) return ExprIterator(&*State, Expressions); return expr_end(); @@ -534,13 +517,13 @@ public: /// Return whether \p Expr is in the state set at \p Point bool count(ProgramPoint Point, const MCInst &Expr) const { auto IdxIter = ExprToIdx.find(&Expr); - assert (IdxIter != ExprToIdx.end() && "Invalid Expr"); + assert(IdxIter != ExprToIdx.end() && "Invalid Expr"); return (*this->getStateAt(Point))[IdxIter->second]; } bool count(const MCInst &Point, const MCInst &Expr) const { auto IdxIter = ExprToIdx.find(&Expr); - assert (IdxIter != ExprToIdx.end() && "Invalid Expr"); + assert(IdxIter != ExprToIdx.end() && "Invalid Expr"); return (*this->getStateAt(Point))[IdxIter->second]; } @@ -561,16 +544,16 @@ public: /// DenseMapInfo allows us to use the DenseMap LLVM data structure to store /// ProgramPoints. -template<> struct DenseMapInfo { +template <> struct DenseMapInfo { static inline bolt::ProgramPoint getEmptyKey() { uintptr_t Val = static_cast(-1); - Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; - return bolt::ProgramPoint(reinterpret_cast(Val)); + Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; + return bolt::ProgramPoint(reinterpret_cast(Val)); } static inline bolt::ProgramPoint getTombstoneKey() { uintptr_t Val = static_cast(-2); - Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; - return bolt::ProgramPoint(reinterpret_cast(Val)); + Val <<= PointerLikeTypeTraits::NumLowBitsAvailable; + return bolt::ProgramPoint(reinterpret_cast(Val)); } static unsigned getHashValue(const bolt::ProgramPoint &PP) { return (unsigned((uintptr_t)PP.Data.BB) >> 4) ^ diff --git a/bolt/include/bolt/Passes/DominatorAnalysis.h b/bolt/include/bolt/Passes/DominatorAnalysis.h index 1a8530a20e53..96fc5ade8708 100644 --- a/bolt/include/bolt/Passes/DominatorAnalysis.h +++ b/bolt/include/bolt/Passes/DominatorAnalysis.h @@ -88,7 +88,7 @@ public: while (BB->size() == 0) { if (BB->succ_size() == 0) return false; - assert (BB->succ_size() == 1); + assert(BB->succ_size() == 1); BB = *BB->succ_begin(); } const MCInst &InstA = *BB->begin(); diff --git a/bolt/include/bolt/Passes/FrameAnalysis.h b/bolt/include/bolt/Passes/FrameAnalysis.h index 7dc3eac23fe8..56755b8d3572 100644 --- a/bolt/include/bolt/Passes/FrameAnalysis.h +++ b/bolt/include/bolt/Passes/FrameAnalysis.h @@ -73,8 +73,7 @@ struct ArgAccesses { : AssumeEverything(AssumeEverything) {} }; -raw_ostream &operator<<(raw_ostream &OS, - const FrameIndexEntry &FIE); +raw_ostream &operator<<(raw_ostream &OS, const FrameIndexEntry &FIE); /// This pass attaches stack access information to instructions. If a load/store /// instruction accesses a stack position, it will identify the CFA offset and @@ -175,8 +174,7 @@ class FrameAnalysis { std::vector SPTAllocatorsId; public: - explicit FrameAnalysis(BinaryContext &BC, - BinaryFunctionCallGraph &CG); + explicit FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG); /// Return true if we could fully analyze \p Func bool hasFrameInfo(const BinaryFunction &Func) const { @@ -198,9 +196,7 @@ public: /// Remove all MCAnnotations attached by this pass void cleanAnnotations(); - ~FrameAnalysis() { - cleanAnnotations(); - } + ~FrameAnalysis() { cleanAnnotations(); } /// Print to standard output statistics about the analysis performed by this /// pass @@ -231,5 +227,4 @@ public: } // namespace bolt } // namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/FrameOptimizer.h b/bolt/include/bolt/Passes/FrameOptimizer.h index e1be0eb86b2f..3c641e88bec2 100644 --- a/bolt/include/bolt/Passes/FrameOptimizer.h +++ b/bolt/include/bolt/Passes/FrameOptimizer.h @@ -90,13 +90,11 @@ class FrameOptimizerPass : public BinaryFunctionPass { /// Perform a dataflow analysis in \p BF to reveal unnecessary reloads from /// the frame. Use the analysis to convert memory loads to register moves or /// immediate loads. Delete redundant register moves. - void removeUnnecessaryLoads(const RegAnalysis &RA, - const FrameAnalysis &FA, + void removeUnnecessaryLoads(const RegAnalysis &RA, const FrameAnalysis &FA, BinaryFunction &BF); /// Use information from stack frame usage to delete unused stores. - void removeUnusedStores(const FrameAnalysis &FA, - BinaryFunction &BF); + void removeUnusedStores(const FrameAnalysis &FA, BinaryFunction &BF); /// Perform shrinkwrapping step void performShrinkWrapping(const RegAnalysis &RA, const FrameAnalysis &FA, @@ -106,9 +104,7 @@ public: explicit FrameOptimizerPass(const cl::opt &PrintPass) : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "frame-optimizer"; - } + const char *getName() const override { return "frame-optimizer"; } /// Pass entry point void runOnFunctions(BinaryContext &BC) override; @@ -122,5 +118,4 @@ public: } // namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/HFSort.h b/bolt/include/bolt/Passes/HFSort.h index 9d09f20624db..2fd11efa4a08 100644 --- a/bolt/include/bolt/Passes/HFSort.h +++ b/bolt/include/bolt/Passes/HFSort.h @@ -56,17 +56,12 @@ public: bool frozen() const { return Frozen; } void freeze() { Frozen = true; } void merge(const Cluster &Other, const double Aw = 0); - void merge(const Cluster &Other, const std::vector& Targets_); + void merge(const Cluster &Other, + const std::vector &Targets_); void clear(); - size_t numTargets() const { - return Targets.size(); - } - const std::vector &targets() const { - return Targets; - } - CallGraph::NodeId target(size_t N) const { - return Targets[N]; - } + size_t numTargets() const { return Targets.size(); } + const std::vector &targets() const { return Targets; } + CallGraph::NodeId target(size_t N) const { return Targets[N]; } void reverseTargets(); bool hasId() const { return Id != -1u; } void setId(uint32_t NewId) { @@ -77,6 +72,7 @@ public: assert(hasId()); return Id; } + private: uint32_t Id{-1u}; std::vector Targets; diff --git a/bolt/include/bolt/Passes/IdenticalCodeFolding.h b/bolt/include/bolt/Passes/IdenticalCodeFolding.h index 13e26598c0fa..ff4da3bd4fc7 100644 --- a/bolt/include/bolt/Passes/IdenticalCodeFolding.h +++ b/bolt/include/bolt/Passes/IdenticalCodeFolding.h @@ -31,13 +31,12 @@ protected: return false; return BinaryFunctionPass::shouldOptimize(BF); } + public: explicit IdenticalCodeFolding(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "identical-code-folding"; - } + const char *getName() const override { return "identical-code-folding"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/IndirectCallPromotion.h b/bolt/include/bolt/Passes/IndirectCallPromotion.h index 30eae48b7f3f..ffa4fbcc25f8 100644 --- a/bolt/include/bolt/Passes/IndirectCallPromotion.h +++ b/bolt/include/bolt/Passes/IndirectCallPromotion.h @@ -104,12 +104,10 @@ class IndirectCallPromotion : public BinaryFunctionPass { struct Location { MCSymbol *Sym{nullptr}; uint64_t Addr{0}; - bool isValid() const { - return Sym || (!Sym && Addr != 0); - } - Location() { } - explicit Location(MCSymbol *Sym) : Sym(Sym) { } - explicit Location(uint64_t Addr) : Addr(Addr) { } + bool isValid() const { return Sym || (!Sym && Addr != 0); } + Location() {} + explicit Location(MCSymbol *Sym) : Sym(Sym) {} + explicit Location(uint64_t Addr) : Addr(Addr) {} }; struct Callsite { @@ -119,15 +117,12 @@ class IndirectCallPromotion : public BinaryFunctionPass { uint64_t Branches{0}; // Indices in the jmp table (jt only) std::vector JTIndices; - bool isValid() const { - return From.isValid() && To.isValid(); - } + bool isValid() const { return From.isValid() && To.isValid(); } Callsite(BinaryFunction &BF, const IndirectCallProfile &ICP); - Callsite(const Location &From, const Location &To, - uint64_t Mispreds, uint64_t Branches, - uint64_t JTIndex) - : From(From), To(To), Mispreds(Mispreds), Branches(Branches), - JTIndices(1, JTIndex) { } + Callsite(const Location &From, const Location &To, uint64_t Mispreds, + uint64_t Branches, uint64_t JTIndex) + : From(From), To(To), Mispreds(Mispreds), Branches(Branches), + JTIndices(1, JTIndex) {} }; std::unordered_set Modified; @@ -216,11 +211,9 @@ class IndirectCallPromotion : public BinaryFunctionPass { public: explicit IndirectCallPromotion(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "indirect-call-promotion"; - } + const char *getName() const override { return "indirect-call-promotion"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0; } diff --git a/bolt/include/bolt/Passes/Inliner.h b/bolt/include/bolt/Passes/Inliner.h index bb656c9aac56..7d4e78fd1c82 100644 --- a/bolt/include/bolt/Passes/Inliner.h +++ b/bolt/include/bolt/Passes/Inliner.h @@ -20,7 +20,6 @@ namespace bolt { class Inliner : public BinaryFunctionPass { private: - enum InliningType : char { INL_NONE = 0, /// Cannot inline INL_TAILCALL, /// Can inline at tail call site @@ -32,9 +31,7 @@ private: uint64_t SizeAfterInlining{0}; uint64_t SizeAfterTailCallInlining{0}; - InliningInfo(InliningType Type = INL_NONE) - : Type(Type) - {} + InliningInfo(InliningType Type = INL_NONE) : Type(Type) {} }; std::unordered_map InliningCandidates; @@ -74,8 +71,7 @@ private: /// Return the location (basic block and instruction iterator) where the code /// of the caller function continues after the inlined code. std::pair - inlineCall(BinaryBasicBlock &CallerBB, - BinaryBasicBlock::iterator CallInst, + inlineCall(BinaryBasicBlock &CallerBB, BinaryBasicBlock::iterator CallInst, const BinaryFunction &Callee); /// Check if the inliner can handle inlining of \p BF. @@ -83,11 +79,9 @@ private: public: explicit Inliner(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "inlining"; - } + const char *getName() const override { return "inlining"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0; diff --git a/bolt/include/bolt/Passes/Instrumentation.h b/bolt/include/bolt/Passes/Instrumentation.h index 697b70b015e9..ef057516b8dc 100644 --- a/bolt/include/bolt/Passes/Instrumentation.h +++ b/bolt/include/bolt/Passes/Instrumentation.h @@ -86,8 +86,8 @@ private: BinaryBasicBlock &FromBB, uint32_t From, BinaryFunction &ToFunc, BinaryBasicBlock *TargetBB, uint32_t ToOffset, bool IsLeaf, bool IsInvoke, - FunctionDescription *FuncDesc, - uint32_t FromNodeID, uint32_t ToNodeID = 0); + FunctionDescription *FuncDesc, uint32_t FromNodeID, + uint32_t ToNodeID = 0); void instrumentLeafNode(BinaryBasicBlock &BB, BinaryBasicBlock::iterator Iter, bool IsLeaf, FunctionDescription &FuncDesc, @@ -123,7 +123,7 @@ private: BinaryFunction *IndCallHandlerExitBBFunction; BinaryFunction *IndTailCallHandlerExitBBFunction; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/JTFootprintReduction.h b/bolt/include/bolt/Passes/JTFootprintReduction.h index 298be37d594c..dbe8770e5ea9 100644 --- a/bolt/include/bolt/Passes/JTFootprintReduction.h +++ b/bolt/include/bolt/Passes/JTFootprintReduction.h @@ -47,28 +47,24 @@ class JTFootprintReduction : public BinaryFunctionPass { /// instructions that depend on the availability of an extra register. /// This saves dcache/dTLB at the expense of icache. bool tryOptimizeNonPIC(BinaryContext &BC, BinaryBasicBlock &BB, - BinaryBasicBlock::iterator Inst, - uint64_t JTAddr, JumpTable *JumpTable, - DataflowInfoManager &Info); + BinaryBasicBlock::iterator Inst, uint64_t JTAddr, + JumpTable *JumpTable, DataflowInfoManager &Info); /// The PIC jump table optimization consists of "de-pic-ifying" it, since the /// PIC jump sequence is larger than its non-PIC counterpart, saving icache. bool tryOptimizePIC(BinaryContext &BC, BinaryBasicBlock &BB, - BinaryBasicBlock::iterator Inst, - uint64_t JTAddr, JumpTable *JumpTable, - DataflowInfoManager &Info); + BinaryBasicBlock::iterator Inst, uint64_t JTAddr, + JumpTable *JumpTable, DataflowInfoManager &Info); /// Run a pass for \p Function void optimizeFunction(BinaryFunction &Function, DataflowInfoManager &Info); public: explicit JTFootprintReduction(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} /// BinaryPass interface functions - const char *getName() const override { - return "jt-footprint-reduction"; - } + const char *getName() const override { return "jt-footprint-reduction"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF) && Modified.count(&BF) > 0; } diff --git a/bolt/include/bolt/Passes/LivenessAnalysis.h b/bolt/include/bolt/Passes/LivenessAnalysis.h index c7640d58eab0..7a60fe9c86f1 100644 --- a/bolt/include/bolt/Passes/LivenessAnalysis.h +++ b/bolt/include/bolt/Passes/LivenessAnalysis.h @@ -18,20 +18,16 @@ namespace opts { extern llvm::cl::opt AssumeABI; extern llvm::cl::opt TimeOpts; -} +} // namespace opts namespace llvm { namespace bolt { -class LivenessAnalysis - : public DataflowAnalysis { - using Parent = DataflowAnalysis; - friend class DataflowAnalysis { + using Parent = + DataflowAnalysis; + friend class DataflowAnalysis; public: @@ -48,9 +44,7 @@ public: return BV.any(); } - void run() { - Parent::run(); - } + void run() { Parent::run(); } // Return a usable general-purpose reg after point P. Return 0 if no reg is // available. @@ -136,7 +130,7 @@ protected: BitVector Used = BitVector(NumRegs, false); if (IsCall) { - RA.getInstUsedRegsList(Point, Used, /*GetClobbers*/true); + RA.getInstUsedRegsList(Point, Used, /*GetClobbers*/ true); if (RA.isConservative(Used)) { Used = BC.MIB->getRegsUsedAsParams(); BC.MIB->getDefaultLiveOut(Used); @@ -149,9 +143,8 @@ protected: Used |= BC.MIB->getAliases(Point.getOperand(I).getReg(), /*OnlySmaller=*/false); } - for (auto - I = InstInfo.getImplicitUses(), - E = InstInfo.getImplicitUses() + InstInfo.getNumImplicitUses(); + for (auto I = InstInfo.getImplicitUses(), + E = InstInfo.getImplicitUses() + InstInfo.getNumImplicitUses(); I != E; ++I) { Used |= BC.MIB->getAliases(*I, false); } @@ -166,9 +159,7 @@ protected: return Next; } - StringRef getAnnotationName() const { - return StringRef("LivenessAnalysis"); - } + StringRef getAnnotationName() const { return StringRef("LivenessAnalysis"); } }; } // end namespace bolt diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h index 618a590c9117..51fe7fabca2d 100644 --- a/bolt/include/bolt/Passes/LongJmp.h +++ b/bolt/include/bolt/Passes/LongJmp.h @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// // //===----------------------------------------------------------------------===// + #ifndef BOLT_PASSES_LONGJMP_H #define BOLT_PASSES_LONGJMP_H @@ -77,8 +78,8 @@ class LongJmpPass : public BinaryFunctionPass { uint64_t DotAddress); uint64_t tentativeLayoutRelocColdPart(const BinaryContext &BC, - std::vector &SortedFunctions, - uint64_t DotAddress); + std::vector &SortedFunctions, + uint64_t DotAddress); void tentativeBBLayout(const BinaryFunction &Func); /// Update stubs addresses with their exact address after a round of stub @@ -151,7 +152,7 @@ public: void runOnFunctions(BinaryContext &BC) override; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/PLTCall.h b/bolt/include/bolt/Passes/PLTCall.h index 1c7a508b0d9b..4a0e0d695976 100644 --- a/bolt/include/bolt/Passes/PLTCall.h +++ b/bolt/include/bolt/Passes/PLTCall.h @@ -18,23 +18,20 @@ namespace bolt { class PLTCall : public BinaryFunctionPass { public: - /// PLT optimization type enum OptType : char { - OT_NONE = 0, /// Do not optimize - OT_HOT = 1, /// Optimize hot PLT calls - OT_ALL = 2 /// Optimize all PLT calls + OT_NONE = 0, /// Do not optimize + OT_HOT = 1, /// Optimize hot PLT calls + OT_ALL = 2 /// Optimize all PLT calls }; explicit PLTCall(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "PLT call optimization"; - } + const char *getName() const override { return "PLT call optimization"; } bool shouldPrint(const BinaryFunction &BF) const override { return BinaryFunctionPass::shouldPrint(BF); - } + } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/PatchEntries.h b/bolt/include/bolt/Passes/PatchEntries.h index 720a095c183d..f3f06c3a2957 100644 --- a/bolt/include/bolt/Passes/PatchEntries.h +++ b/bolt/include/bolt/Passes/PatchEntries.h @@ -31,12 +31,9 @@ class PatchEntries : public BinaryFunctionPass { }; public: - explicit PatchEntries() : BinaryFunctionPass(false) { - } + explicit PatchEntries() : BinaryFunctionPass(false) {} - const char *getName() const override { - return "patch-entries"; - } + const char *getName() const override { return "patch-entries"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/ReachingDefOrUse.h b/bolt/include/bolt/Passes/ReachingDefOrUse.h index d0c81a0d6cfc..a98210e0f14f 100644 --- a/bolt/include/bolt/Passes/ReachingDefOrUse.h +++ b/bolt/include/bolt/Passes/ReachingDefOrUse.h @@ -59,9 +59,7 @@ public: return (*this->getStateAt(B))[this->ExprToIdx[&A]]; } - void run() { - InstrsDataflowAnalysis, !Def>::run(); - } + void run() { InstrsDataflowAnalysis, !Def>::run(); } protected: /// Reference to the result of reg analysis @@ -132,8 +130,7 @@ protected: if (TrackingReg == None) { // Track all instructions Next.set(this->ExprToIdx[&Point]); - } - else { + } else { // Track only instructions relevant to TrackingReg BitVector Regs = BitVector(this->BC.MRI->getNumRegs(), false); if (Def) @@ -158,5 +155,4 @@ protected: } // end namespace bolt } // end namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/RegAnalysis.h b/bolt/include/bolt/Passes/RegAnalysis.h index 17654db5de24..c424c827b77f 100644 --- a/bolt/include/bolt/Passes/RegAnalysis.h +++ b/bolt/include/bolt/Passes/RegAnalysis.h @@ -38,8 +38,7 @@ public: /// set of clobbered registers. BitVector getFunctionClobberList(const BinaryFunction *Func); - RegAnalysis(BinaryContext &BC, - std::map *BFs, + RegAnalysis(BinaryContext &BC, std::map *BFs, BinaryFunctionCallGraph *CG); /// Compute the set of registers \p Inst may read from, marking them in @@ -57,13 +56,8 @@ public: /// expressing no specific knowledge of reg usage. bool isConservative(BitVector &Vec) const; - /// Set what to do when lacking information about a call - enum class ConservativeStrategy { - CLOBBERS_ALL, - CLOBBERS_ABI, - CLOBBERS_NONE - }; + enum class ConservativeStrategy { CLOBBERS_ALL, CLOBBERS_ABI, CLOBBERS_NONE }; void setConservativeStrategy(ConservativeStrategy S) { CS = S; } /// Print stats about the quality of our analysis @@ -89,10 +83,9 @@ private: /// Helper function used to get the set of clobbered/used regs whenever /// we know nothing about the function. void beConservative(BitVector &Result) const; - }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/RegReAssign.h b/bolt/include/bolt/Passes/RegReAssign.h index 6a309e061bcb..6fd73a8bf074 100644 --- a/bolt/include/bolt/Passes/RegReAssign.h +++ b/bolt/include/bolt/Passes/RegReAssign.h @@ -1,4 +1,5 @@ -//===--- Passes/RegReAssign.h ----------------------------------------------===// +//===--- Passes/RegReAssign.h +//----------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -57,7 +58,7 @@ public: void runOnFunctions(BinaryContext &BC) override; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Passes/ReorderAlgorithm.h b/bolt/include/bolt/Passes/ReorderAlgorithm.h index 4664ed7d109f..c718188b5ef9 100644 --- a/bolt/include/bolt/Passes/ReorderAlgorithm.h +++ b/bolt/include/bolt/Passes/ReorderAlgorithm.h @@ -18,15 +18,12 @@ #include #include - namespace llvm { class raw_ostream; - namespace bolt { - /// Objects of this class implement various basic block clustering algorithms. /// Basic block clusters are chains of basic blocks that should be laid out /// in this order to maximize performace. These algorithms group basic blocks @@ -71,32 +68,33 @@ protected: uint64_t Count; EdgeTy(const BinaryBasicBlock *Src, const BinaryBasicBlock *Dst, - uint64_t Count) : - Src(Src), Dst(Dst), Count(Count) {} + uint64_t Count) + : Src(Src), Dst(Dst), Count(Count) {} void print(raw_ostream &OS) const; }; struct EdgeHash { - size_t operator() (const EdgeTy &E) const; + size_t operator()(const EdgeTy &E) const; }; struct EdgeEqual { - bool operator() (const EdgeTy &A, const EdgeTy &B) const; + bool operator()(const EdgeTy &A, const EdgeTy &B) const; }; // Virtual methods that allow custom specialization of the heuristic used by // the algorithm to select edges. - virtual void initQueue( - std::vector &Queue, const BinaryFunction &BF) = 0; - virtual void adjustQueue( - std::vector &Queue, const BinaryFunction &BF) = 0; - virtual bool areClustersCompatible( - const ClusterTy &Front, const ClusterTy &Back, const EdgeTy &E) const = 0; + virtual void initQueue(std::vector &Queue, + const BinaryFunction &BF) = 0; + virtual void adjustQueue(std::vector &Queue, + const BinaryFunction &BF) = 0; + virtual bool areClustersCompatible(const ClusterTy &Front, + const ClusterTy &Back, + const EdgeTy &E) const = 0; // Map from basic block to owning cluster index. - using BBToClusterMapTy = std::unordered_map; + using BBToClusterMapTy = + std::unordered_map; BBToClusterMapTy BBToClusterMap; public: @@ -105,21 +103,17 @@ public: void reset() override; }; - /// This clustering algorithm is based on a greedy heuristic suggested by /// Pettis and Hansen (PLDI '90). class PHGreedyClusterAlgorithm : public GreedyClusterAlgorithm { protected: - void initQueue( - std::vector &Queue, const BinaryFunction &BF) override; - void adjustQueue( - std::vector &Queue, const BinaryFunction &BF) override; - bool areClustersCompatible( - const ClusterTy &Front, const ClusterTy &Back, const EdgeTy &E) const - override; + void initQueue(std::vector &Queue, const BinaryFunction &BF) override; + void adjustQueue(std::vector &Queue, + const BinaryFunction &BF) override; + bool areClustersCompatible(const ClusterTy &Front, const ClusterTy &Back, + const EdgeTy &E) const override; }; - /// This clustering algorithm is based on a greedy heuristic that is a /// modification of the heuristic suggested by Pettis (PLDI '90). It is /// geared towards minimizing branches. @@ -140,19 +134,16 @@ private: int64_t calculateWeight(const EdgeTy &E, const BinaryFunction &BF) const; protected: - void initQueue( - std::vector &Queue, const BinaryFunction &BF) override; - void adjustQueue( - std::vector &Queue, const BinaryFunction &BF) override; - bool areClustersCompatible( - const ClusterTy &Front, const ClusterTy &Back, const EdgeTy &E) const - override; + void initQueue(std::vector &Queue, const BinaryFunction &BF) override; + void adjustQueue(std::vector &Queue, + const BinaryFunction &BF) override; + bool areClustersCompatible(const ClusterTy &Front, const ClusterTy &Back, + const EdgeTy &E) const override; public: void reset() override; }; - /// Objects of this class implement various basic block reordering alogrithms. /// Most of these algorithms depend on a clustering alogrithm. /// Here we have 3 conflicting goals as to how to layout clusters. If we want @@ -168,25 +159,24 @@ protected: std::unique_ptr CAlgo; public: - ReorderAlgorithm() { } - explicit ReorderAlgorithm(std::unique_ptr CAlgo) : - CAlgo(std::move(CAlgo)) { } + ReorderAlgorithm() {} + explicit ReorderAlgorithm(std::unique_ptr CAlgo) + : CAlgo(std::move(CAlgo)) {} using BasicBlockOrder = BinaryFunction::BasicBlockOrderType; /// Reorder the basic blocks of the given function and store the new order in /// the new Clusters vector. - virtual void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const = 0; + virtual void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const = 0; void setClusterAlgorithm(ClusterAlgorithm *CAlgo) { this->CAlgo.reset(CAlgo); } - virtual ~ReorderAlgorithm() { } + virtual ~ReorderAlgorithm() {} }; - /// Dynamic programming implementation for the TSP, applied to BB layout. Find /// the optimal way to maximize weight during a path traversing all BBs. In /// this way, we will convert the hottest branches into fall-throughs. @@ -195,73 +185,70 @@ public: /// only be used for small functions. class TSPReorderAlgorithm : public ReorderAlgorithm { public: - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; - /// Simple algorithm that groups basic blocks into clusters and then /// lays them out cluster after cluster. class OptimizeReorderAlgorithm : public ReorderAlgorithm { public: - explicit OptimizeReorderAlgorithm(std::unique_ptr CAlgo) : - ReorderAlgorithm(std::move(CAlgo)) { } + explicit OptimizeReorderAlgorithm(std::unique_ptr CAlgo) + : ReorderAlgorithm(std::move(CAlgo)) {} - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; - /// This reorder algorithm tries to ensure that all inter-cluster edges are /// predicted as not-taken, by enforcing a topological order to make /// targets appear after sources, creating forward branches. class OptimizeBranchReorderAlgorithm : public ReorderAlgorithm { public: explicit OptimizeBranchReorderAlgorithm( - std::unique_ptr CAlgo) : - ReorderAlgorithm(std::move(CAlgo)) { } + std::unique_ptr CAlgo) + : ReorderAlgorithm(std::move(CAlgo)) {} - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; - /// This reorder tries to separate hot from cold blocks to maximize the /// probability that unfrequently executed code doesn't pollute the cache, by /// putting clusters in descending order of hotness. class OptimizeCacheReorderAlgorithm : public ReorderAlgorithm { public: explicit OptimizeCacheReorderAlgorithm( - std::unique_ptr CAlgo) : - ReorderAlgorithm(std::move(CAlgo)) { } + std::unique_ptr CAlgo) + : ReorderAlgorithm(std::move(CAlgo)) {} - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; /// A new reordering algorithm for basic blocks, ext-tsp class ExtTSPReorderAlgorithm : public ReorderAlgorithm { public: - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; /// Toy example that simply reverses the original basic block order. class ReverseReorderAlgorithm : public ReorderAlgorithm { public: - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; /// Create clusters as usual and place them in random order. class RandomClusterReorderAlgorithm : public ReorderAlgorithm { public: explicit RandomClusterReorderAlgorithm( - std::unique_ptr CAlgo) : - ReorderAlgorithm(std::move(CAlgo)) { } + std::unique_ptr CAlgo) + : ReorderAlgorithm(std::move(CAlgo)) {} - void reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const override; + void reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const override; }; } // namespace bolt diff --git a/bolt/include/bolt/Passes/ReorderData.h b/bolt/include/bolt/Passes/ReorderData.h index 0489ce8076b4..c5bb875739ee 100644 --- a/bolt/include/bolt/Passes/ReorderData.h +++ b/bolt/include/bolt/Passes/ReorderData.h @@ -22,8 +22,7 @@ public: using DataOrder = std::vector>; private: - DataOrder baseOrder(BinaryContext &BC, - const BinarySection &Section) const; + DataOrder baseOrder(BinaryContext &BC, const BinarySection &Section) const; std::unordered_map BinaryDataCounts; @@ -35,30 +34,24 @@ private: sortedByCount(BinaryContext &BC, const BinarySection &Section) const; std::pair - sortedByFunc(BinaryContext &BC, - const BinarySection &Section, + sortedByFunc(BinaryContext &BC, const BinarySection &Section, std::map &BFs) const; - void printOrder(const BinarySection &Section, - DataOrder::const_iterator Begin, + void printOrder(const BinarySection &Section, DataOrder::const_iterator Begin, DataOrder::const_iterator End) const; /// Set the ordering of the section with \p SectionName. \p NewOrder is a /// vector of [old address, size] pairs. The new symbol order is implicit /// in the order of the vector. - void setSectionOrder(BinaryContext &BC, - BinarySection &OutputSection, - DataOrder::iterator Begin, - DataOrder::iterator End); + void setSectionOrder(BinaryContext &BC, BinarySection &OutputSection, + DataOrder::iterator Begin, DataOrder::iterator End); + + bool markUnmoveableSymbols(BinaryContext &BC, BinarySection &Section) const; - bool markUnmoveableSymbols(BinaryContext &BC, - BinarySection &Section) const; public: explicit ReorderData() : BinaryFunctionPass(false) {} - const char *getName() const override { - return "reorder-data"; - } + const char *getName() const override { return "reorder-data"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/ReorderFunctions.h b/bolt/include/bolt/Passes/ReorderFunctions.h index de2b3b986e06..bf3019568904 100644 --- a/bolt/include/bolt/Passes/ReorderFunctions.h +++ b/bolt/include/bolt/Passes/ReorderFunctions.h @@ -24,6 +24,7 @@ class ReorderFunctions : public BinaryFunctionPass { void reorder(std::vector &&Clusters, std::map &BFs); + public: enum ReorderType : char { RT_NONE = 0, @@ -36,11 +37,9 @@ public: }; explicit ReorderFunctions(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "reorder-functions"; - } + const char *getName() const override { return "reorder-functions"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/RetpolineInsertion.h b/bolt/include/bolt/Passes/RetpolineInsertion.h index a65bdaf68d7c..ea72377a2932 100644 --- a/bolt/include/bolt/Passes/RetpolineInsertion.h +++ b/bolt/include/bolt/Passes/RetpolineInsertion.h @@ -1,4 +1,4 @@ -//===--- Passes/RetpolineInsertion.h ---------------------------------------===// +//===--- Passes/RetpolineInsertion.h --------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/bolt/include/bolt/Passes/ShrinkWrapping.h b/bolt/include/bolt/Passes/ShrinkWrapping.h index 45a9ee89aab3..4e8a3107c874 100644 --- a/bolt/include/bolt/Passes/ShrinkWrapping.h +++ b/bolt/include/bolt/Passes/ShrinkWrapping.h @@ -58,8 +58,8 @@ public: std::vector OffsetsByReg; BitVector HasRestores; std::vector SavingCost; - std::vector SaveFIEByReg; - std::vector LoadFIEByReg; + std::vector SaveFIEByReg; + std::vector LoadFIEByReg; CalleeSavedAnalysis(const FrameAnalysis &FA, BinaryFunction &BF, DataflowInfoManager &Info, @@ -106,14 +106,9 @@ public: /// Returns the identifying string used to annotate instructions with metadata /// for this analysis. These are deleted in the destructor. - static StringRef getSaveTagName() { - return StringRef("CSA-SavedReg"); - } - - static StringRef getRestoreTagName() { - return StringRef("CSA-RestoredReg"); - } + static StringRef getSaveTagName() { return StringRef("CSA-SavedReg"); } + static StringRef getRestoreTagName() { return StringRef("CSA-RestoredReg"); } }; /// Identifies in a given binary function all stack regions being used and allow @@ -165,8 +160,8 @@ public: WorklistItem(ActionType Action, int OffsetUpdate) : Action(Action), OffsetUpdate(OffsetUpdate) {} }; -private: +private: /// Mark the stack region identified by \p Offset and \p Size to be a /// no-touch zone, whose accesses cannot be relocated to another region. void blacklistRegion(int64_t Offset, int64_t Size); @@ -216,7 +211,7 @@ private: if (OffsetCFIRegTagIndex) return *OffsetCFIRegTagIndex; OffsetCFIRegTagIndex = - BC.MIB->getOrCreateAnnotationIndex(getOffsetCFIRegTagName()); + BC.MIB->getOrCreateAnnotationIndex(getOffsetCFIRegTagName()); return *OffsetCFIRegTagIndex; } @@ -240,8 +235,7 @@ public: /// Retrieves the value of the callee-saved register that is restored by this /// instruction or 0 if this is not a CSR restore instruction. uint16_t getOffsetCFIReg(const MCInst &Inst) { - auto Val = - BC.MIB->tryGetAnnotationAs(Inst, getOffsetCFIRegTag()); + auto Val = BC.MIB->tryGetAnnotationAs(Inst, getOffsetCFIRegTag()); if (Val) return *Val; return 0; @@ -279,18 +273,13 @@ public: /// accesses. void initialize(); - static StringRef getTodoTagName() { - return StringRef("SLM-TodoTag"); - } + static StringRef getTodoTagName() { return StringRef("SLM-TodoTag"); } - static StringRef getSlotTagName() { - return StringRef("SLM-SlotTag"); - } + static StringRef getSlotTagName() { return StringRef("SLM-SlotTag"); } static StringRef getOffsetCFIRegTagName() { return StringRef("SLM-OffsetCFIReg"); } - }; /// Implements a pass to optimize callee-saved register spills. These spills @@ -350,12 +339,10 @@ public: /// Insertion todo items scheduled to happen at the end of BBs. Since we /// can't annotate BBs we maintain this bookkeeping here. - DenseMap> Todo; + DenseMap> Todo; /// Annotation name used to tag instructions with removal or insertion actions - static StringRef getAnnotationName() { - return StringRef("ShrinkWrap-Todo"); - } + static StringRef getAnnotationName() { return StringRef("ShrinkWrap-Todo"); } unsigned getAnnotationIndex() { if (AnnotationIndex) @@ -386,8 +373,7 @@ private: uint64_t &TotalEstimatedWin); /// Populate the Todo map with worklistitems to change the function - template - void scheduleChange(ProgramPoint PP, T&& ...Item) { + template void scheduleChange(ProgramPoint PP, T &&...Item) { if (PP.isInst()) { auto &WList = BC.MIB->getOrCreateAnnotationAs>( *PP.getInst(), getAnnotationIndex(), AllocatorId); diff --git a/bolt/include/bolt/Passes/SplitFunctions.h b/bolt/include/bolt/Passes/SplitFunctions.h index 930429fe3794..1f7c0cd6e702 100644 --- a/bolt/include/bolt/Passes/SplitFunctions.h +++ b/bolt/include/bolt/Passes/SplitFunctions.h @@ -23,10 +23,10 @@ class SplitFunctions : public BinaryFunctionPass { public: /// Settings for splitting function bodies into hot/cold partitions. enum SplittingType : char { - ST_NONE = 0, /// Do not split functions. - ST_LARGE, /// In non-relocation mode, only split functions that - /// are too large to fit into the original space. - ST_ALL, /// Split all functions. + ST_NONE = 0, /// Do not split functions. + ST_LARGE, /// In non-relocation mode, only split functions that + /// are too large to fit into the original space. + ST_ALL, /// Split all functions. }; private: @@ -38,13 +38,11 @@ private: public: explicit SplitFunctions(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) { } + : BinaryFunctionPass(PrintPass) {} bool shouldOptimize(const BinaryFunction &BF) const override; - const char *getName() const override { - return "split-functions"; - } + const char *getName() const override { return "split-functions"; } void runOnFunctions(BinaryContext &BC) override; }; diff --git a/bolt/include/bolt/Passes/StackAllocationAnalysis.h b/bolt/include/bolt/Passes/StackAllocationAnalysis.h index 3c1b96a424b0..8a88efca4519 100644 --- a/bolt/include/bolt/Passes/StackAllocationAnalysis.h +++ b/bolt/include/bolt/Passes/StackAllocationAnalysis.h @@ -38,9 +38,7 @@ public: SPT(SPT) {} virtual ~StackAllocationAnalysis() {} - void run() { - InstrsDataflowAnalysis::run(); - } + void run() { InstrsDataflowAnalysis::run(); } protected: void preflight(); diff --git a/bolt/include/bolt/Passes/StackAvailableExpressions.h b/bolt/include/bolt/Passes/StackAvailableExpressions.h index ef2a58b27135..ebff31129234 100644 --- a/bolt/include/bolt/Passes/StackAvailableExpressions.h +++ b/bolt/include/bolt/Passes/StackAvailableExpressions.h @@ -33,9 +33,7 @@ public: BinaryFunction &BF); virtual ~StackAvailableExpressions() {} - void run() { - InstrsDataflowAnalysis::run(); - } + void run() { InstrsDataflowAnalysis::run(); } protected: const RegAnalysis &RA; diff --git a/bolt/include/bolt/Passes/StackPointerTracking.h b/bolt/include/bolt/Passes/StackPointerTracking.h index 76d2d088298c..3d0075906559 100644 --- a/bolt/include/bolt/Passes/StackPointerTracking.h +++ b/bolt/include/bolt/Passes/StackPointerTracking.h @@ -202,9 +202,7 @@ public: MCPlusBuilder::AllocatorIdTy AllocatorId = 0); virtual ~StackPointerTracking() {} - void run() { - StackPointerTrackingBase::run(); - } + void run() { StackPointerTrackingBase::run(); } }; } // end namespace bolt @@ -214,5 +212,4 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, } // end namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/StackReachingUses.h b/bolt/include/bolt/Passes/StackReachingUses.h index 6272b636fa3a..7977dd7c4235 100644 --- a/bolt/include/bolt/Passes/StackReachingUses.h +++ b/bolt/include/bolt/Passes/StackReachingUses.h @@ -48,9 +48,7 @@ public: bool isStoreUsed(const FrameIndexEntry &StoreFIE, ExprIterator Candidates, bool IncludeLocalAccesses = true) const; - void run() { - InstrsDataflowAnalysis::run(); - } + void run() { InstrsDataflowAnalysis::run(); } protected: // Reference to the result of stack frame analysis diff --git a/bolt/include/bolt/Passes/StokeInfo.h b/bolt/include/bolt/Passes/StokeInfo.h index 883248fb2373..bebc4ac1e274 100644 --- a/bolt/include/bolt/Passes/StokeInfo.h +++ b/bolt/include/bolt/Passes/StokeInfo.h @@ -54,9 +54,7 @@ struct StokeFuncInfo { bool HasRipAddr; bool Omitted; - StokeFuncInfo() { - reset(); - } + StokeFuncInfo() { reset(); } void reset() { FuncName = ""; @@ -64,40 +62,30 @@ struct StokeFuncInfo { NumLoops = MaxLoopDepth = 0; HotSize = TotalSize = 0; Score = 0; - IsLoopFree - = HasCall - = HeapOut - = StackOut - = HasRipAddr - = Omitted - = false; + IsLoopFree = HasCall = HeapOut = StackOut = HasRipAddr = Omitted = false; DefIn.clear(); LiveOut.clear(); } void printCsvHeader(std::ofstream &Outfile) { if (Outfile.is_open()) { - Outfile - << "FuncName,Offset,Size,NumInstrs,NumBlocks," - << "IsLoopFree,NumLoops,MaxLoopDepth," - << "HotSize,TotalSize," - << "Score," - << "HasCall," - << "DefIn,LiveOut,HeapOut,StackOut," - << "HasRipAddr," - << "Omitted\n"; + Outfile << "FuncName,Offset,Size,NumInstrs,NumBlocks," + << "IsLoopFree,NumLoops,MaxLoopDepth," + << "HotSize,TotalSize," + << "Score," + << "HasCall," + << "DefIn,LiveOut,HeapOut,StackOut," + << "HasRipAddr," + << "Omitted\n"; } } void printData(std::ofstream &Outfile) { if (Outfile.is_open()) { - Outfile - << FuncName << "," - << Offset << "," << Size << "," << NumInstrs << "," << NumBlocks << "," - << IsLoopFree << "," << NumLoops << "," << MaxLoopDepth << "," - << HotSize << "," << TotalSize << "," - << Score << "," - << HasCall << ",\"{ "; + Outfile << FuncName << "," << Offset << "," << Size << "," << NumInstrs + << "," << NumBlocks << "," << IsLoopFree << "," << NumLoops << "," + << MaxLoopDepth << "," << HotSize << "," << TotalSize << "," + << Score << "," << HasCall << ",\"{ "; for (std::string S : DefIn) { Outfile << "%" << S << " "; } @@ -105,9 +93,8 @@ struct StokeFuncInfo { for (std::string S : LiveOut) { Outfile << "%" << S << " "; } - Outfile << "}\"," << HeapOut << "," << StackOut << "," - << HasRipAddr << "," - << Omitted << "\n"; + Outfile << "}\"," << HeapOut << "," << StackOut << "," << HasRipAddr + << "," << Omitted << "\n"; } } }; @@ -126,9 +113,7 @@ private: public: StokeInfo(const cl::opt &PrintPass) : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "stoke-get-stat"; - } + const char *getName() const override { return "stoke-get-stat"; } void checkInstr(const BinaryFunction &BF, StokeFuncInfo &FuncInfo); @@ -142,5 +127,4 @@ public: } // namespace bolt } // namespace llvm - #endif diff --git a/bolt/include/bolt/Passes/ValidateInternalCalls.h b/bolt/include/bolt/Passes/ValidateInternalCalls.h index dc1fb8367c20..ec1293b661b0 100644 --- a/bolt/include/bolt/Passes/ValidateInternalCalls.h +++ b/bolt/include/bolt/Passes/ValidateInternalCalls.h @@ -51,11 +51,9 @@ namespace bolt { class ValidateInternalCalls : public BinaryFunctionPass { public: explicit ValidateInternalCalls(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) {} + : BinaryFunctionPass(PrintPass) {} - const char *getName() const override { - return "validate-internal-calls"; - } + const char *getName() const override { return "validate-internal-calls"; } void runOnFunctions(BinaryContext &BC) override; @@ -83,9 +81,7 @@ private: /// The annotation tag we use to keep track of internal calls we already /// processed. - StringRef getProcessedICTag() const { - return "ProcessedInternalCall"; - } + StringRef getProcessedICTag() const { return "ProcessedInternalCall"; } void clearAnnotations(BinaryFunction &Function) const { const BinaryContext &BC = Function.getBinaryContext(); @@ -97,7 +93,7 @@ private: } }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 404b54ce21e6..264d3c503e6f 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -97,9 +97,9 @@ public: /// taken in the path started at FirstLBR.To and ending at SecondLBR.From. /// Return NoneType if trace is invalid or the list of fall-throughs /// otherwise. - Optional - getFallthroughsInTrace(const BinaryFunction &Func, uint64_t From, - uint64_t To) const; + Optional getFallthroughsInTrace(const BinaryFunction &Func, + uint64_t From, + uint64_t To) const; /// If available, fetch the address of the hot part linked to the cold part /// at \p Address. Return 0 otherwise. @@ -128,8 +128,8 @@ private: /// translation map entry const static uint32_t BRANCHENTRY = 0x80000000; }; -} +} // namespace bolt -} +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h index 8296ba71d842..84963b9c4c17 100644 --- a/bolt/include/bolt/Profile/DataAggregator.h +++ b/bolt/include/bolt/Profile/DataAggregator.h @@ -49,20 +49,15 @@ class BoltAddressTranslation; /// specified by the user. class DataAggregator : public DataReader { public: - explicit DataAggregator(StringRef Filename) - : DataReader(Filename) { + explicit DataAggregator(StringRef Filename) : DataReader(Filename) { start(); } ~DataAggregator(); - StringRef getReaderName() const override { - return "perf data aggregator"; - } + StringRef getReaderName() const override { return "perf data aggregator"; } - bool isTrustedSource() const override { - return true; - } + bool isTrustedSource() const override { return true; } Error preprocessProfile(BinaryContext &BC) override; @@ -110,8 +105,7 @@ private: struct Trace { uint64_t From; uint64_t To; - Trace(uint64_t From, uint64_t To) - : From(From), To(To) {} + Trace(uint64_t From, uint64_t To) : From(From), To(To) {} bool operator==(const Trace &Other) const { return From == Other.From && To == Other.To; } @@ -141,7 +135,7 @@ private: std::unordered_map BasicSamples; std::vector MemSamples; - template void clear(T& Container) { + template void clear(T &Container) { T TempContainer; TempContainer.swap(Container); } @@ -204,9 +198,8 @@ private: /// execution order. /// /// Return true if the trace is valid, false otherwise. - bool recordTrace(BinaryFunction &BF, - const LBREntry &First, - const LBREntry &Second, + bool recordTrace( + BinaryFunction &BF, const LBREntry &First, const LBREntry &Second, uint64_t Count = 1, SmallVector, 16> *Branches = nullptr) const; @@ -481,7 +474,7 @@ private: void dump(const PerfBranchSample &Sample) const; void dump(const PerfMemSample &Sample) const; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/DataReader.h b/bolt/include/bolt/Profile/DataReader.h index 53857ca31264..04ba70eb7fc9 100644 --- a/bolt/include/bolt/Profile/DataReader.h +++ b/bolt/include/bolt/Profile/DataReader.h @@ -38,8 +38,8 @@ struct LBREntry { }; inline raw_ostream &operator<<(raw_ostream &OS, const LBREntry &LBR) { - OS << "0x" << Twine::utohexstr(LBR.From) - << " -> 0x" << Twine::utohexstr(LBR.To); + OS << "0x" << Twine::utohexstr(LBR.From) << " -> 0x" + << Twine::utohexstr(LBR.To); return OS; } @@ -55,8 +55,7 @@ struct Location { : IsSymbol(IsSymbol), Name(Name), Offset(Offset) {} bool operator==(const Location &RHS) const { - return IsSymbol == RHS.IsSymbol && - Name == RHS.Name && + return IsSymbol == RHS.IsSymbol && Name == RHS.Name && (Name == "[heap]" || Offset == RHS.Offset); } @@ -86,8 +85,7 @@ struct BranchInfo { Branches(Branches) {} bool operator==(const BranchInfo &RHS) const { - return From == RHS.From && - To == RHS.To; + return From == RHS.From && To == RHS.To; } bool operator<(const BranchInfo &RHS) const { @@ -122,10 +120,10 @@ struct FuncBranchData { FuncBranchData() {} FuncBranchData(StringRef Name, ContainerTy Data) - : Name(Name), Data(std::move(Data)) {} + : Name(Name), Data(std::move(Data)) {} FuncBranchData(StringRef Name, ContainerTy Data, ContainerTy EntryData) - : Name(Name), Data(std::move(Data)), EntryData(std::move(EntryData)) {} + : Name(Name), Data(std::move(Data)), EntryData(std::move(EntryData)) {} ErrorOr getBranch(uint64_t From, uint64_t To) const; @@ -136,8 +134,8 @@ struct FuncBranchData { ErrorOr getDirectCallBranch(uint64_t From) const; /// Find all the branches originating at From. - iterator_range getBranchRange( - uint64_t From) const; + iterator_range + getBranchRange(uint64_t From) const; /// Append the branch data of another function located \p Offset bytes away /// from the entry of this function. @@ -182,15 +180,13 @@ struct MemInfo { return false; } - void mergeWith(const MemInfo &MI) { - Count += MI.Count; - } + void mergeWith(const MemInfo &MI) { Count += MI.Count; } void print(raw_ostream &OS) const; void prettyPrint(raw_ostream &OS) const; MemInfo(const Location &Offset, const Location &Addr, uint64_t Count = 0) - : Offset(Offset), Addr(Addr), Count(Count) {} + : Offset(Offset), Addr(Addr), Count(Count) {} friend raw_ostream &operator<<(raw_ostream &OS, const MemInfo &MI) { MI.prettyPrint(OS); @@ -219,7 +215,7 @@ struct FuncMemData { FuncMemData() {} FuncMemData(StringRef Name, ContainerTy Data) - : Name(Name), Data(std::move(Data)) {} + : Name(Name), Data(std::move(Data)) {} }; /// Similar to BranchInfo, but instead of recording from-to address (an edge), @@ -229,12 +225,9 @@ struct SampleInfo { Location Loc; int64_t Hits; - SampleInfo(Location Loc, int64_t Hits) - : Loc(std::move(Loc)), Hits(Hits) {} + SampleInfo(Location Loc, int64_t Hits) : Loc(std::move(Loc)), Hits(Hits) {} - bool operator==(const SampleInfo &RHS) const { - return Loc == RHS.Loc; - } + bool operator==(const SampleInfo &RHS) const { return Loc == RHS.Loc; } bool operator<(const SampleInfo &RHS) const { if (Loc < RHS.Loc) @@ -273,16 +266,11 @@ struct FuncSampleData { class DataReader : public ProfileReaderBase { public: explicit DataReader(StringRef Filename) - : ProfileReaderBase(Filename), - Diag(errs()) {} + : ProfileReaderBase(Filename), Diag(errs()) {} - StringRef getReaderName() const override { - return "branch profile reader"; - } + StringRef getReaderName() const override { return "branch profile reader"; } - bool isTrustedSource() const override { - return false; - } + bool isTrustedSource() const override { return false; } virtual Error preprocessProfile(BinaryContext &BC) override; @@ -295,9 +283,7 @@ public: virtual bool mayHaveProfileData(const BinaryFunction &BF) override; /// Return all event names used to collect this profile - virtual StringSet<> getEventNames() const override { - return EventNames; - } + virtual StringSet<> getEventNames() const override { return EventNames; } protected: /// Read profile information available for the function. @@ -337,9 +323,8 @@ protected: /// \p Count could be 0 if verification of the branch is required. /// /// Return true if the branch is valid, false otherwise. - bool recordBranch(BinaryFunction &BF, - uint64_t From, uint64_t To, uint64_t Count = 1, - uint64_t Mispreds = 0) const; + bool recordBranch(BinaryFunction &BF, uint64_t From, uint64_t To, + uint64_t Count = 1, uint64_t Mispreds = 0) const; /// Parses the input bolt data file into internal data structures. We expect /// the file format to follow the syntax below. @@ -403,8 +388,7 @@ protected: /// Return mem data matching one of the names in \p FuncNames. FuncMemData *getMemDataForNames(const std::vector &FuncNames); - FuncSampleData * - getFuncSampleData(const std::vector &FuncNames); + FuncSampleData *getFuncSampleData(const std::vector &FuncNames); /// Return a vector of all FuncBranchData matching the list of names. /// Internally use fuzzy matching to match special names like LTO-generated @@ -450,9 +434,9 @@ protected: using NamesToSamplesMapTy = StringMap; using NamesToMemEventsMapTy = StringMap; using FuncsToBranchesMapTy = - std::unordered_map; + std::unordered_map; using FuncsToMemDataMapTy = - std::unordered_map; + std::unordered_map; /// Dumps the entire data structures parsed. Used for debugging. void dump() const; @@ -485,14 +469,14 @@ protected: bool expectAndConsumeFS(); void consumeAllRemainingFS(); bool checkAndConsumeNewLine(); - ErrorOr parseString(char EndChar, bool EndNl=false); - ErrorOr parseNumberField(char EndChar, bool EndNl=false); - ErrorOr parseHexField(char EndChar, bool EndNl=false); + ErrorOr parseString(char EndChar, bool EndNl = false); + ErrorOr parseNumberField(char EndChar, bool EndNl = false); + ErrorOr parseHexField(char EndChar, bool EndNl = false); ErrorOr parseLocation(char EndChar, bool EndNl, bool ExpectMemLoc); - ErrorOr parseLocation(char EndChar, bool EndNl=false) { + ErrorOr parseLocation(char EndChar, bool EndNl = false) { return parseLocation(EndChar, EndNl, false); } - ErrorOr parseMemLocation(char EndChar, bool EndNl=false) { + ErrorOr parseMemLocation(char EndChar, bool EndNl = false) { return parseLocation(EndChar, EndNl, true); } ErrorOr parseBranchInfo(); @@ -524,28 +508,28 @@ protected: StringMap> LTOCommonNameMemMap; }; -} +} // namespace bolt /// DenseMapInfo allows us to use the DenseMap LLVM data structure to store /// Locations -template<> struct DenseMapInfo { +template <> struct DenseMapInfo { static inline bolt::Location getEmptyKey() { return bolt::Location(true, StringRef(), static_cast(-1LL)); } static inline bolt::Location getTombstoneKey() { - return bolt::Location(true, StringRef(), static_cast(-2LL));; + return bolt::Location(true, StringRef(), static_cast(-2LL)); + ; } static unsigned getHashValue(const bolt::Location &L) { return (unsigned(DenseMapInfo::getHashValue(L.Name)) >> 4) ^ (unsigned(L.Offset)); } - static bool isEqual(const bolt::Location &LHS, - const bolt::Location &RHS) { + static bool isEqual(const bolt::Location &LHS, const bolt::Location &RHS) { return LHS.IsSymbol == RHS.IsSymbol && LHS.Name == RHS.Name && LHS.Offset == RHS.Offset; } }; -} +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/Heatmap.h b/bolt/include/bolt/Profile/Heatmap.h index f2471d7fef95..8dfa359855f6 100644 --- a/bolt/include/bolt/Profile/Heatmap.h +++ b/bolt/include/bolt/Profile/Heatmap.h @@ -37,11 +37,10 @@ class Heatmap { std::map Map; public: - explicit Heatmap(uint64_t BucketSize = 4096, - uint64_t MinAddress = 0, + explicit Heatmap(uint64_t BucketSize = 4096, uint64_t MinAddress = 0, uint64_t MaxAddress = std::numeric_limits::max()) - : BucketSize(BucketSize), MinAddress(MinAddress), MaxAddress(MaxAddress) - {}; + : BucketSize(BucketSize), MinAddress(MinAddress), + MaxAddress(MaxAddress){}; inline bool ignoreAddress(uint64_t Address) const { return (Address > MaxAddress) || (Address < MinAddress); @@ -58,9 +57,7 @@ public: uint64_t Count); /// Return the number of ranges that failed to register. - uint64_t getNumInvalidRanges() const { - return NumSkippedRanges; - } + uint64_t getNumInvalidRanges() const { return NumSkippedRanges; } void print(StringRef FileName) const; @@ -70,9 +67,7 @@ public: void printCDF(raw_ostream &OS) const; - size_t size() const { - return Map.size(); - } + size_t size() const { return Map.size(); } }; } // namespace bolt diff --git a/bolt/include/bolt/Profile/ProfileReaderBase.h b/bolt/include/bolt/Profile/ProfileReaderBase.h index fa9455d470f2..72cf8e6c6124 100644 --- a/bolt/include/bolt/Profile/ProfileReaderBase.h +++ b/bolt/include/bolt/Profile/ProfileReaderBase.h @@ -62,8 +62,7 @@ public: ProfileReaderBase &operator=(ProfileReaderBase &&) = delete; /// Construct a reader for a given file. - explicit ProfileReaderBase(StringRef Filename) - : Filename(Filename) {} + explicit ProfileReaderBase(StringRef Filename) : Filename(Filename) {} virtual ~ProfileReaderBase() = default; @@ -96,14 +95,10 @@ public: /// Return true if the profile contains an entry for a local object /// that has an associated file name. - virtual bool hasLocalsWithFileName() const { - return true; - } + virtual bool hasLocalsWithFileName() const { return true; } /// Return all event names used to collect this profile. - virtual StringSet<> getEventNames() const { - return StringSet<>(); - } + virtual StringSet<> getEventNames() const { return StringSet<>(); } /// Return true if the source of the profile should be trusted. E.g., even /// good source of profile data may contain discrepancies. Nevertheless, the @@ -111,7 +106,7 @@ public: virtual bool isTrustedSource() const = 0; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h index d46686dcf783..d13d20cde8cd 100644 --- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h +++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h @@ -32,14 +32,11 @@ struct CallSiteInfo { uint64_t Mispreds{0}; bool operator==(const CallSiteInfo &Other) const { - return Offset == Other.Offset && - DestId == Other.DestId && + return Offset == Other.Offset && DestId == Other.DestId && EntryDiscriminator == Other.EntryDiscriminator; } - bool operator!=(const CallSiteInfo &Other) const { - return !(*this == Other); - } + bool operator!=(const CallSiteInfo &Other) const { return !(*this == Other); } bool operator<(const CallSiteInfo &Other) const { if (Offset < Other.Offset) @@ -166,14 +163,13 @@ template <> struct MappingTraits { YamlIO.mapRequired("exec", BFP.ExecCount); YamlIO.mapRequired("nblocks", BFP.NumBasicBlocks); YamlIO.mapOptional("blocks", BFP.Blocks, - std::vector()); + std::vector()); } }; LLVM_YAML_STRONG_TYPEDEF(uint16_t, PROFILE_PF) -template <> -struct ScalarBitSetTraits { +template <> struct ScalarBitSetTraits { static void bitset(IO &io, PROFILE_PF &value) { io.bitSetCase(value, "lbr", BinaryFunction::PF_LBR); io.bitSetCase(value, "sample", BinaryFunction::PF_SAMPLE); @@ -184,12 +180,12 @@ struct ScalarBitSetTraits { namespace bolt { struct BinaryProfileHeader { uint32_t Version{1}; - std::string FileName; // Name of the profiled binary. - std::string Id; // BuildID. + std::string FileName; // Name of the profiled binary. + std::string Id; // BuildID. PROFILE_PF Flags{BinaryFunction::PF_NONE}; - // Type of the profile. - std::string Origin; // How the profile was obtained. - std::string EventNames; // Events used for sample profile. + // Type of the profile. + std::string Origin; // How the profile was obtained. + std::string EventNames; // Events used for sample profile. }; } // end namespace bolt @@ -217,7 +213,7 @@ struct BinaryProfile { BinaryProfileHeader Header; std::vector Functions; }; -} +} // namespace bolt template <> struct MappingTraits { static void mapping(IO &YamlIO, bolt::BinaryProfile &BP) { @@ -229,5 +225,4 @@ template <> struct MappingTraits { } // end namespace yaml } // end namespace llvm - #endif diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h index 8934f270920f..18f86c2be174 100644 --- a/bolt/include/bolt/Profile/YAMLProfileReader.h +++ b/bolt/include/bolt/Profile/YAMLProfileReader.h @@ -21,15 +21,11 @@ namespace bolt { class YAMLProfileReader : public ProfileReaderBase { public: explicit YAMLProfileReader(StringRef Filename) - : ProfileReaderBase(Filename) {} + : ProfileReaderBase(Filename) {} - StringRef getReaderName() const override { - return "YAML profile reader"; - } + StringRef getReaderName() const override { return "YAML profile reader"; } - bool isTrustedSource() const override { - return false; - } + bool isTrustedSource() const override { return false; } Error readProfilePreCFG(BinaryContext &BC) override { return Error::success(); @@ -67,7 +63,7 @@ private: /// Map a common LTO prefix to a set of binary functions. StringMap> - LTOCommonNameFunctionMap; + LTOCommonNameFunctionMap; /// Strict matching of a name in a profile to its contents. StringMap ProfileNameToProfile; @@ -96,7 +92,7 @@ private: bool usesEvent(StringRef Name) const; }; -} -} +} // namespace bolt +} // namespace llvm #endif diff --git a/bolt/include/bolt/Profile/YAMLProfileWriter.h b/bolt/include/bolt/Profile/YAMLProfileWriter.h index 9d52bbc3c2e9..21a187bf34e9 100644 --- a/bolt/include/bolt/Profile/YAMLProfileWriter.h +++ b/bolt/include/bolt/Profile/YAMLProfileWriter.h @@ -28,8 +28,7 @@ class YAMLProfileWriter { public: explicit YAMLProfileWriter(const std::string &Filename) - : Filename(Filename) { - } + : Filename(Filename) {} /// Save execution profile for that instance. std::error_code writeProfile(const RewriteInstance &RI); diff --git a/bolt/include/bolt/Rewrite/BinaryPassManager.h b/bolt/include/bolt/Rewrite/BinaryPassManager.h index d773d6e3678c..7cd156f37738 100644 --- a/bolt/include/bolt/Rewrite/BinaryPassManager.h +++ b/bolt/include/bolt/Rewrite/BinaryPassManager.h @@ -25,20 +25,18 @@ class BinaryContext; class BinaryFunctionPassManager { private: BinaryContext &BC; - std::vector>> Passes; + std::vector>> + Passes; - public: +public: static const char TimerGroupName[]; static const char TimerGroupDesc[]; - BinaryFunctionPassManager(BinaryContext &BC) - : BC(BC) {} + BinaryFunctionPassManager(BinaryContext &BC) : BC(BC) {} /// Adds a pass to this manager based on the value of its corresponding /// command-line option. - void registerPass(std::unique_ptr Pass, - const bool Run) { + void registerPass(std::unique_ptr Pass, const bool Run) { Passes.emplace_back(Run, std::move(Pass)); } diff --git a/bolt/include/bolt/Rewrite/DWARFRewriter.h b/bolt/include/bolt/Rewrite/DWARFRewriter.h index 8137f8fbfa05..f20662cfeffc 100644 --- a/bolt/include/bolt/Rewrite/DWARFRewriter.h +++ b/bolt/include/bolt/Rewrite/DWARFRewriter.h @@ -121,25 +121,21 @@ class DWARFRewriter { DWARFUnit *Unit; DWARFDebugInfoEntry DIE; - DWARFDieWrapper(DWARFUnit *Unit, DWARFDebugInfoEntry DIE) : - Unit(Unit), - DIE(DIE) {} + DWARFDieWrapper(DWARFUnit *Unit, DWARFDebugInfoEntry DIE) + : Unit(Unit), DIE(DIE) {} - DWARFDieWrapper(DWARFDie &Die) : - Unit(Die.getDwarfUnit()), - DIE(*Die.getDebugInfoEntry()) {} + DWARFDieWrapper(DWARFDie &Die) + : Unit(Die.getDwarfUnit()), DIE(*Die.getDebugInfoEntry()) {} - operator DWARFDie() { - return DWARFDie(Unit, &DIE); - } + operator DWARFDie() { return DWARFDie(Unit, &DIE); } }; /// DIEs with abbrevs that were not converted to DW_AT_ranges. /// We only update those when all DIEs have been processed to guarantee that /// the abbrev (which is shared) is intact. using PendingRangesType = std::unordered_map< - const DWARFAbbreviationDeclaration *, - std::vector>>; + const DWARFAbbreviationDeclaration *, + std::vector>>; PendingRangesType PendingRanges; diff --git a/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h b/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h index 4974559a8c2f..e31e65fbfddf 100644 --- a/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h +++ b/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h @@ -24,12 +24,9 @@ class BinaryContext; /// Class responsible for allocating and managing code and data sections. class ExecutableFileMemoryManager : public SectionMemoryManager { private: - uint8_t *allocateSection(intptr_t Size, - unsigned Alignment, - unsigned SectionID, - StringRef SectionName, - bool IsCode, - bool IsReadOnly); + uint8_t *allocateSection(intptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool IsCode, bool IsReadOnly); BinaryContext &BC; bool AllowStubs; diff --git a/bolt/include/bolt/Rewrite/MachORewriteInstance.h b/bolt/include/bolt/Rewrite/MachORewriteInstance.h index 02eb7810db21..96d7431f6a99 100644 --- a/bolt/include/bolt/Rewrite/MachORewriteInstance.h +++ b/bolt/include/bolt/Rewrite/MachORewriteInstance.h @@ -60,7 +60,8 @@ class MachORewriteInstance { void runOptimizationPasses(); void emitAndLink(); - void writeInstrumentationSection(StringRef SectionName, raw_pwrite_stream &OS); + void writeInstrumentationSection(StringRef SectionName, + raw_pwrite_stream &OS); void rewriteFile(); public: diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h index 171c1babc134..d8a84eb87212 100644 --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -40,9 +40,8 @@ class ProfileReaderBase; /// events. class RewriteInstance { public: - RewriteInstance(llvm::object::ELFObjectFileBase *File, - const int Argc, const char *const *Argv, - StringRef ToolPath); + RewriteInstance(llvm::object::ELFObjectFileBase *File, const int Argc, + const char *const *Argv, StringRef ToolPath); ~RewriteInstance(); /// Assign profile from \p Filename to this instance. @@ -56,14 +55,10 @@ public: void compare(RewriteInstance &RI2); /// Return binary context. - const BinaryContext &getBinaryContext() const { - return *BC; - } + const BinaryContext &getBinaryContext() const { return *BC; } /// Return total score of all functions for this instance. - uint64_t getTotalScore() const { - return BC->TotalScore; - } + uint64_t getTotalScore() const { return BC->TotalScore; } /// Return the name of the input file. StringRef getInputFilename() const { @@ -278,7 +273,7 @@ private: ELF_FUNCTION(finalizeSectionStringTable); /// Return a name of the input file section in the output file. - template + template std::string getOutputSectionName(const ELFObjType &Obj, const ELFShdrTy &Section); @@ -351,9 +346,9 @@ private: public: /// Standard ELF sections we overwrite. static constexpr const char *SectionsToOverwrite[] = { - ".shstrtab", - ".symtab", - ".strtab", + ".shstrtab", + ".symtab", + ".strtab", }; /// Debug section to we overwrite while updating the debug info. diff --git a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h index e07d3a39541c..1119604c2658 100644 --- a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h +++ b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h @@ -14,7 +14,7 @@ #ifndef BOLT_RUNTIMELIBS_RUNTIME_LIBRARY_H #define BOLT_RUNTIMELIBS_RUNTIME_LIBRARY_H -#include +#include "llvm/ADT/StringRef.h" #include #include diff --git a/bolt/include/bolt/Utils/CommandLineOpts.h b/bolt/include/bolt/Utils/CommandLineOpts.h index 0eb085de8230..1a05ff83d7fc 100644 --- a/bolt/include/bolt/Utils/CommandLineOpts.h +++ b/bolt/include/bolt/Utils/CommandLineOpts.h @@ -71,12 +71,12 @@ extern llvm::cl::opt Verbosity; /// Return true if we should process all functions in the binary. bool processAllFunctions(); -} +} // namespace opts namespace llvm { namespace bolt { extern const char *BoltRevision; } -} +} // namespace llvm #endif diff --git a/bolt/include/bolt/Utils/Utils.h b/bolt/include/bolt/Utils/Utils.h index 372cc754cf01..4a93f159e6d0 100644 --- a/bolt/include/bolt/Utils/Utils.h +++ b/bolt/include/bolt/Utils/Utils.h @@ -22,7 +22,7 @@ class MCCFIInstruction; namespace bolt { /// Free memory allocated for \p List. -template void clearList(T& List) { +template void clearList(T &List) { T TempList; TempList.swap(List); } diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp index b5aecca46460..0f5827d2f8bb 100644 --- a/bolt/lib/Core/BinaryBasicBlock.cpp +++ b/bolt/lib/Core/BinaryBasicBlock.cpp @@ -15,7 +15,6 @@ #include "llvm/MC/MCInst.h" #include "llvm/Support/Errc.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" namespace llvm { @@ -27,9 +26,7 @@ bool operator<(const BinaryBasicBlock &LHS, const BinaryBasicBlock &RHS) { return LHS.Index < RHS.Index; } -bool BinaryBasicBlock::hasCFG() const { - return getParent()->hasCFG(); -} +bool BinaryBasicBlock::hasCFG() const { return getParent()->hasCFG(); } bool BinaryBasicBlock::isEntryPoint() const { return getParent()->isEntryPoint(*this); @@ -62,8 +59,8 @@ BinaryBasicBlock::iterator BinaryBasicBlock::getFirstNonPseudo() { BinaryBasicBlock::reverse_iterator BinaryBasicBlock::getLastNonPseudo() { const BinaryContext &BC = Function->getBinaryContext(); - for (auto RII = Instructions.rbegin(), E = Instructions.rend(); - RII != E; ++RII) { + for (auto RII = Instructions.rbegin(), E = Instructions.rend(); RII != E; + ++RII) { if (!BC.MIB->isPseudo(*RII)) return RII; } @@ -89,12 +86,11 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { auto Itr = UniqueSyms.find(Succ->getLabel()); if (Itr != UniqueSyms.end()) { UniqueSyms.erase(Itr); - } else { + } else { // Work on the assumption that jump table blocks don't // have a conditional successor. Valid = false; - errs() << "BOLT-WARNING: Jump table successor " - << Succ->getName() + errs() << "BOLT-WARNING: Jump table successor " << Succ->getName() << " not contained in the jump table.\n"; } } @@ -126,18 +122,18 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { Valid = !CondBranch && !UncondBranch; break; case 1: { - const bool HasCondBlock = CondBranch && - Function->getBasicBlockForLabel(BC.MIB->getTargetSymbol(*CondBranch)); + const bool HasCondBlock = + CondBranch && Function->getBasicBlockForLabel( + BC.MIB->getTargetSymbol(*CondBranch)); Valid = !CondBranch || !HasCondBlock; break; } case 2: - Valid = - (CondBranch && - (TBB == getConditionalSuccessor(true)->getLabel() && - ((!UncondBranch && !FBB) || - (UncondBranch && - FBB == getConditionalSuccessor(false)->getLabel())))); + Valid = (CondBranch && + (TBB == getConditionalSuccessor(true)->getLabel() && + ((!UncondBranch && !FBB) || + (UncondBranch && + FBB == getConditionalSuccessor(false)->getLabel())))); break; } } @@ -167,9 +163,8 @@ BinaryBasicBlock *BinaryBasicBlock::getSuccessor(const MCSymbol *Label) const { return nullptr; } -BinaryBasicBlock * -BinaryBasicBlock::getSuccessor(const MCSymbol *Label, - BinaryBranchInfo &BI) const { +BinaryBasicBlock *BinaryBasicBlock::getSuccessor(const MCSymbol *Label, + BinaryBranchInfo &BI) const { auto BIIter = branch_info_begin(); for (BinaryBasicBlock *BB : successors()) { if (BB->getLabel() == Label) { @@ -202,8 +197,8 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const { // Find the last CFI preceding Instr in this basic block. const MCInst *LastCFI = nullptr; bool InstrSeen = (Instr == nullptr); - for (auto RII = Instructions.rbegin(), E = Instructions.rend(); - RII != E; ++RII) { + for (auto RII = Instructions.rbegin(), E = Instructions.rend(); RII != E; + ++RII) { if (!InstrSeen) { InstrSeen = (&*RII == Instr); continue; @@ -253,8 +248,8 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const { assert(Depth == 0 && "unbalanced RememberState/RestoreState stack"); // Skip any GNU_args_size. - while (State >= 0 && - FDEProgram[State].getOperation() == MCCFIInstruction::OpGnuArgsSize){ + while (State >= 0 && FDEProgram[State].getOperation() == + MCCFIInstruction::OpGnuArgsSize) { --State; } } @@ -263,8 +258,7 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const { return State + 1; } -void BinaryBasicBlock::addSuccessor(BinaryBasicBlock *Succ, - uint64_t Count, +void BinaryBasicBlock::addSuccessor(BinaryBasicBlock *Succ, uint64_t Count, uint64_t MispredictedCount) { Successors.push_back(Succ); BranchInfo.push_back({Count, MispredictedCount}); @@ -323,7 +317,7 @@ void BinaryBasicBlock::removePredecessor(BinaryBasicBlock *Pred, bool Multiple) { // Note: the predecessor could be listed multiple times. bool Erased = false; - for (auto PredI = Predecessors.begin(); PredI != Predecessors.end(); ) { + for (auto PredI = Predecessors.begin(); PredI != Predecessors.end();) { if (*PredI == Pred) { Erased = true; PredI = Predecessors.erase(PredI); @@ -374,17 +368,12 @@ void BinaryBasicBlock::adjustExecutionCount(double Ratio) { } } -bool BinaryBasicBlock::analyzeBranch(const MCSymbol *&TBB, - const MCSymbol *&FBB, +bool BinaryBasicBlock::analyzeBranch(const MCSymbol *&TBB, const MCSymbol *&FBB, MCInst *&CondBranch, MCInst *&UncondBranch) { auto &MIB = Function->getBinaryContext().MIB; - return MIB->analyzeBranch(Instructions.begin(), - Instructions.end(), - TBB, - FBB, - CondBranch, - UncondBranch); + return MIB->analyzeBranch(Instructions.begin(), Instructions.end(), TBB, FBB, + CondBranch, UncondBranch); } bool BinaryBasicBlock::isMacroOpFusionPair(const_iterator I) const { @@ -503,9 +492,8 @@ uint32_t BinaryBasicBlock::getNumPseudos() const { } if (N != NumPseudos) { errs() << "BOLT-ERROR: instructions for basic block " << getName() - << " in function " << *Function << ": calculated pseudos " - << N << ", set pseudos " << NumPseudos << ", size " << size() - << '\n'; + << " in function " << *Function << ": calculated pseudos " << N + << ", set pseudos " << NumPseudos << ", size " << size() << '\n'; llvm_unreachable("pseudos mismatch"); } #endif @@ -529,7 +517,8 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const { assert(Itr != Successors.end()); const BinaryBranchInfo &BI = BranchInfo[Itr - Successors.begin()]; if (BI.Count && BI.Count != COUNT_NO_PROFILE) { - if (TotalMispreds == 0) TotalMispreds = 1; + if (TotalMispreds == 0) + TotalMispreds = 1; return std::make_pair(double(BI.Count) / TotalCount, double(BI.MispredictedCount) / TotalMispreds); } @@ -540,7 +529,8 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const { void BinaryBasicBlock::dump() const { BinaryContext &BC = Function->getBinaryContext(); - if (Label) outs() << Label->getName() << ":\n"; + if (Label) + outs() << Label->getName() << ":\n"; BC.printInstructions(outs(), Instructions.begin(), Instructions.end(), getOffset()); outs() << "preds:"; diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index 1e1df7a93a63..9754448d6f20 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -77,8 +77,7 @@ namespace bolt { BinaryContext::BinaryContext(std::unique_ptr Ctx, std::unique_ptr DwCtx, std::unique_ptr TheTriple, - const Target *TheTarget, - std::string TripleName, + const Target *TheTarget, std::string TripleName, std::unique_ptr MCE, std::unique_ptr MOFI, std::unique_ptr AsmInfo, @@ -89,21 +88,12 @@ BinaryContext::BinaryContext(std::unique_ptr Ctx, std::unique_ptr MIB, std::unique_ptr MRI, std::unique_ptr DisAsm) - : Ctx(std::move(Ctx)), - DwCtx(std::move(DwCtx)), - TheTriple(std::move(TheTriple)), - TheTarget(TheTarget), - TripleName(TripleName), - MCE(std::move(MCE)), - MOFI(std::move(MOFI)), - AsmInfo(std::move(AsmInfo)), - MII(std::move(MII)), - STI(std::move(STI)), - InstPrinter(std::move(InstPrinter)), - MIA(std::move(MIA)), - MIB(std::move(MIB)), - MRI(std::move(MRI)), - DisAsm(std::move(DisAsm)) { + : Ctx(std::move(Ctx)), DwCtx(std::move(DwCtx)), + TheTriple(std::move(TheTriple)), TheTarget(TheTarget), + TripleName(TripleName), MCE(std::move(MCE)), MOFI(std::move(MOFI)), + AsmInfo(std::move(AsmInfo)), MII(std::move(MII)), STI(std::move(STI)), + InstPrinter(std::move(InstPrinter)), MIA(std::move(MIA)), + MIB(std::move(MIB)), MRI(std::move(MRI)), DisAsm(std::move(DisAsm)) { Relocation::Arch = this->TheTriple->getArch(); PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize; } @@ -242,8 +232,8 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, std::move(Ctx), std::move(DwCtx), std::move(TheTriple), TheTarget, std::string(TripleName), std::move(MCE), std::move(MOFI), std::move(AsmInfo), std::move(MII), std::move(STI), - std::move(InstructionPrinter), std::move(MIA), nullptr, - std::move(MRI), std::move(DisAsm)); + std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI), + std::move(DisAsm)); BC->TTypeEncoding = TTypeEncoding; BC->LSDAEncoding = LSDAEncoding; @@ -259,12 +249,12 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC, } bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const { - if (opts::HotText && (SymbolName == "__hot_start" || - SymbolName == "__hot_end")) + if (opts::HotText && + (SymbolName == "__hot_start" || SymbolName == "__hot_end")) return true; - if (opts::HotData && (SymbolName == "__hot_data_start" || - SymbolName == "__hot_data_end")) + if (opts::HotData && + (SymbolName == "__hot_data_start" || SymbolName == "__hot_data_end")) return true; if (SymbolName == "_end") @@ -327,8 +317,8 @@ void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) { const uint64_t Address = GAI->second->getAddress(); const uint64_t Size = GAI->second->getSize(); - auto fixParents = - [&](BinaryDataMapType::iterator Itr, BinaryData *NewParent) { + auto fixParents = [&](BinaryDataMapType::iterator Itr, + BinaryData *NewParent) { BinaryData *OldParent = Itr->second->Parent; Itr->second->Parent = NewParent; ++Itr; @@ -357,9 +347,9 @@ void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) { if (Size != 0) { BinaryData *BD = GAI->second->Parent ? GAI->second->Parent : GAI->second; auto Itr = std::next(GAI); - while (Itr != BinaryDataMap.end() && - BD->containsRange(Itr->second->getAddress(), - Itr->second->getSize())) { + while ( + Itr != BinaryDataMap.end() && + BD->containsRange(Itr->second->getAddress(), Itr->second->getSize())) { Itr->second->Parent = BD; ++Itr; } @@ -370,8 +360,7 @@ iterator_range BinaryContext::getSubBinaryData(BinaryData *BD) { auto Start = std::next(BinaryDataMap.find(BD->getAddress())); auto End = Start; - while (End != BinaryDataMap.end() && - BD->isAncestorOf(End->second)) { + while (End != BinaryDataMap.end() && BD->isAncestorOf(End->second)) { ++End; } return make_range(Start, End); @@ -406,19 +395,17 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF, // a section, it could be an absolute address too. ErrorOr Section = getSectionForAddress(Address); if (Section && Section->isText()) { - if (BF.containsAddress(Address, /*UseMaxSize=*/ isAArch64())) { + if (BF.containsAddress(Address, /*UseMaxSize=*/isAArch64())) { if (Address != BF.getAddress()) { // The address could potentially escape. Mark it as another entry // point into the function. if (opts::Verbosity >= 1) { outs() << "BOLT-INFO: potentially escaped address 0x" - << Twine::utohexstr(Address) << " in function " - << BF << '\n'; + << Twine::utohexstr(Address) << " in function " << BF << '\n'; } BF.HasInternalLabelReference = true; return std::make_pair( - BF.addEntryPointAtOffset(Address - BF.getAddress()), - Addend); + BF.addEntryPointAtOffset(Address - BF.getAddress()), Addend); } } else { BF.InterproceduralReferences.insert(Address); @@ -431,7 +418,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF, const MemoryContentsType MemType = analyzeMemoryAt(Address, BF); if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) { const MCSymbol *Symbol = - getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC); + getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC); return std::make_pair(Symbol, Addend); } @@ -447,8 +434,8 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF, return std::make_pair(TargetSymbol, Addend); } -MemoryContentsType -BinaryContext::analyzeMemoryAt(uint64_t Address, BinaryFunction &BF) { +MemoryContentsType BinaryContext::analyzeMemoryAt(uint64_t Address, + BinaryFunction &BF) { if (!isX86()) return MemoryContentsType::UNKNOWN; @@ -459,8 +446,8 @@ BinaryContext::analyzeMemoryAt(uint64_t Address, BinaryFunction &BF) { // consider it a tail call. if (opts::Verbosity > 1) { errs() << "BOLT-WARNING: no section for address 0x" - << Twine::utohexstr(Address) << " referenced from function " - << BF << '\n'; + << Twine::utohexstr(Address) << " referenced from function " << BF + << '\n'; } return MemoryContentsType::UNKNOWN; } @@ -571,9 +558,10 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address, } } - const uint64_t Value = (Type == JumpTable::JTT_PIC) - ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize) - : *getPointerAtAddress(EntryAddress); + const uint64_t Value = + (Type == JumpTable::JTT_PIC) + ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize) + : *getPointerAtAddress(EntryAddress); // __builtin_unreachable() case. if (Value == BF.getAddress() + BF.getSize()) { @@ -732,8 +720,7 @@ void BinaryContext::skipMarkedFragments() { FragmentsToSkip.clear(); } -MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, - Twine Prefix, +MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, Twine Prefix, uint64_t Size, uint16_t Alignment, unsigned Flags) { @@ -874,11 +861,8 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { const uint64_t StartOffset = Offset; for (; Offset < BF.getMaxSize(); Offset += InstrSize, InstrAddress += InstrSize) { - if (!DisAsm->getInstruction(Instr, - InstrSize, - FunctionData->slice(Offset), - InstrAddress, - nulls())) + if (!DisAsm->getInstruction(Instr, InstrSize, FunctionData->slice(Offset), + InstrAddress, nulls())) break; if (!Predicate(Instr)) break; @@ -919,8 +903,7 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { }; // Skip over nops, jumps, and zero padding. Allow interleaving (this happens). - while (skipInstructions(isNoop) || - skipInstructions(isSkipJump) || + while (skipInstructions(isNoop) || skipInstructions(isSkipJump) || skipZeros()) ; @@ -930,9 +913,8 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) { if (opts::Verbosity >= 1) { errs() << "BOLT-WARNING: bad padding at address 0x" << Twine::utohexstr(BF.getAddress() + BF.getSize()) - << " starting at offset " - << (Offset - BF.getSize()) << " in function " - << BF << '\n' + << " starting at offset " << (Offset - BF.getSize()) + << " in function " << BF << '\n' << FunctionData->slice(BF.getSize(), BF.getMaxSize() - BF.getSize()) << '\n'; } @@ -960,8 +942,7 @@ void BinaryContext::adjustCodePadding() { } } -MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, - uint64_t Address, +MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, uint64_t Address, uint64_t Size, uint16_t Alignment, unsigned Flags) { @@ -974,12 +955,8 @@ MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, ErrorOr SectionOrErr = getSectionForAddress(Address); BinarySection &Section = SectionOrErr ? SectionOrErr.get() : absoluteSection(); - BD = new BinaryData(*Symbol, - Address, - Size, - Alignment ? Alignment : 1, - Section, - Flags); + BD = new BinaryData(*Symbol, Address, Size, Alignment ? Alignment : 1, + Section, Flags); GAI = BinaryDataMap.emplace(Address, BD).first; GlobalSymbols[Name] = BD; updateObjectNesting(GAI); @@ -1055,8 +1032,7 @@ void BinaryContext::generateSymbolHashes() { // First check if a non-anonymous alias exists and move it to the front. if (BD.getSymbols().size() > 1) { - auto Itr = std::find_if(BD.getSymbols().begin(), - BD.getSymbols().end(), + auto Itr = std::find_if(BD.getSymbols().begin(), BD.getSymbols().end(), [&](const MCSymbol *Symbol) { return !isInternalSymbolName(Symbol->getName()); }); @@ -1074,8 +1050,8 @@ void BinaryContext::generateSymbolHashes() { const uint64_t Hash = BD.getSection().hash(BD); const size_t Idx = Name.find("0x"); - std::string NewName = (Twine(Name.substr(0, Idx)) + - "_" + Twine::utohexstr(Hash)).str(); + std::string NewName = + (Twine(Name.substr(0, Idx)) + "_" + Twine::utohexstr(Hash)).str(); if (getBinaryDataByName(NewName)) { // Ignore collisions for symbols that appear to be padding // (i.e. all zeros or a "hole") @@ -1088,8 +1064,7 @@ void BinaryContext::generateSymbolHashes() { } continue; } - BD.Symbols.insert(BD.Symbols.begin(), - Ctx->getOrCreateSymbol(NewName)); + BD.Symbols.insert(BD.Symbols.begin(), Ctx->getOrCreateSymbol(NewName)); GlobalSymbols[NewName] = &BD; } if (NumCollisions) { @@ -1115,8 +1090,8 @@ bool BinaryContext::registerFragment(BinaryFunction &TargetFunction, Function.setSimple(false); } if (opts::Verbosity >= 1) { - outs() << "BOLT-INFO: marking " << TargetFunction - << " as a fragment of " << Function << '\n'; + outs() << "BOLT-INFO: marking " << TargetFunction << " as a fragment of " + << Function << '\n'; } return true; } @@ -1160,16 +1135,14 @@ void BinaryContext::processInterproceduralReferences(BinaryFunction &Function) { if (opts::processAllFunctions()) { errs() << "BOLT-ERROR: cannot process binaries with unmarked " - << "object in code at address 0x" - << Twine::utohexstr(Address) << " belonging to section " - << SectionName << " in current mode\n"; + << "object in code at address 0x" << Twine::utohexstr(Address) + << " belonging to section " << SectionName << " in current mode\n"; exit(1); } - TargetFunction = - getBinaryFunctionContainingAddress(Address, - /*CheckPastEnd=*/false, - /*UseMaxSize=*/true); + TargetFunction = getBinaryFunctionContainingAddress(Address, + /*CheckPastEnd=*/false, + /*UseMaxSize=*/true); // We are not going to overwrite non-simple functions, but for simple // ones - adjust the padding size. if (TargetFunction && TargetFunction->isSimple()) { @@ -1190,9 +1163,7 @@ void BinaryContext::postProcessSymbolTable() { BinaryData *BD = Entry.second; if ((BD->getName().startswith("SYMBOLat") || BD->getName().startswith("DATAat")) && - !BD->getParent() && - !BD->getSize() && - !BD->isAbsolute() && + !BD->getParent() && !BD->getSize() && !BD->isAbsolute() && BD->getSection()) { errs() << "BOLT-WARNING: zero-sized top level symbol: " << *BD << "\n"; Valid = false; @@ -1272,9 +1243,7 @@ void BinaryContext::fixBinaryDataHoles() { auto isNotHole = [&Section](const binary_data_iterator &Itr) { BinaryData *BD = Itr->second; - bool isHole = (!BD->getParent() && - !BD->getSize() && - BD->isObject() && + bool isHole = (!BD->getParent() && !BD->getSize() && BD->isObject() && (BD->getName().startswith("SYMBOLat0x") || BD->getName().startswith("DATAat0x") || BD->getName().startswith("ANONYMOUS"))); @@ -1321,8 +1290,8 @@ void BinaryContext::fixBinaryDataHoles() { assert(validateHoles() && "top level hole detected in object map"); } -void BinaryContext::printGlobalSymbols(raw_ostream& OS) const { - const BinarySection* CurrentSection = nullptr; +void BinaryContext::printGlobalSymbols(raw_ostream &OS) const { + const BinarySection *CurrentSection = nullptr; bool FirstSection = true; for (auto &Entry : BinaryDataMap) { @@ -1340,8 +1309,7 @@ void BinaryContext::printGlobalSymbols(raw_ostream& OS) const { } OS << "BOLT-INFO: Section " << Name << ", " << "0x" + Twine::utohexstr(Address) << ":" - << "0x" + Twine::utohexstr(Address + Size) << "/" - << Size << "\n"; + << "0x" + Twine::utohexstr(Address + Size) << "/" << Size << "\n"; CurrentSection = &Section; FirstSection = false; } @@ -1403,7 +1371,7 @@ std::vector BinaryContext::getSortedFunctions() { }); std::stable_sort(SortedFunctions.begin(), SortedFunctions.end(), - [] (const BinaryFunction *A, const BinaryFunction *B) { + [](const BinaryFunction *A, const BinaryFunction *B) { if (A->hasValidIndex() && B->hasValidIndex()) { return A->getIndex() < B->getIndex(); } @@ -1468,9 +1436,7 @@ void BinaryContext::preprocessDebugInfo() { uint64_t HighPC; DWARFUnit *Unit; - bool operator<(const CURange &Other) const { - return LowPC < Other.LowPC; - } + bool operator<(const CURange &Other) const { return LowPC < Other.LowPC; } }; // Building a map of address ranges to CUs similar to .debug_aranges and use @@ -1498,7 +1464,8 @@ void BinaryContext::preprocessDebugInfo() { const uint64_t FunctionAddress = KV.first; BinaryFunction &Function = KV.second; - auto It = std::partition_point(AllRanges.begin(), AllRanges.end(), + auto It = std::partition_point( + AllRanges.begin(), AllRanges.end(), [=](CURange R) { return R.HighPC <= FunctionAddress; }); if (It != AllRanges.end() && It->LowPC <= FunctionAddress) { Function.setDWARFUnit(It->Unit); @@ -1632,12 +1599,10 @@ void BinaryContext::printCFI(raw_ostream &OS, const MCCFIInstruction &Inst) { } } -void BinaryContext::printInstruction(raw_ostream &OS, - const MCInst &Instruction, +void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction, uint64_t Offset, - const BinaryFunction* Function, - bool PrintMCInst, - bool PrintMemData, + const BinaryFunction *Function, + bool PrintMCInst, bool PrintMemData, bool PrintRelocations) const { if (MIB->isEHLabel(Instruction)) { OS << " EH_LABEL: " << *MIB->getTargetSymbol(Instruction) << '\n'; @@ -1761,17 +1726,14 @@ BinarySection &BinaryContext::registerSection(SectionRef Section) { BinarySection & BinaryContext::registerSection(StringRef SectionName, const BinarySection &OriginalSection) { - return registerSection(new BinarySection(*this, - SectionName, - OriginalSection)); + return registerSection( + new BinarySection(*this, SectionName, OriginalSection)); } -BinarySection &BinaryContext::registerOrUpdateSection(StringRef Name, - unsigned ELFType, - unsigned ELFFlags, - uint8_t *Data, - uint64_t Size, - unsigned Alignment) { +BinarySection & +BinaryContext::registerOrUpdateSection(StringRef Name, unsigned ELFType, + unsigned ELFFlags, uint8_t *Data, + uint64_t Size, unsigned Alignment) { auto NamedSections = getSectionByName(Name); if (NamedSections.begin() != NamedSections.end()) { assert(std::next(NamedSections.begin()) == NamedSections.end() && @@ -1790,8 +1752,8 @@ BinarySection &BinaryContext::registerOrUpdateSection(StringRef Name, return *Section; } - return registerSection(new BinarySection(*this, Name, Data, Size, Alignment, - ELFType, ELFFlags)); + return registerSection( + new BinarySection(*this, Name, Data, Size, Alignment, ELFType, ELFFlags)); } bool BinaryContext::deregisterSection(BinarySection &Section) { @@ -1836,9 +1798,8 @@ BinarySection &BinaryContext::absoluteSection() { return registerOrUpdateSection("", ELF::SHT_NULL, 0u); } -ErrorOr -BinaryContext::getUnsignedValueAtAddress(uint64_t Address, - size_t Size) const { +ErrorOr BinaryContext::getUnsignedValueAtAddress(uint64_t Address, + size_t Size) const { const ErrorOr Section = getSectionForAddress(Address); if (!Section) return std::make_error_code(std::errc::bad_address); @@ -1852,9 +1813,8 @@ BinaryContext::getUnsignedValueAtAddress(uint64_t Address, return DE.getUnsigned(&ValueOffset, Size); } -ErrorOr -BinaryContext::getSignedValueAtAddress(uint64_t Address, - size_t Size) const { +ErrorOr BinaryContext::getSignedValueAtAddress(uint64_t Address, + size_t Size) const { const ErrorOr Section = getSectionForAddress(Address); if (!Section) return std::make_error_code(std::errc::bad_address); @@ -1868,32 +1828,22 @@ BinaryContext::getSignedValueAtAddress(uint64_t Address, return DE.getSigned(&ValueOffset, Size); } -void BinaryContext::addRelocation(uint64_t Address, - MCSymbol *Symbol, - uint64_t Type, - uint64_t Addend, +void BinaryContext::addRelocation(uint64_t Address, MCSymbol *Symbol, + uint64_t Type, uint64_t Addend, uint64_t Value) { ErrorOr Section = getSectionForAddress(Address); assert(Section && "cannot find section for address"); - Section->addRelocation(Address - Section->getAddress(), - Symbol, - Type, - Addend, + Section->addRelocation(Address - Section->getAddress(), Symbol, Type, Addend, Value); } -void BinaryContext::addDynamicRelocation(uint64_t Address, - MCSymbol *Symbol, - uint64_t Type, - uint64_t Addend, +void BinaryContext::addDynamicRelocation(uint64_t Address, MCSymbol *Symbol, + uint64_t Type, uint64_t Addend, uint64_t Value) { ErrorOr Section = getSectionForAddress(Address); assert(Section && "cannot find section for address"); - Section->addDynamicRelocation(Address - Section->getAddress(), - Symbol, - Type, - Addend, - Value); + Section->addDynamicRelocation(Address - Section->getAddress(), Symbol, Type, + Addend, Value); } bool BinaryContext::removeRelocationAt(uint64_t Address) { @@ -2097,16 +2047,15 @@ bool BinaryContext::validateEncoding(const MCInst &Inst, uint64_t BinaryContext::getHotThreshold() const { static uint64_t Threshold = 0; if (Threshold == 0) { - Threshold = std::max((uint64_t)opts::ExecutionCountThreshold, + Threshold = std::max( + (uint64_t)opts::ExecutionCountThreshold, NumProfiledFuncs ? SumExecutionCount / (2 * NumProfiledFuncs) : 1); } return Threshold; } -BinaryFunction * -BinaryContext::getBinaryFunctionContainingAddress(uint64_t Address, - bool CheckPastEnd, - bool UseMaxSize) { +BinaryFunction *BinaryContext::getBinaryFunctionContainingAddress( + uint64_t Address, bool CheckPastEnd, bool UseMaxSize) { auto FI = BinaryFunctions.upper_bound(Address); if (FI == BinaryFunctions.begin()) return nullptr; @@ -2121,8 +2070,7 @@ BinaryContext::getBinaryFunctionContainingAddress(uint64_t Address, return &FI->second; } -BinaryFunction * -BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) { +BinaryFunction *BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) { // First, try to find a function starting at the given address. If the // function was folded, this will get us the original folded function if it // wasn't removed from the list, e.g. in non-relocation mode. @@ -2145,7 +2093,7 @@ BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) { } DebugAddressRangesVector BinaryContext::translateModuleAddressRanges( - const DWARFAddressRangesVector &InputRanges) const { + const DWARFAddressRangesVector &InputRanges) const { DebugAddressRangesVector OutputRanges; for (const DWARFAddressRange Range : InputRanges) { @@ -2156,8 +2104,7 @@ DebugAddressRangesVector BinaryContext::translateModuleAddressRanges( break; const DebugAddressRangesVector FunctionRanges = Function.getOutputAddressRanges(); - std::move(std::begin(FunctionRanges), - std::end(FunctionRanges), + std::move(std::begin(FunctionRanges), std::end(FunctionRanges), std::back_inserter(OutputRanges)); std::advance(BFI, 1); } diff --git a/bolt/lib/Core/BinaryData.cpp b/bolt/lib/Core/BinaryData.cpp index 99306ae10aa9..58a3fcedac3e 100644 --- a/bolt/lib/Core/BinaryData.cpp +++ b/bolt/lib/Core/BinaryData.cpp @@ -16,7 +16,6 @@ using namespace llvm; using namespace bolt; -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" namespace opts { @@ -31,14 +30,10 @@ PrintSymbolAliases("print-aliases", cl::cat(BoltCategory)); } -bool BinaryData::isAbsolute() const { - return Flags & SymbolRef::SF_Absolute; -} +bool BinaryData::isAbsolute() const { return Flags & SymbolRef::SF_Absolute; } bool BinaryData::isMoveable() const { - return (!isAbsolute() && - (IsMoveable && - (!Parent || isTopLevelJumpTable()))); + return (!isAbsolute() && (IsMoveable && (!Parent || isTopLevelJumpTable()))); } void BinaryData::merge(const BinaryData *Other) { @@ -78,9 +73,7 @@ bool BinaryData::nameStartsWith(StringRef Prefix) const { return false; } -StringRef BinaryData::getSectionName() const { - return getSection().getName(); -} +StringRef BinaryData::getSectionName() const { return getSection().getName(); } StringRef BinaryData::getOutputSectionName() const { return getOutputSection().getName(); @@ -105,9 +98,7 @@ bool BinaryData::isMoved() const { return (getOffset() != OutputOffset || OutputSection != Section); } -void BinaryData::print(raw_ostream &OS) const { - printBrief(OS); -} +void BinaryData::print(raw_ostream &OS) const { printBrief(OS); } void BinaryData::printBrief(raw_ostream &OS) const { OS << "("; @@ -133,27 +124,17 @@ void BinaryData::printBrief(raw_ostream &OS) const { OS << ")"; } - OS << ", 0x" << Twine::utohexstr(getAddress()) - << ":0x" << Twine::utohexstr(getEndAddress()) - << "/" << getSize() << "/" << getAlignment() - << "/0x" << Twine::utohexstr(Flags); + OS << ", 0x" << Twine::utohexstr(getAddress()) << ":0x" + << Twine::utohexstr(getEndAddress()) << "/" << getSize() << "/" + << getAlignment() << "/0x" << Twine::utohexstr(Flags); OS << ")"; } -BinaryData::BinaryData(MCSymbol &Symbol, - uint64_t Address, - uint64_t Size, - uint16_t Alignment, - BinarySection &Section, +BinaryData::BinaryData(MCSymbol &Symbol, uint64_t Address, uint64_t Size, + uint16_t Alignment, BinarySection &Section, unsigned Flags) -: Section(&Section), - Address(Address), - Size(Size), - Alignment(Alignment), - Flags(Flags), - OutputSection(&Section), - OutputOffset(getOffset()) -{ + : Section(&Section), Address(Address), Size(Size), Alignment(Alignment), + Flags(Flags), OutputSection(&Section), OutputOffset(getOffset()) { Symbols.push_back(&Symbol); } diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp index d7b561bbbb04..0d93b0517a88 100644 --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -20,7 +20,6 @@ #include "llvm/Support/LEB128.h" #include "llvm/Support/SMLoc.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" using namespace llvm; @@ -97,7 +96,7 @@ size_t padFunction(const BinaryFunction &Function) { if (N == std::string::npos) continue; std::string Name = Spec.substr(0, N); - size_t Padding = std::stoull(Spec.substr(N+1)); + size_t Padding = std::stoull(Spec.substr(N + 1)); FunctionPadding[Name] = Padding; } } @@ -128,8 +127,7 @@ private: public: BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC) - : Streamer(Streamer), - BC(BC) {} + : Streamer(Streamer), BC(BC) {} /// Emit all code and data. void emitAll(StringRef OrgSecPrefix); @@ -293,9 +291,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) { if (BC.HasRelocations) { Streamer.emitCodeAlignment(BinaryFunction::MinAlign, &*BC.STI); - uint16_t MaxAlignBytes = EmitColdPart - ? Function.getMaxColdAlignmentBytes() - : Function.getMaxAlignmentBytes(); + uint16_t MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes() + : Function.getMaxAlignmentBytes(); if (MaxAlignBytes > 0) Streamer.emitCodeAlignment(Function.getAlignment(), &*BC.STI, MaxAlignBytes); @@ -411,8 +408,8 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart, if (EmitColdPart != BB->isCold()) continue; - if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) - && BB->getAlignment() > 1) { + if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) && + BB->getAlignment() > 1) { Streamer.emitCodeAlignment(BB->getAlignment(), &*BC.STI, BB->getAlignmentMaxBytes()); } @@ -425,9 +422,8 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart, // Check if special alignment for macro-fusion is needed. bool MayNeedMacroFusionAlignment = - (opts::AlignMacroOpFusion == MFT_ALL) || - (opts::AlignMacroOpFusion == MFT_HOT && - BB->getKnownExecutionCount()); + (opts::AlignMacroOpFusion == MFT_ALL) || + (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount()); BinaryBasicBlock::const_iterator MacroFusionPair; if (MayNeedMacroFusionAlignment) { MacroFusionPair = BB->getMacroOpFusionPair(); @@ -460,7 +456,8 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart, // Handle macro-fusion alignment. If we emitted a prefix as // the last instruction, we should've already emitted the associated // alignment hint, so don't emit it twice. - if (MayNeedMacroFusionAlignment && !LastIsPrefix && I == MacroFusionPair){ + if (MayNeedMacroFusionAlignment && !LastIsPrefix && + I == MacroFusionPair) { // This assumes the second instruction in the macro-op pair will get // assigned to its own MCRelaxableFragment. Since all JCC instructions // are relaxable, we should be safe. @@ -515,10 +512,8 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, StringRef SectionContents = BF.getOriginSection()->getContents(); // Raw contents of the function. - StringRef FunctionContents = - SectionContents.substr( - BF.getAddress() - BF.getOriginSection()->getAddress(), - BF.getMaxSize()); + StringRef FunctionContents = SectionContents.substr( + BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize()); if (opts::Verbosity && !OnBehalfOf) outs() << "BOLT-INFO: emitting constant island for function " << BF << "\n"; @@ -526,8 +521,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, // We split the island into smaller blocks and output labels between them. auto IS = Islands.Offsets.begin(); for (auto DataIter = Islands.DataOffsets.begin(); - DataIter != Islands.DataOffsets.end(); - ++DataIter) { + DataIter != Islands.DataOffsets.end(); ++DataIter) { uint64_t FunctionOffset = *DataIter; uint64_t EndOffset = 0ULL; @@ -546,7 +540,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, } if (FunctionOffset == EndOffset) - continue; // Size is zero, nothing to emit + continue; // Size is zero, nothing to emit auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) { if (FunctionOffset >= EndOffset) @@ -583,7 +577,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, // Emit labels, relocs and data while (IS != Islands.Offsets.end() && IS->first < EndOffset) { auto NextLabelOffset = - IS == Islands.Offsets.end() ? EndOffset : IS->first; + IS == Islands.Offsets.end() ? EndOffset : IS->first; auto NextStop = std::min(NextLabelOffset, EndOffset); assert(NextStop <= EndOffset && "internal overflow error"); emitCI(FunctionOffset, NextStop); @@ -665,8 +659,8 @@ SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, CurrentLineTable = BC.DwCtx->getLineTableForUnit( BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex)); // Add filename from the inlined function to the current CU. - CurrentFilenum = - BC.addDebugFilenameToUnit(FunctionUnitIndex, CurrentUnitIndex, + CurrentFilenum = BC.addDebugFilenameToUnit( + FunctionUnitIndex, CurrentUnitIndex, CurrentLineTable->Rows[RowReference.RowIndex - 1].File); } @@ -684,13 +678,8 @@ SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, if (FirstInstr) Flags |= DWARF2_FLAG_IS_STMT; - BC.Ctx->setCurrentDwarfLoc( - CurrentFilenum, - CurrentRow.Line, - CurrentRow.Column, - Flags, - CurrentRow.Isa, - CurrentRow.Discriminator); + BC.Ctx->setCurrentDwarfLoc(CurrentFilenum, CurrentRow.Line, CurrentRow.Column, + Flags, CurrentRow.Isa, CurrentRow.Discriminator); const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc(); BC.Ctx->clearDwarfLocSeen(); @@ -909,8 +898,8 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { Streamer.emitLabel(LSDASymbol); // Corresponding FDE start. - const MCSymbol *StartSymbol = EmitColdPart ? BF.getColdSymbol() - : BF.getSymbol(); + const MCSymbol *StartSymbol = + EmitColdPart ? BF.getColdSymbol() : BF.getSymbol(); // Emit the LSDA header. @@ -953,7 +942,7 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { }; } - Streamer.emitIntValue(TTypeEncoding, 1); // TType format + Streamer.emitIntValue(TTypeEncoding, 1); // TType format // See the comment in EHStreamer::emitExceptionTable() on to use // uleb128 encoding (which can use variable number of bytes to encode the same @@ -961,17 +950,16 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { // iteratively fixing sizes of the tables. unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength); unsigned TTypeBaseOffset = - sizeof(int8_t) + // Call site format - CallSiteTableLengthSize + // Call site table length size - CallSiteTableLength + // Call site table length - BF.getLSDAActionTable().size() + // Actions table size - BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size + sizeof(int8_t) + // Call site format + CallSiteTableLengthSize + // Call site table length size + CallSiteTableLength + // Call site table length + BF.getLSDAActionTable().size() + // Actions table size + BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset); - unsigned TotalSize = - sizeof(int8_t) + // LPStart format - sizeof(int8_t) + // TType format - TTypeBaseOffsetSize + // TType base offset size - TTypeBaseOffset; // TType base offset + unsigned TotalSize = sizeof(int8_t) + // LPStart format + sizeof(int8_t) + // TType format + TTypeBaseOffsetSize + // TType base offset size + TTypeBaseOffset; // TType base offset unsigned SizeAlign = (4 - TotalSize) & 3; // Account for any extra padding that will be added to the call site table @@ -1029,7 +1017,7 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { case dwarf::DW_EH_PE_pcrel: { if (TypeAddress) { const MCSymbol *TypeSymbol = - BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment); + BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment); MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol(); Streamer.emitLabel(DotSymbol); const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub( @@ -1144,8 +1132,8 @@ void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) { StringRef SectionName = Section.getName(); std::string EmitName = Section.isReordered() - ? std::string(Section.getOutputName()) - : OrgSecPrefix.str() + std::string(SectionName); + ? std::string(Section.getOutputName()) + : OrgSecPrefix.str() + std::string(SectionName); Section.emitAsData(Streamer, EmitName); Section.clearRelocations(); } @@ -1161,8 +1149,8 @@ void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC, void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF, bool EmitColdPart, bool EmitCodeOnly) { - BinaryEmitter(Streamer, BF.getBinaryContext()). - emitFunctionBody(BF, EmitColdPart, EmitCodeOnly); + BinaryEmitter(Streamer, BF.getBinaryContext()) + .emitFunctionBody(BF, EmitColdPart, EmitCodeOnly); } } // namespace bolt diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index eab1f5f1251c..b75107b3676d 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -40,7 +40,6 @@ #include #include -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" using namespace llvm; @@ -175,8 +174,7 @@ constexpr unsigned BinaryFunction::MinAlign; namespace { -template -bool emptyRange(const R &Range) { +template bool emptyRange(const R &Range) { return Range.begin() == Range.end(); } @@ -185,14 +183,15 @@ bool emptyRange(const R &Range) { /// to point to this information, which is represented by a /// DebugLineTableRowRef. The returned pointer is null if no debug line /// information for this instruction was found. -SMLoc findDebugLineInformationForInstructionAt(uint64_t Address, - DWARFUnit *Unit, const DWARFDebugLine::LineTable *LineTable) { +SMLoc findDebugLineInformationForInstructionAt( + uint64_t Address, DWARFUnit *Unit, + const DWARFDebugLine::LineTable *LineTable) { // We use the pointer in SMLoc to store an instance of DebugLineTableRowRef, // which occupies 64 bits. Thus, we can only proceed if the struct fits into // the pointer itself. - assert( - sizeof(decltype(SMLoc().getPointer())) >= sizeof(DebugLineTableRowRef) && - "Cannot fit instruction debug line information into SMLoc's pointer"); + assert(sizeof(decltype(SMLoc().getPointer())) >= + sizeof(DebugLineTableRowRef) && + "Cannot fit instruction debug line information into SMLoc's pointer"); SMLoc NullResult = DebugLineTableRowRef::NULL_ROW.toSMLoc(); uint32_t RowIndex = LineTable->lookupAddress( @@ -205,7 +204,7 @@ SMLoc findDebugLineInformationForInstructionAt(uint64_t Address, decltype(SMLoc().getPointer()) Ptr; DebugLineTableRowRef *InstructionLocation = - reinterpret_cast(&Ptr); + reinterpret_cast(&Ptr); InstructionLocation->DwCompileUnitIndex = Unit->getOffset(); InstructionLocation->RowIndex = RowIndex + 1; @@ -221,6 +220,19 @@ std::string buildSectionName(StringRef Prefix, StringRef Name, return (Prefix + Twine(NS.getID(Name))).str(); } +raw_ostream &operator<<(raw_ostream &OS, const BinaryFunction::State State) { + switch (State) { + case BinaryFunction::State::Empty: OS << "empty"; break; + case BinaryFunction::State::Disassembled: OS << "disassembled"; break; + case BinaryFunction::State::CFG: OS << "CFG constructed"; break; + case BinaryFunction::State::CFG_Finalized: OS << "CFG finalized"; break; + case BinaryFunction::State::EmittedCFG: OS << "emitted with CFG"; break; + case BinaryFunction::State::Emitted: OS << "emitted"; break; + } + + return OS; +} + } // namespace std::string BinaryFunction::buildCodeSectionName(StringRef Name, @@ -236,8 +248,7 @@ std::string BinaryFunction::buildColdCodeSectionName(StringRef Name, uint64_t BinaryFunction::Count = 0; -Optional -BinaryFunction::hasNameRegex(const StringRef Name) const { +Optional BinaryFunction::hasNameRegex(const StringRef Name) const { const std::string RegexName = (Twine("^") + StringRef(Name) + "$").str(); Regex MatchName(RegexName); Optional Match = forEachName( @@ -276,8 +287,7 @@ BinaryFunction::getBasicBlockContainingOffset(uint64_t Offset) { * BasicBlockOffsets.end(), * CompareBasicBlockOffsets()))); */ - auto I = std::upper_bound(BasicBlockOffsets.begin(), - BasicBlockOffsets.end(), + auto I = std::upper_bound(BasicBlockOffsets.begin(), BasicBlockOffsets.end(), BasicBlockOffset(Offset, nullptr), CompareBasicBlockOffsets()); assert(I != BasicBlockOffsets.begin() && "first basic block not at offset 0"); @@ -372,7 +382,7 @@ bool BinaryFunction::isForwardCall(const MCSymbol *CalleeSymbol) const { // should have been ordered with a stable sort. const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CalleeSymbol); if (CalleeBF) { - if(CalleeBF->isInjected()) + if (CalleeBF->isInjected()) return true; if (hasValidIndex() && CalleeBF->hasValidIndex()) { @@ -427,24 +437,18 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, << "\n IsSplit : " << isSplit() << "\n BB Count : " << size(); - if (HasFixedIndirectBranch) { + if (HasFixedIndirectBranch) OS << "\n HasFixedIndirectBranch : true"; - } - if (HasUnknownControlFlow) { + if (HasUnknownControlFlow) OS << "\n Unknown CF : true"; - } - if (getPersonalityFunction()) { + if (getPersonalityFunction()) OS << "\n Personality : " << getPersonalityFunction()->getName(); - } - if (IsFragment) { + if (IsFragment) OS << "\n IsFragment : true"; - } - if (isFolded()) { + if (isFolded()) OS << "\n FoldedInto : " << *getFoldedIntoFunction(); - } - for (BinaryFunction *ParentFragment : ParentFragments) { + for (BinaryFunction *ParentFragment : ParentFragments) OS << "\n Parent : " << *ParentFragment; - } if (!Fragments.empty()) { OS << "\n Fragments : "; const char *Sep = ""; @@ -453,9 +457,8 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, Sep = ", "; } } - if (hasCFG()) { - OS << "\n Hash : " << Twine::utohexstr(computeHash()); - } + if (hasCFG()) + OS << "\n Hash : " << Twine::utohexstr(computeHash()); if (isMultiEntry()) { OS << "\n Secondary Entry Points : "; const char *Sep = ""; @@ -464,9 +467,8 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, Sep = ", "; } } - if (FrameInstructions.size()) { - OS << "\n CFI Instrs : " << FrameInstructions.size(); - } + if (FrameInstructions.size()) + OS << "\n CFI Instrs : " << FrameInstructions.size(); if (BasicBlocksLayout.size()) { OS << "\n BB Layout : "; const char *Sep = ""; @@ -516,13 +518,11 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, for (uint32_t I = 0, E = BasicBlocksLayout.size(); I != E; ++I) { BinaryBasicBlock *BB = BasicBlocksLayout[I]; - if (I != 0 && - BB->isCold() != BasicBlocksLayout[I - 1]->isCold()) + if (I != 0 && BB->isCold() != BasicBlocksLayout[I - 1]->isCold()) OS << "------- HOT-COLD SPLIT POINT -------\n\n"; - OS << BB->getName() << " (" - << BB->size() << " instructions, align : " << BB->getAlignment() - << ")\n"; + OS << BB->getName() << " (" << BB->size() + << " instructions, align : " << BB->getAlignment() << ")\n"; if (isEntryPoint(*BB)) { if (MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB)) @@ -668,8 +668,7 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, OS << "End of Function \"" << *this << "\"\n\n"; } -void BinaryFunction::printRelocations(raw_ostream &OS, - uint64_t Offset, +void BinaryFunction::printRelocations(raw_ostream &OS, uint64_t Offset, uint64_t Size) const { const char *Sep = " # Relocs: "; @@ -774,8 +773,7 @@ const MCCFIInstruction *BinaryFunction::mutateCFIOffsetFor(const MCInst &Instr, } IndirectBranchType -BinaryFunction::processIndirectBranch(MCInst &Instruction, - unsigned Size, +BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size, uint64_t Offset, uint64_t &TargetAddress) { const unsigned PtrSize = BC.AsmInfo->getCodePointerSize(); @@ -813,17 +811,9 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, } } - IndirectBranchType BranchType = - BC.MIB->analyzeIndirectBranch(Instruction, - Begin, - Instructions.end(), - PtrSize, - MemLocInstr, - BaseRegNum, - IndexRegNum, - DispValue, - DispExpr, - PCRelBaseInstr); + IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch( + Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum, + IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); if (BranchType == IndirectBranchType::UNKNOWN && !MemLocInstr) return BranchType; @@ -1056,8 +1046,8 @@ bool BinaryFunction::disassemble() { // basic block. Labels[0] = Ctx->createNamedTempSymbol("BB0"); - auto handlePCRelOperand = - [&](MCInst &Instruction, uint64_t Address, uint64_t Size) { + auto handlePCRelOperand = [&](MCInst &Instruction, uint64_t Address, + uint64_t Size) { uint64_t TargetAddress = 0; if (!MIB->evaluateMemOperandTarget(Instruction, TargetAddress, Address, Size)) { @@ -1082,20 +1072,17 @@ bool BinaryFunction::disassemble() { const MCSymbol *TargetSymbol; uint64_t TargetOffset; std::tie(TargetSymbol, TargetOffset) = - BC.handleAddressRef(TargetAddress, *this, /*IsPCRel*/ true); - const MCExpr *Expr = MCSymbolRefExpr::create(TargetSymbol, - MCSymbolRefExpr::VK_None, - *BC.Ctx); + BC.handleAddressRef(TargetAddress, *this, /*IsPCRel*/ true); + const MCExpr *Expr = MCSymbolRefExpr::create( + TargetSymbol, MCSymbolRefExpr::VK_None, *BC.Ctx); if (TargetOffset) { const MCConstantExpr *Offset = MCConstantExpr::create(TargetOffset, *BC.Ctx); Expr = MCBinaryExpr::createAdd(Expr, Offset, *BC.Ctx); } - MIB->replaceMemOperandDisp( - Instruction, MCOperand::createExpr(BC.MIB->getTargetExprFor( - Instruction, - Expr, - *BC.Ctx, 0))); + MIB->replaceMemOperandDisp(Instruction, + MCOperand::createExpr(BC.MIB->getTargetExprFor( + Instruction, Expr, *BC.Ctx, 0))); }; // Used to fix the target of linker-generated AArch64 stubs with no relocation @@ -1217,7 +1204,7 @@ bool BinaryFunction::disassemble() { } }; - uint64_t Size = 0; // instruction size + uint64_t Size = 0; // instruction size for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) { MCInst Instruction; const uint64_t AbsoluteInstrAddr = getAddress() + Offset; @@ -1228,11 +1215,9 @@ bool BinaryFunction::disassemble() { continue; } - if (!BC.DisAsm->getInstruction(Instruction, - Size, + if (!BC.DisAsm->getInstruction(Instruction, Size, FunctionData.slice(Offset), - AbsoluteInstrAddr, - nulls())) { + AbsoluteInstrAddr, nulls())) { // Functions with "soft" boundaries, e.g. coming from assembly source, // can have 0-byte padding at the end. if (isZeroPaddingAt(Offset)) @@ -1240,8 +1225,8 @@ bool BinaryFunction::disassemble() { errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x" << Twine::utohexstr(Offset) << " (address 0x" - << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " - << *this << '\n'; + << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " << *this + << '\n'; // Some AVX-512 instructions could not be disassembled at all. if (BC.HasRelocations && opts::TrapOnAVX512 && BC.isX86()) { setTrapOnEntry(); @@ -1289,7 +1274,8 @@ bool BinaryFunction::disassemble() { // Check if there's a relocation associated with this instruction. bool UsedReloc = false; for (auto Itr = Relocations.lower_bound(Offset), - ItrE = Relocations.lower_bound(Offset + Size); Itr != ItrE; ++Itr) { + ItrE = Relocations.lower_bound(Offset + Size); + Itr != ItrE; ++Itr) { const Relocation &Relocation = Itr->second; LLVM_DEBUG(dbgs() << "BOLT-DEBUG: replacing immediate 0x" @@ -1303,8 +1289,7 @@ bool BinaryFunction::disassemble() { // Process reference to the primary symbol. if (!Relocation.isPCRelative()) - BC.handleAddressRef(Relocation.Value - Relocation.Addend, - *this, + BC.handleAddressRef(Relocation.Value - Relocation.Addend, *this, /*IsPCRel*/ false); int64_t Value = Relocation.Value; @@ -1325,9 +1310,9 @@ bool BinaryFunction::disassemble() { if (BC.isX86()) { assert(truncateToSize(static_cast(Value), Relocation::getSizeForType(Relocation.Type)) == - truncateToSize(Relocation.Value, - Relocation::getSizeForType(Relocation.Type)) && - "immediate value mismatch in function"); + truncateToSize(Relocation.Value, Relocation::getSizeForType( + Relocation.Type)) && + "immediate value mismatch in function"); } } @@ -1351,7 +1336,7 @@ bool BinaryFunction::disassemble() { if (BC.MIB->isUnsupportedBranch(Instruction.getOpcode())) { setIgnored(); if (BinaryFunction *TargetFunc = - BC.getBinaryFunctionContainingAddress(TargetAddress)) + BC.getBinaryFunctionContainingAddress(TargetAddress)) TargetFunc->setIgnored(); } @@ -1425,10 +1410,8 @@ bool BinaryFunction::disassemble() { add_instruction: if (getDWARFLineTable()) { - Instruction.setLoc( - findDebugLineInformationForInstructionAt(AbsoluteInstrAddr, - getDWARFUnit(), - getDWARFLineTable())); + Instruction.setLoc(findDebugLineInformationForInstructionAt( + AbsoluteInstrAddr, getDWARFUnit(), getDWARFLineTable())); } // Record offset of the instruction for profile matching. @@ -1478,7 +1461,7 @@ bool BinaryFunction::scanExternalRefs() { assert(FunctionData.size() == getMaxSize() && "function size does not match raw data size"); - uint64_t Size = 0; // instruction size + uint64_t Size = 0; // instruction size for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) { // Check for data inside code and ignore it if (const size_t DataInCodeSize = getSizeOfDataInCodeAt(Offset)) { @@ -1488,11 +1471,9 @@ bool BinaryFunction::scanExternalRefs() { const uint64_t AbsoluteInstrAddr = getAddress() + Offset; MCInst Instruction; - if (!BC.DisAsm->getInstruction(Instruction, - Size, + if (!BC.DisAsm->getInstruction(Instruction, Size, FunctionData.slice(Offset), - AbsoluteInstrAddr, - nulls())) { + AbsoluteInstrAddr, nulls())) { if (opts::Verbosity >= 1 && !isZeroPaddingAt(Offset)) { errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x" << Twine::utohexstr(Offset) << " (address 0x" @@ -1557,10 +1538,10 @@ bool BinaryFunction::scanExternalRefs() { BC.getBinaryFunctionContainingAddress(TargetAddress); if (TargetFunction && !ignoreFunctionRef(*TargetFunction)) { const uint64_t FunctionOffset = - TargetAddress - TargetFunction->getAddress(); + TargetAddress - TargetFunction->getAddress(); TargetSymbol = FunctionOffset - ? TargetFunction->addEntryPointAtOffset(FunctionOffset) - : TargetFunction->getSymbol(); + ? TargetFunction->addEntryPointAtOffset(FunctionOffset) + : TargetFunction->getSymbol(); } // Can't find more references and not creating relocations. @@ -1574,33 +1555,27 @@ bool BinaryFunction::scanExternalRefs() { BC.MIB->replaceBranchTarget(Instruction, TargetSymbol, Emitter.LocalCtx.get()); } else if (IsPCRel) { - const MCExpr *Expr = MCSymbolRefExpr::create(TargetSymbol, - MCSymbolRefExpr::VK_None, - *Emitter.LocalCtx.get()); + const MCExpr *Expr = MCSymbolRefExpr::create( + TargetSymbol, MCSymbolRefExpr::VK_None, *Emitter.LocalCtx.get()); BC.MIB->replaceMemOperandDisp( Instruction, MCOperand::createExpr(BC.MIB->getTargetExprFor( - Instruction, - Expr, - *Emitter.LocalCtx.get(), 0))); + Instruction, Expr, *Emitter.LocalCtx.get(), 0))); } } // Create more relocations based on input file relocations. bool HasRel = false; for (auto Itr = Relocations.lower_bound(Offset), - ItrE = Relocations.lower_bound(Offset + Size); Itr != ItrE; ++Itr) { + ItrE = Relocations.lower_bound(Offset + Size); + Itr != ItrE; ++Itr) { Relocation &Relocation = Itr->second; if (ignoreReference(Relocation.Symbol)) continue; int64_t Value = Relocation.Value; - const bool Result = - BC.MIB->replaceImmWithSymbolRef(Instruction, - Relocation.Symbol, - Relocation.Addend, - Emitter.LocalCtx.get(), - Value, - Relocation.Type); + const bool Result = BC.MIB->replaceImmWithSymbolRef( + Instruction, Relocation.Symbol, Relocation.Addend, + Emitter.LocalCtx.get(), Value, Relocation.Type); (void)Result; assert(Result && "cannot replace immediate with relocation"); @@ -1631,7 +1606,7 @@ bool BinaryFunction::scanExternalRefs() { Success = false; continue; } - Rel->Offset += getAddress() - getOriginSection()->getAddress() + Offset; + Rel->Offset += getAddress() - getOriginSection()->getAddress() + Offset; FunctionRelocations.push_back(*Rel); } @@ -1702,8 +1677,8 @@ void BinaryFunction::postProcessEntryPoints() { } errs() << "BOLT-WARNING: reference in the middle of instruction " - "detected in function " << *this - << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; + "detected in function " + << *this << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; if (BC.HasRelocations) { setIgnored(); } @@ -1719,7 +1694,8 @@ void BinaryFunction::postProcessJumpTables() { if (JT.Type == JumpTable::JTT_PIC && opts::JumpTables == JTS_BASIC) { opts::JumpTables = JTS_MOVE; outs() << "BOLT-INFO: forcing -jump-tables=move as PIC jump table was " - "detected in function " << *this << '\n'; + "detected in function " + << *this << '\n'; } for (unsigned I = 0; I < JT.OffsetEntries.size(); ++I) { MCSymbol *Label = @@ -1889,8 +1865,8 @@ bool BinaryFunction::postProcessIndirectBranches( if (opts::Verbosity >= 2) { outs() << "BOLT-INFO: rejected potential indirect tail call in " - << "function " << *this << " in basic block " - << BB->getName() << ".\n"; + << "function " << *this << " in basic block " << BB->getName() + << ".\n"; LLVM_DEBUG(BC.printInstructions(dbgs(), BB->begin(), BB->end(), BB->getOffset(), this, true)); } @@ -2003,14 +1979,14 @@ bool BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) { bool IsLastInstrNop = false; uint64_t LastInstrOffset = 0; - auto addCFIPlaceholders = - [this](uint64_t CFIOffset, BinaryBasicBlock *InsertBB) { - for (auto FI = OffsetToCFI.lower_bound(CFIOffset), - FE = OffsetToCFI.upper_bound(CFIOffset); - FI != FE; ++FI) { - addCFIPseudo(InsertBB, InsertBB->end(), FI->second); - } - }; + auto addCFIPlaceholders = [this](uint64_t CFIOffset, + BinaryBasicBlock *InsertBB) { + for (auto FI = OffsetToCFI.lower_bound(CFIOffset), + FE = OffsetToCFI.upper_bound(CFIOffset); + FI != FE; ++FI) { + addCFIPseudo(InsertBB, InsertBB->end(), FI->second); + } + }; // For profiling purposes we need to save the offset of the last instruction // in the basic block. But in certain cases we don't if the instruction was @@ -2314,11 +2290,10 @@ void BinaryFunction::removeConditionalTailCalls() { uint64_t CTCTakenCount = BinaryBasicBlock::COUNT_NO_PROFILE; uint64_t CTCMispredCount = BinaryBasicBlock::COUNT_NO_PROFILE; if (hasValidProfile()) { - CTCTakenCount = - BC.MIB->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); - CTCMispredCount = - BC.MIB->getAnnotationWithDefault(*CTCInstr, - "CTCMispredCount"); + CTCTakenCount = BC.MIB->getAnnotationWithDefault( + *CTCInstr, "CTCTakenCount"); + CTCMispredCount = BC.MIB->getAnnotationWithDefault( + *CTCInstr, "CTCMispredCount"); } // Assert that the tail call does not throw. @@ -2360,8 +2335,7 @@ void BinaryFunction::removeConditionalTailCalls() { BC.MIB->unsetConditionalTailCall(*CTCInstr); } - insertBasicBlocks(std::prev(end()), - std::move(NewBlocks), + insertBasicBlocks(std::prev(end()), std::move(NewBlocks), /* UpdateLayout */ true, /* UpdateCFIState */ false); } @@ -2676,8 +2650,7 @@ bool BinaryFunction::replayCFIInstrs(int32_t FromState, int32_t ToState, if (Instr->getOperation() == MCCFIInstruction::OpRestoreState) { auto Iter = FrameRestoreEquivalents.find(CurState); assert(Iter != FrameRestoreEquivalents.end()); - NewCFIs.insert(NewCFIs.end(), Iter->second.begin(), - Iter->second.end()); + NewCFIs.insert(NewCFIs.end(), Iter->second.begin(), Iter->second.end()); // RestoreState / Remember will be filtered out later by CFISnapshotDiff, // so we might as well fall-through here. } @@ -2702,7 +2675,7 @@ BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState, SmallVector NewStates; CFISnapshot ToCFITable(CIEFrameInstructions, FrameInstructions, - FrameRestoreEquivalents, ToState); + FrameRestoreEquivalents, ToState); CFISnapshotDiff FromCFITable(ToCFITable); FromCFITable.advanceTo(FromState); @@ -2889,11 +2862,10 @@ bool BinaryFunction::finalizeCFIState() { LLVM_DEBUG(dbgs() << "\n"); for (BinaryBasicBlock *BB : BasicBlocksLayout) { - for (auto II = BB->begin(); II != BB->end(); ) { + for (auto II = BB->begin(); II != BB->end();) { const MCCFIInstruction *CFI = getCFIFor(*II); - if (CFI && - (CFI->getOperation() == MCCFIInstruction::OpRememberState || - CFI->getOperation() == MCCFIInstruction::OpRestoreState)) { + if (CFI && (CFI->getOperation() == MCCFIInstruction::OpRememberState || + CFI->getOperation() == MCCFIInstruction::OpRestoreState)) { II = BB->eraseInstruction(II); } else { ++II; @@ -2916,9 +2888,7 @@ uint64_t BinaryFunction::getInstructionCount() const { return Count; } -bool BinaryFunction::hasLayoutChanged() const { - return ModifiedLayout; -} +bool BinaryFunction::hasLayoutChanged() const { return ModifiedLayout; } uint64_t BinaryFunction::getEditDistance() const { return ComputeEditDistance(BasicBlocksPreviousLayout, @@ -3052,8 +3022,7 @@ namespace { #define MAX_PATH 255 #endif -std::string constructFilename(std::string Filename, - std::string Annotation, +std::string constructFilename(std::string Filename, std::string Annotation, std::string Suffix) { std::replace(Filename.begin(), Filename.end(), '/', '-'); if (!Annotation.empty()) { @@ -3072,7 +3041,7 @@ std::string constructFilename(std::string Filename, return Filename; } -std::string formatEscapes(const std::string& Str) { +std::string formatEscapes(const std::string &Str) { std::string Result; for (unsigned I = 0; I < Str.size(); ++I) { char C = Str[I]; @@ -3090,36 +3059,29 @@ std::string formatEscapes(const std::string& Str) { return Result; } -} +} // namespace -void BinaryFunction::dumpGraph(raw_ostream& OS) const { +void BinaryFunction::dumpGraph(raw_ostream &OS) const { OS << "strict digraph \"" << getPrintName() << "\" {\n"; uint64_t Offset = Address; for (BinaryBasicBlock *BB : BasicBlocks) { - auto LayoutPos = std::find(BasicBlocksLayout.begin(), - BasicBlocksLayout.end(), - BB); + auto LayoutPos = + std::find(BasicBlocksLayout.begin(), BasicBlocksLayout.end(), BB); unsigned Layout = LayoutPos - BasicBlocksLayout.begin(); - const char* ColdStr = BB->isCold() ? " (cold)" : ""; + const char *ColdStr = BB->isCold() ? " (cold)" : ""; OS << format("\"%s\" [label=\"%s%s\\n(C:%lu,O:%lu,I:%u,L:%u:CFI:%u)\"]\n", - BB->getName().data(), - BB->getName().data(), - ColdStr, + BB->getName().data(), BB->getName().data(), ColdStr, (BB->ExecutionCount != BinaryBasicBlock::COUNT_NO_PROFILE - ? BB->ExecutionCount - : 0), - BB->getOffset(), - getIndex(BB), - Layout, - BB->getCFIState()); + ? BB->ExecutionCount + : 0), + BB->getOffset(), getIndex(BB), Layout, BB->getCFIState()); OS << format("\"%s\" [shape=box]\n", BB->getName().data()); if (opts::DotToolTipCode) { std::string Str; raw_string_ostream CS(Str); Offset = BC.printInstructions(CS, BB->begin(), BB->end(), Offset, this); const std::string Code = formatEscapes(CS.str()); - OS << format("\"%s\" [tooltip=\"%s\"]\n", - BB->getName().data(), + OS << format("\"%s\" [tooltip=\"%s\"]\n", BB->getName().data(), Code.c_str()); } @@ -3129,10 +3091,7 @@ void BinaryFunction::dumpGraph(raw_ostream& OS) const { const MCSymbol *FBB = nullptr; MCInst *CondBranch = nullptr; MCInst *UncondBranch = nullptr; - const bool Success = BB->analyzeBranch(TBB, - FBB, - CondBranch, - UncondBranch); + const bool Success = BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch); const MCInst *LastInstr = BB->getLastNonPseudoInstr(); const bool IsJumpTable = LastInstr && BC.MIB->getJumpTable(*LastInstr); @@ -3156,10 +3115,8 @@ void BinaryFunction::dumpGraph(raw_ostream& OS) const { if (IsJumpTable) { Branch = "JT"; } - OS << format("\"%s\" -> \"%s\" [label=\"%s", - BB->getName().data(), - Succ->getName().data(), - Branch.c_str()); + OS << format("\"%s\" -> \"%s\" [label=\"%s", BB->getName().data(), + Succ->getName().data(), Branch.c_str()); if (BB->getExecutionCount() != COUNT_NO_PROFILE && BI->MispredictedCount != BinaryBasicBlock::COUNT_INFERRED) { @@ -3174,8 +3131,7 @@ void BinaryFunction::dumpGraph(raw_ostream& OS) const { } for (BinaryBasicBlock *LP : BB->landing_pads()) { OS << format("\"%s\" -> \"%s\" [constraint=false style=dashed]\n", - BB->getName().data(), - LP->getName().data()); + BB->getName().data(), LP->getName().data()); } } OS << "}\n"; @@ -3268,30 +3224,28 @@ bool BinaryFunction::validateCFG() const { std::unordered_set BBThrowers; for (const BinaryBasicBlock *Thrower : BB->throwers()) { if (BBThrowers.count(Thrower)) { - errs() << "BOLT-ERROR: duplicate thrower detected in" - << BB->getName() << " in function " << *this << '\n'; + errs() << "BOLT-ERROR: duplicate thrower detected in" << BB->getName() + << " in function " << *this << '\n'; return false; } BBThrowers.insert(Thrower); } for (const BinaryBasicBlock *LPBlock : BB->landing_pads()) { - if (std::find(LPBlock->throw_begin(), LPBlock->throw_end(), BB) - == LPBlock->throw_end()) { - errs() << "BOLT-ERROR: inconsistent landing pad detected in " - << *this << ": " << BB->getName() - << " is in LandingPads but not in " << LPBlock->getName() - << " Throwers\n"; + if (std::find(LPBlock->throw_begin(), LPBlock->throw_end(), BB) == + LPBlock->throw_end()) { + errs() << "BOLT-ERROR: inconsistent landing pad detected in " << *this + << ": " << BB->getName() << " is in LandingPads but not in " + << LPBlock->getName() << " Throwers\n"; return false; } } for (const BinaryBasicBlock *Thrower : BB->throwers()) { - if (std::find(Thrower->lp_begin(), Thrower->lp_end(), BB) - == Thrower->lp_end()) { - errs() << "BOLT-ERROR: inconsistent thrower detected in " - << *this << ": " << BB->getName() - << " is in Throwers list but not in " << Thrower->getName() - << " LandingPads\n"; + if (std::find(Thrower->lp_begin(), Thrower->lp_end(), BB) == + Thrower->lp_end()) { + errs() << "BOLT-ERROR: inconsistent thrower detected in " << *this + << ": " << BB->getName() << " is in Throwers list but not in " + << Thrower->getName() << " LandingPads\n"; return false; } } @@ -3376,8 +3330,8 @@ void BinaryFunction::fixBranches() { // terminator) or more than 2 (switch table) don't require branch // instruction adjustments. } - assert((!isSimple() || validateCFG()) - && "Invalid CFG detected after fixing branches"); + assert((!isSimple() || validateCFG()) && + "Invalid CFG detected after fixing branches"); } void BinaryFunction::propagateGnuArgsSizeInfo( @@ -3393,7 +3347,7 @@ void BinaryFunction::propagateGnuArgsSizeInfo( // assigning the value. uint64_t CurrentGnuArgsSize = 0; for (BinaryBasicBlock *BB : BasicBlocks) { - for (auto II = BB->begin(); II != BB->end(); ) { + for (auto II = BB->begin(); II != BB->end();) { MCInst &Instr = *II; if (BC.MIB->isCFI(Instr)) { const MCCFIInstruction *CFI = getCFIFor(Instr); @@ -3470,9 +3424,8 @@ MCSymbol *BinaryFunction::addEntryPointAtOffset(uint64_t Offset) { if (BinaryData *EntryBD = BC.getBinaryDataAtAddress(EntryPointAddress)) { EntrySymbol = EntryBD->getSymbol(); } else { - EntrySymbol = - BC.getOrCreateGlobalSymbol(EntryPointAddress, - Twine("__ENTRY_") + getOneName() + "@"); + EntrySymbol = BC.getOrCreateGlobalSymbol( + EntryPointAddress, Twine("__ENTRY_") + getOneName() + "@"); } SecondaryEntryPoints[LocalSymbol] = EntrySymbol; @@ -3493,7 +3446,7 @@ MCSymbol *BinaryFunction::addEntryPoint(const BinaryBasicBlock &BB) { return EntrySymbol; EntrySymbol = - BC.Ctx->getOrCreateSymbol("__ENTRY_" + BB.getLabel()->getName()); + BC.Ctx->getOrCreateSymbol("__ENTRY_" + BB.getLabel()->getName()); SecondaryEntryPoints[BB.getLabel()] = EntrySymbol; @@ -3616,8 +3569,8 @@ BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const { const MCSymbol *FBB = nullptr; MCInst *CondBranch = nullptr; MCInst *UncondBranch = nullptr; - if (BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch) && - CondBranch && BB->succ_size() == 2) { + if (BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch) && CondBranch && + BB->succ_size() == 2) { if (BC.MIB->getCanonicalBranchCondCode(BC.MIB->getCondCode( *CondBranch)) == BC.MIB->getCondCode(*CondBranch)) { Stack.push(BB->getConditionalSuccessor(true)); @@ -3682,14 +3635,12 @@ size_t BinaryFunction::computeHash(bool UseDFS, void BinaryFunction::insertBasicBlocks( BinaryBasicBlock *Start, std::vector> &&NewBBs, - const bool UpdateLayout, - const bool UpdateCFIState, + const bool UpdateLayout, const bool UpdateCFIState, const bool RecomputeLandingPads) { const auto StartIndex = Start ? getIndex(Start) : -1; const size_t NumNewBlocks = NewBBs.size(); - BasicBlocks.insert(BasicBlocks.begin() + (StartIndex + 1), - NumNewBlocks, + BasicBlocks.insert(BasicBlocks.begin() + (StartIndex + 1), NumNewBlocks, nullptr); auto I = StartIndex + 1; @@ -3716,8 +3667,7 @@ void BinaryFunction::insertBasicBlocks( BinaryFunction::iterator BinaryFunction::insertBasicBlocks( BinaryFunction::iterator StartBB, std::vector> &&NewBBs, - const bool UpdateLayout, - const bool UpdateCFIState, + const bool UpdateLayout, const bool UpdateCFIState, const bool RecomputeLandingPads) { const unsigned StartIndex = getIndex(&*StartBB); const size_t NumNewBlocks = NewBBs.size(); @@ -3840,8 +3790,7 @@ void BinaryFunction::disambiguateJumpTables( if (!IndJmpMatcher->match( *BC.MRI, *BC.MIB, MutableArrayRef(&*BB->begin(), &Inst + 1), -1) || - BaseReg1 != BC.MIB->getNoRegister() || - Scale != 8) { + BaseReg1 != BC.MIB->getNoRegister() || Scale != 8) { MCPhysReg BaseReg2; uint64_t Offset; // Standard JT matching failed. Trying now: @@ -3979,7 +3928,7 @@ void BinaryFunction::deleteConservativeEdges() { continue; auto NextBB = std::next(I); - MCInst* Last = BB->getLastNonPseudoInstr(); + MCInst *Last = BB->getLastNonPseudoInstr(); // Fallthrough is a landing pad? Delete this edge (as long as we don't // have a direct jump to it) if ((*BB->succ_begin())->isLandingPad() && NextBB != E && @@ -3998,11 +3947,9 @@ void BinaryFunction::deleteConservativeEdges() { continue; StringRef CalleeName = CalleeSymbol->getName(); - if (CalleeName != "__cxa_throw@PLT" && - CalleeName != "_Unwind_Resume@PLT" && - CalleeName != "__cxa_rethrow@PLT" && - CalleeName != "exit@PLT" && - CalleeName != "abort@PLT" ) + if (CalleeName != "__cxa_throw@PLT" && CalleeName != "_Unwind_Resume@PLT" && + CalleeName != "__cxa_rethrow@PLT" && CalleeName != "exit@PLT" && + CalleeName != "abort@PLT") continue; BB->removeAllSuccessors(); @@ -4070,7 +4017,7 @@ void BinaryFunction::adjustExecutionCount(uint64_t Count) { if (ExecutionCount < Count) Count = ExecutionCount; - double AdjustmentRatio = ((double) ExecutionCount - Count) / ExecutionCount; + double AdjustmentRatio = ((double)ExecutionCount - Count) / ExecutionCount; if (AdjustmentRatio < 0.0) AdjustmentRatio = 0.0; @@ -4201,8 +4148,7 @@ void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) { } } else { setOutputAddress(getAddress()); - setOutputSize( - Layout.getSymbolOffset(*getFunctionEndLabel())); + setOutputSize(Layout.getSymbolOffset(*getFunctionEndLabel())); } // Update basic block output ranges for the debug info, if we have @@ -4238,8 +4184,7 @@ void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) { if (PrevBB) { uint64_t PrevBBEndAddress = BBAddress; if (BB->isCold() != PrevBB->isCold()) { - PrevBBEndAddress = - getOutputAddress() + getOutputSize(); + PrevBBEndAddress = getOutputAddress() + getOutputSize(); } PrevBB->setOutputEndAddress(PrevBBEndAddress); } @@ -4247,9 +4192,9 @@ void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) { BB->updateOutputValues(Layout); } - PrevBB->setOutputEndAddress(PrevBB->isCold() ? - cold().getAddress() + cold().getImageSize() : - getOutputAddress() + getOutputSize()); + PrevBB->setOutputEndAddress(PrevBB->isCold() + ? cold().getAddress() + cold().getImageSize() + : getOutputAddress() + getOutputSize()); } DebugAddressRangesVector BinaryFunction::getOutputAddressRanges() const { @@ -4326,8 +4271,7 @@ DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges( // If the function hasn't changed return the same ranges. if (!isEmitted()) { OutputRanges.resize(InputRanges.size()); - std::transform(InputRanges.begin(), InputRanges.end(), - OutputRanges.begin(), + std::transform(InputRanges.begin(), InputRanges.end(), OutputRanges.begin(), [](const DWARFAddressRange &Range) { return DebugAddressRange(Range.LowPC, Range.HighPC); }); @@ -4350,10 +4294,9 @@ DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges( const uint64_t InputEndOffset = std::min(Range.HighPC - getAddress(), getSize()); - auto BBI = std::upper_bound(BasicBlockOffsets.begin(), - BasicBlockOffsets.end(), - BasicBlockOffset(InputOffset, nullptr), - CompareBasicBlockOffsets()); + auto BBI = std::upper_bound( + BasicBlockOffsets.begin(), BasicBlockOffsets.end(), + BasicBlockOffset(InputOffset, nullptr), CompareBasicBlockOffsets()); --BBI; do { const BinaryBasicBlock *BB = BBI->second; @@ -4375,8 +4318,8 @@ DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges( EndAddress = StartAddress + InputEndOffset - InputOffset; if (StartAddress == PrevEndAddress) { - OutputRanges.back().HighPC = std::max(OutputRanges.back().HighPC, - EndAddress); + OutputRanges.back().HighPC = + std::max(OutputRanges.back().HighPC, EndAddress); } else { OutputRanges.emplace_back(StartAddress, std::max(StartAddress, EndAddress)); @@ -4395,8 +4338,8 @@ DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges( PrevEndAddress = 0; for (const DebugAddressRange &Range : OutputRanges) { if (Range.LowPC <= PrevEndAddress) { - MergedRanges.back().HighPC = std::max(MergedRanges.back().HighPC, - Range.HighPC); + MergedRanges.back().HighPC = + std::max(MergedRanges.back().HighPC, Range.HighPC); } else { MergedRanges.emplace_back(Range.LowPC, Range.HighPC); } @@ -4448,17 +4391,16 @@ DebugLocationsVector BinaryFunction::translateInputToOutputLocationList( const uint64_t End = Entry.HighPC; if (!containsAddress(Start)) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: invalid debug address range detected " - "for " + "for " << *this << " : [0x" << Twine::utohexstr(Start) << ", 0x" << Twine::utohexstr(End) << "]\n"); continue; } uint64_t InputOffset = Start - getAddress(); const uint64_t InputEndOffset = std::min(End - getAddress(), getSize()); - auto BBI = std::upper_bound(BasicBlockOffsets.begin(), - BasicBlockOffsets.end(), - BasicBlockOffset(InputOffset, nullptr), - CompareBasicBlockOffsets()); + auto BBI = std::upper_bound( + BasicBlockOffsets.begin(), BasicBlockOffsets.end(), + BasicBlockOffset(InputOffset, nullptr), CompareBasicBlockOffsets()); --BBI; do { const BinaryBasicBlock *BB = BBI->second; @@ -4482,10 +4424,8 @@ DebugLocationsVector BinaryFunction::translateInputToOutputLocationList( if (StartAddress == PrevEndAddress && Entry.Expr == *PrevExpr) { OutputLL.back().HighPC = std::max(OutputLL.back().HighPC, EndAddress); } else { - OutputLL.emplace_back( - DebugLocationEntry{StartAddress, - std::max(StartAddress, EndAddress), - Entry.Expr}); + OutputLL.emplace_back(DebugLocationEntry{ + StartAddress, std::max(StartAddress, EndAddress), Entry.Expr}); } PrevEndAddress = OutputLL.back().HighPC; PrevExpr = &OutputLL.back().Expr; @@ -4497,8 +4437,9 @@ DebugLocationsVector BinaryFunction::translateInputToOutputLocationList( } // Sort and merge adjacent entries with identical location. - std::stable_sort(OutputLL.begin(), OutputLL.end(), - [] (const DebugLocationEntry &A, const DebugLocationEntry &B) { + std::stable_sort( + OutputLL.begin(), OutputLL.end(), + [](const DebugLocationEntry &A, const DebugLocationEntry &B) { return A.LowPC < B.LowPC; }); DebugLocationsVector MergedLL; @@ -4541,8 +4482,8 @@ void BinaryFunction::printLoopInfo(raw_ostream &OS) const { if (!hasValidProfile()) continue; - OS << (L->getLoopDepth() > 1 ? "Nested" : "Outer") << " loop header: " - << L->getHeader()->getName(); + OS << (L->getLoopDepth() > 1 ? "Nested" : "Outer") + << " loop header: " << L->getHeader()->getName(); OS << "\n"; OS << "Loop basic blocks: "; const char *Sep = ""; @@ -4564,7 +4505,7 @@ void BinaryFunction::printLoopInfo(raw_ostream &OS) const { OS << "----\n"; } - OS << "Total number of loops: "<< BLI->TotalLoops << "\n"; + OS << "Total number of loops: " << BLI->TotalLoops << "\n"; OS << "Number of outer loops: " << BLI->OuterLoops << "\n"; OS << "Maximum nested loop depth: " << BLI->MaximumDepth << "\n\n"; } diff --git a/bolt/lib/Core/BinaryFunctionProfile.cpp b/bolt/lib/Core/BinaryFunctionProfile.cpp index 3d34f5e93650..441cf29c766e 100644 --- a/bolt/lib/Core/BinaryFunctionProfile.cpp +++ b/bolt/lib/Core/BinaryFunctionProfile.cpp @@ -199,8 +199,8 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const { // Update basic block count. if (BB->getExecutionCount() != BinaryBasicBlock::COUNT_NO_PROFILE) { - BBMerge->setExecutionCount( - BBMerge->getKnownExecutionCount() + BB->getExecutionCount()); + BBMerge->setExecutionCount(BBMerge->getKnownExecutionCount() + + BB->getExecutionCount()); } // Update edge count for successors of this basic block. @@ -280,8 +280,8 @@ void BinaryFunction::inferFallThroughCounts() { uint64_t CTCTakenCount = 0; const MCInst *CTCInstr = BB->getLastNonPseudoInstr(); if (CTCInstr && BC.MIB->getConditionalTailCall(*CTCInstr)) { - CTCTakenCount = - BC.MIB->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); + CTCTakenCount = BC.MIB->getAnnotationWithDefault( + *CTCInstr, "CTCTakenCount"); } // Calculate frequency of throws from this node according to LBR data @@ -294,7 +294,7 @@ void BinaryFunction::inferFallThroughCounts() { } const uint64_t TotalReportedJumps = - ReportedBranches + CTCTakenCount + ReportedThrows; + ReportedBranches + CTCTakenCount + ReportedThrows; // Infer the frequency of the fall-through edge, representing not taking the // branch. @@ -303,21 +303,18 @@ void BinaryFunction::inferFallThroughCounts() { Inferred = BBExecCount - TotalReportedJumps; LLVM_DEBUG( - if (BBExecCount < TotalReportedJumps) - dbgs() + if (BBExecCount < TotalReportedJumps) dbgs() << "Fall-through inference is slightly inconsistent. " "exec frequency is less than the outgoing edges frequency (" << BBExecCount << " < " << ReportedBranches << ") for BB at offset 0x" - << Twine::utohexstr(getAddress() + BB->getOffset()) << '\n'; - ); + << Twine::utohexstr(getAddress() + BB->getOffset()) << '\n';); if (BB->succ_size() <= 2) { // Skip if the last instruction is an unconditional jump. const MCInst *LastInstr = BB->getLastNonPseudoInstr(); - if (LastInstr && - (BC.MIB->isUnconditionalBranch(*LastInstr) || - BC.MIB->isIndirectBranch(*LastInstr))) + if (LastInstr && (BC.MIB->isUnconditionalBranch(*LastInstr) || + BC.MIB->isIndirectBranch(*LastInstr))) continue; // If there is an FT it will be the last successor. auto &SuccBI = *BB->branch_info_rbegin(); diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp index 6c7da86c4c3a..8c368d15f683 100644 --- a/bolt/lib/Core/BinarySection.cpp +++ b/bolt/lib/Core/BinarySection.cpp @@ -14,7 +14,6 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/Support/CommandLine.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" using namespace llvm; @@ -23,15 +22,11 @@ using namespace bolt; namespace opts { extern cl::opt PrintRelocations; extern cl::opt HotData; -} +} // namespace opts -bool BinarySection::isELF() const { - return BC.isELF(); -} +bool BinarySection::isELF() const { return BC.isELF(); } -bool BinarySection::isMachO() const { - return BC.isMachO(); -} +bool BinarySection::isMachO() const { return BC.isMachO(); } uint64_t BinarySection::hash(const BinaryData &BD, @@ -48,23 +43,21 @@ BinarySection::hash(const BinaryData &BD, auto End = Relocations.upper_bound(Relocation{EndOffset, 0, 0, 0, 0}); const StringRef Contents = getContents(); - hash_code Hash = hash_combine(hash_value(BD.getSize()), - hash_value(BD.getSectionName())); + hash_code Hash = + hash_combine(hash_value(BD.getSize()), hash_value(BD.getSectionName())); while (Begin != End) { const Relocation &Rel = *Begin++; Hash = hash_combine( - Hash, - hash_value(Contents.substr(Offset, Begin->Offset - Offset))); + Hash, hash_value(Contents.substr(Offset, Begin->Offset - Offset))); if (BinaryData *RelBD = BC.getBinaryDataByName(Rel.Symbol->getName())) { Hash = hash_combine(Hash, hash(*RelBD, Cache)); } Offset = Rel.Offset + Rel.getSize(); } - Hash = hash_combine( - Hash, - hash_value(Contents.substr(Offset, EndOffset - Offset))); + Hash = hash_combine(Hash, + hash_value(Contents.substr(Offset, EndOffset - Offset))); Cache[&BD] = Hash; @@ -97,9 +90,8 @@ void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) const { if (BC.UndefinedSymbols.count(Relocation.Symbol)) continue; if (SectionOffset < Relocation.Offset) { - Streamer.emitBytes( - SectionContents.substr(SectionOffset, - Relocation.Offset - SectionOffset)); + Streamer.emitBytes(SectionContents.substr( + SectionOffset, Relocation.Offset - SectionOffset)); SectionOffset = Relocation.Offset; } LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitting relocation for symbol " @@ -132,8 +124,8 @@ void BinarySection::flushPendingRelocations(raw_pwrite_stream &OS, // this means using their input file offsets, since the output file offset // could change (e.g. for new instance of .text). For non-allocatable // sections, the output offset should always be a valid one. - const uint64_t SectionFileOffset = isAllocatable() ? getInputFileOffset() - : getOutputFileOffset(); + const uint64_t SectionFileOffset = + isAllocatable() ? getInputFileOffset() : getOutputFileOffset(); LLVM_DEBUG( dbgs() << "BOLT-DEBUG: flushing pending relocations for section " << getName() << '\n' @@ -141,30 +133,28 @@ void BinarySection::flushPendingRelocations(raw_pwrite_stream &OS, << " offset: 0x" << Twine::utohexstr(SectionFileOffset) << '\n'); for (BinaryPatch &Patch : Patches) { - OS.pwrite(Patch.Bytes.data(), - Patch.Bytes.size(), + OS.pwrite(Patch.Bytes.data(), Patch.Bytes.size(), SectionFileOffset + Patch.Offset); } - for (Relocation &Reloc : PendingRelocations) { uint64_t Value = Reloc.Addend; if (Reloc.Symbol) Value += Resolver(Reloc.Symbol); - switch(Reloc.Type) { + switch (Reloc.Type) { default: LLVM_DEBUG(dbgs() << Reloc.Type << '\n';); llvm_unreachable("unhandled relocation type"); case ELF::R_X86_64_64: case ELF::R_X86_64_32: { - OS.pwrite(reinterpret_cast(&Value), + OS.pwrite(reinterpret_cast(&Value), Relocation::getSizeForType(Reloc.Type), SectionFileOffset + Reloc.Offset); break; } case ELF::R_X86_64_PC32: { Value -= SectionAddress + Reloc.Offset; - OS.pwrite(reinterpret_cast(&Value), + OS.pwrite(reinterpret_cast(&Value), Relocation::getSizeForType(Reloc.Type), SectionFileOffset + Reloc.Offset); LLVM_DEBUG(dbgs() << "BOLT-DEBUG: writing value 0x" @@ -205,18 +195,13 @@ BinarySection::~BinarySection() { } } -void BinarySection::clearRelocations() { - clearList(Relocations); -} +void BinarySection::clearRelocations() { clearList(Relocations); } void BinarySection::print(raw_ostream &OS) const { OS << getName() << ", " - << "0x" << Twine::utohexstr(getAddress()) << ", " - << getSize() - << " (0x" << Twine::utohexstr(getOutputAddress()) << ", " - << getOutputSize() << ")" - << ", data = " << getData() - << ", output data = " << getOutputData(); + << "0x" << Twine::utohexstr(getAddress()) << ", " << getSize() << " (0x" + << Twine::utohexstr(getOutputAddress()) << ", " << getOutputSize() << ")" + << ", data = " << getData() << ", output data = " << getOutputData(); if (isAllocatable()) OS << " (allocatable)"; diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp index d5721997f08e..844e5c94979e 100644 --- a/bolt/lib/Core/DebugData.cpp +++ b/bolt/lib/Core/DebugData.cpp @@ -22,7 +22,6 @@ #include #include -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt-debug-info" namespace opts { @@ -41,10 +40,9 @@ namespace { // the form (begin address, range size), otherwise (begin address, end address). // Terminates the list by writing a pair of two zeroes. // Returns the number of written bytes. -uint64_t writeAddressRanges( - raw_svector_ostream &Stream, - const DebugAddressRangesVector &AddressRanges, - const bool WriteRelativeRanges = false) { +uint64_t writeAddressRanges(raw_svector_ostream &Stream, + const DebugAddressRangesVector &AddressRanges, + const bool WriteRelativeRanges = false) { for (const DebugAddressRange &Range : AddressRanges) { support::endian::write(Stream, Range.LowPC, support::little); support::endian::write( @@ -125,7 +123,7 @@ void DebugARangesSectionWriter::writeARangesSection( // + 2*sizeof(uint64_t) bytes for each of the ranges, plus an extra // pair of uint64_t's for the terminating, zero-length range. // Does not include size field itself. - uint32_t Size = 8 + 4 + 2*sizeof(uint64_t) * (AddressRanges.size() + 1); + uint32_t Size = 8 + 4 + 2 * sizeof(uint64_t) * (AddressRanges.size() + 1); // Header field #1: set size. support::endian::write(RangesStream, Size, support::little); @@ -367,7 +365,8 @@ void SimpleBinaryPatcher::addLEPatch(uint32_t Offset, uint64_t NewValue, Patches.emplace_back(Offset, LE64); } -void SimpleBinaryPatcher::addUDataPatch(uint32_t Offset, uint64_t Value, uint64_t Size) { +void SimpleBinaryPatcher::addUDataPatch(uint32_t Offset, uint64_t Value, + uint64_t Size) { std::string Buff; raw_string_ostream OS(Buff); encodeULEB128(Value, OS, Size); @@ -389,7 +388,7 @@ void SimpleBinaryPatcher::patchBinary(std::string &BinaryContents, uint32_t Offset = Patch.first - DWPOffset; const std::string &ByteSequence = Patch.second; assert(Offset + ByteSequence.size() <= BinaryContents.size() && - "Applied patch runs over binary size."); + "Applied patch runs over binary size."); for (uint64_t I = 0, Size = ByteSequence.size(); I < Size; ++I) { BinaryContents[Offset + I] = ByteSequence[I]; } diff --git a/bolt/lib/Core/DynoStats.cpp b/bolt/lib/Core/DynoStats.cpp index e566cefd7262..e1350b7bb3fc 100644 --- a/bolt/lib/Core/DynoStats.cpp +++ b/bolt/lib/Core/DynoStats.cpp @@ -53,27 +53,22 @@ constexpr const char *DynoStats::Desc[]; bool DynoStats::operator<(const DynoStats &Other) const { return std::lexicographical_compare( - &Stats[FIRST_DYNO_STAT], &Stats[LAST_DYNO_STAT], - &Other.Stats[FIRST_DYNO_STAT], &Other.Stats[LAST_DYNO_STAT] - ); + &Stats[FIRST_DYNO_STAT], &Stats[LAST_DYNO_STAT], + &Other.Stats[FIRST_DYNO_STAT], &Other.Stats[LAST_DYNO_STAT]); } bool DynoStats::operator==(const DynoStats &Other) const { - return std::equal( - &Stats[FIRST_DYNO_STAT], &Stats[LAST_DYNO_STAT], - &Other.Stats[FIRST_DYNO_STAT] - ); + return std::equal(&Stats[FIRST_DYNO_STAT], &Stats[LAST_DYNO_STAT], + &Other.Stats[FIRST_DYNO_STAT]); } bool DynoStats::lessThan(const DynoStats &Other, ArrayRef Keys) const { return std::lexicographical_compare( - Keys.begin(), Keys.end(), - Keys.begin(), Keys.end(), - [this,&Other](const Category A, const Category) { - return Stats[A] < Other.Stats[A]; - } - ); + Keys.begin(), Keys.end(), Keys.begin(), Keys.end(), + [this, &Other](const Category A, const Category) { + return Stats[A] < Other.Stats[A]; + }); } void DynoStats::print(raw_ostream &OS, const DynoStats *Other, @@ -83,10 +78,9 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other, OS << format("%'20lld : ", Stat * opts::DynoStatsScale) << Name; if (Other) { if (Stat != OtherStat) { - OtherStat = std::max(OtherStat, uint64_t(1)); // to prevent divide by 0 - OS << format(" (%+.1f%%)", - ( (float) Stat - (float) OtherStat ) * 100.0 / - (float) (OtherStat) ); + OtherStat = std::max(OtherStat, uint64_t(1)); // to prevent divide by 0 + OS << format(" (%+.1f%%)", ((float)Stat - (float)OtherStat) * 100.0 / + (float)(OtherStat)); } else { OS << " (=)"; } @@ -95,8 +89,7 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other, }; for (auto Stat = DynoStats::FIRST_DYNO_STAT + 1; - Stat < DynoStats::LAST_DYNO_STAT; - ++Stat) { + Stat < DynoStats::LAST_DYNO_STAT; ++Stat) { if (!PrintAArch64Stats && Stat == DynoStats::VENEER_CALLS_AARCH64) continue; @@ -118,8 +111,7 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other, // count. for (auto Stat = SortedHistogram.rbegin(); Stat != SortedHistogram.rend(); ++Stat) { - OS << format("%20s,%'18lld", - Printer->getOpcodeName(Stat->second).data(), + OS << format("%20s,%'18lld", Printer->getOpcodeName(Stat->second).data(), Stat->first * opts::DynoStatsScale); MaxOpcodeHistogramTy MaxMultiMap = @@ -136,8 +128,7 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other, void DynoStats::operator+=(const DynoStats &Other) { for (auto Stat = DynoStats::FIRST_DYNO_STAT + 1; - Stat < DynoStats::LAST_DYNO_STAT; - ++Stat) { + Stat < DynoStats::LAST_DYNO_STAT; ++Stat) { Stats[Stat] += Other[Stat]; } for (const OpcodeStatTy &Stat : Other.OpcodeHistogram) { @@ -183,15 +174,15 @@ DynoStats getDynoStats(const BinaryFunction &BF) { // frequencies. This may deviate from the sum of outgoing branches of the // basic block especially since the block may contain a function that // does not return or a function that throws an exception. - const uint64_t BBExecutionCount = BB->getKnownExecutionCount(); + const uint64_t BBExecutionCount = BB->getKnownExecutionCount(); // Ignore empty blocks and blocks that were not executed. if (BB->getNumNonPseudos() == 0 || BBExecutionCount == 0) continue; // Count AArch64 linker-inserted veneers - if(BF.isAArch64Veneer()) - Stats[DynoStats::VENEER_CALLS_AARCH64] += BF.getKnownExecutionCount(); + if (BF.isAArch64Veneer()) + Stats[DynoStats::VENEER_CALLS_AARCH64] += BF.getKnownExecutionCount(); // Count various instruction types by iterating through all instructions. // When -print-dyno-opcode-stats is on, count per each opcode and record @@ -205,7 +196,7 @@ DynoStats getDynoStats(const BinaryFunction &BF) { MMap.emplace(BBExecutionCount, std::make_pair(BF.getOneName(), BB->getOffset())); Stats.OpcodeHistogram.emplace(Opcode, - std::make_pair(BBExecutionCount, MMap)); + std::make_pair(BBExecutionCount, MMap)); } else { I->second.first += BBExecutionCount; bool Insert = true; @@ -236,7 +227,7 @@ DynoStats getDynoStats(const BinaryFunction &BF) { uint64_t CallFreq = BBExecutionCount; if (BC.MIB->getConditionalTailCall(Instr)) { CallFreq = - BC.MIB->getAnnotationWithDefault(Instr, "CTCTakenCount"); + BC.MIB->getAnnotationWithDefault(Instr, "CTCTakenCount"); } Stats[DynoStats::FUNCTION_CALLS] += CallFreq; if (BC.MIB->isIndirectCall(Instr)) { diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp index a4c2cf156938..175a3f3928fa 100644 --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -130,8 +130,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, if (opts::PrintExceptions) { outs() << "[LSDA at 0x" << Twine::utohexstr(getLSDAAddress()) << " for function " << *this << "]:\n"; - outs() << "LPStart Encoding = 0x" - << Twine::utohexstr(LPStartEncoding) << '\n'; + outs() << "LPStart Encoding = 0x" << Twine::utohexstr(LPStartEncoding) + << '\n'; outs() << "LPStart = 0x" << Twine::utohexstr(LPStart) << '\n'; outs() << "TType Encoding = 0x" << Twine::utohexstr(TTypeEncoding) << '\n'; outs() << "TType End = " << TTypeEnd << '\n'; @@ -168,8 +168,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, while (CallSitePtr < CallSiteTableEnd) { uint64_t Start = *Data.getEncodedPointer(&CallSitePtr, CallSiteEncoding, CallSitePtr + LSDASectionAddress); - uint64_t Length = *Data.getEncodedPointer( - &CallSitePtr, CallSiteEncoding, CallSitePtr + LSDASectionAddress); + uint64_t Length = *Data.getEncodedPointer(&CallSitePtr, CallSiteEncoding, + CallSitePtr + LSDASectionAddress); uint64_t LandingPad = *Data.getEncodedPointer( &CallSitePtr, CallSiteEncoding, CallSitePtr + LSDASectionAddress); uint64_t ActionEntry = Data.getULEB128(&CallSitePtr); @@ -262,8 +262,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, outs() << "cleanup"; } else if (ActionType > 0) { // It's an index into a type table. - MaxTypeIndex = std::max(MaxTypeIndex, - static_cast(ActionType)); + MaxTypeIndex = + std::max(MaxTypeIndex, static_cast(ActionType)); if (opts::PrintExceptions) { outs() << "catch type "; printType(ActionType, outs()); @@ -285,9 +285,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, TSep = ", "; } } - MaxTypeIndexTableOffset = - std::max(MaxTypeIndexTableOffset, - TypeIndexTablePtr - TypeIndexTableStart); + MaxTypeIndexTableOffset = std::max( + MaxTypeIndexTableOffset, TypeIndexTablePtr - TypeIndexTableStart); } Sep = "; "; @@ -394,9 +393,7 @@ void BinaryFunction::updateEHRanges() { std::tie(LP, Action) = *EHInfo; // No action if the exception handler has not changed. - if (Throws && - StartRange && - PreviousEH.LP == LP && + if (Throws && StartRange && PreviousEH.LP == LP && PreviousEH.Action == Action) continue; @@ -432,8 +429,8 @@ void BinaryFunction::updateEHRanges() { // Close the previous range. if (EndRange) { - Sites->emplace_back(CallSite{StartRange, EndRange, - PreviousEH.LP, PreviousEH.Action}); + Sites->emplace_back( + CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action}); } if (Throws) { @@ -452,8 +449,8 @@ void BinaryFunction::updateEHRanges() { assert((!isSplit() || Sites == &ColdCallSites) && "sites mismatch"); const MCSymbol *EndRange = IsStartInCold ? getFunctionColdEndLabel() : getFunctionEndLabel(); - Sites->emplace_back(CallSite{StartRange, EndRange, - PreviousEH.LP, PreviousEH.Action}); + Sites->emplace_back( + CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action}); } } @@ -476,9 +473,8 @@ CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) { } else if (opts::Verbosity > 0) { errs() << "BOLT-WARNING: different FDEs for function at 0x" << Twine::utohexstr(FDEI->first) - << " detected; sizes: " - << FDEI->second->getAddressRange() << " and " - << CurFDE->getAddressRange() << '\n'; + << " detected; sizes: " << FDEI->second->getAddressRange() + << " and " << CurFDE->getAddressRange() << '\n'; } } } else { @@ -508,145 +504,142 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const { *CurFDE.getLinkedCIE()->getPersonalityEncoding()); } - auto decodeFrameInstruction = - [&Function, &Offset, Address, CodeAlignment, DataAlignment]( - const CFIProgram::Instruction &Instr) { - uint8_t Opcode = Instr.Opcode; - if (Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) - Opcode &= DWARF_CFI_PRIMARY_OPCODE_MASK; - switch (Instr.Opcode) { - case DW_CFA_nop: - break; - case DW_CFA_advance_loc4: - case DW_CFA_advance_loc2: - case DW_CFA_advance_loc1: - case DW_CFA_advance_loc: - // Advance our current address - Offset += CodeAlignment * int64_t(Instr.Ops[0]); - break; - case DW_CFA_offset_extended_sf: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createOffset( - nullptr, Instr.Ops[0], - DataAlignment * int64_t(Instr.Ops[1]))); - break; - case DW_CFA_offset_extended: - case DW_CFA_offset: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createOffset( - nullptr, Instr.Ops[0], DataAlignment * Instr.Ops[1])); - break; - case DW_CFA_restore_extended: - case DW_CFA_restore: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createRestore(nullptr, Instr.Ops[0])); - break; - case DW_CFA_set_loc: - assert(Instr.Ops[0] >= Address && "set_loc out of function bounds"); - assert(Instr.Ops[0] <= Address + Function.getSize() && - "set_loc out of function bounds"); - Offset = Instr.Ops[0] - Address; - break; + auto decodeFrameInstruction = [&Function, &Offset, Address, CodeAlignment, + DataAlignment]( + const CFIProgram::Instruction &Instr) { + uint8_t Opcode = Instr.Opcode; + if (Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) + Opcode &= DWARF_CFI_PRIMARY_OPCODE_MASK; + switch (Instr.Opcode) { + case DW_CFA_nop: + break; + case DW_CFA_advance_loc4: + case DW_CFA_advance_loc2: + case DW_CFA_advance_loc1: + case DW_CFA_advance_loc: + // Advance our current address + Offset += CodeAlignment * int64_t(Instr.Ops[0]); + break; + case DW_CFA_offset_extended_sf: + Function.addCFIInstruction( + Offset, + MCCFIInstruction::createOffset( + nullptr, Instr.Ops[0], DataAlignment * int64_t(Instr.Ops[1]))); + break; + case DW_CFA_offset_extended: + case DW_CFA_offset: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createOffset(nullptr, Instr.Ops[0], + DataAlignment * Instr.Ops[1])); + break; + case DW_CFA_restore_extended: + case DW_CFA_restore: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createRestore(nullptr, Instr.Ops[0])); + break; + case DW_CFA_set_loc: + assert(Instr.Ops[0] >= Address && "set_loc out of function bounds"); + assert(Instr.Ops[0] <= Address + Function.getSize() && + "set_loc out of function bounds"); + Offset = Instr.Ops[0] - Address; + break; - case DW_CFA_undefined: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createUndefined(nullptr, Instr.Ops[0])); - break; - case DW_CFA_same_value: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createSameValue(nullptr, Instr.Ops[0])); - break; - case DW_CFA_register: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createRegister(nullptr, Instr.Ops[0], - Instr.Ops[1])); - break; - case DW_CFA_remember_state: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createRememberState(nullptr)); - break; - case DW_CFA_restore_state: - Function.addCFIInstruction( - Offset, MCCFIInstruction::createRestoreState(nullptr)); - break; - case DW_CFA_def_cfa: - Function.addCFIInstruction( - Offset, MCCFIInstruction::cfiDefCfa(nullptr, Instr.Ops[0], - Instr.Ops[1])); - break; - case DW_CFA_def_cfa_sf: - Function.addCFIInstruction( - Offset, MCCFIInstruction::cfiDefCfa( - nullptr, Instr.Ops[0], - DataAlignment * int64_t(Instr.Ops[1]))); - break; - case DW_CFA_def_cfa_register: - Function.addCFIInstruction( - Offset, - MCCFIInstruction::createDefCfaRegister(nullptr, Instr.Ops[0])); - break; - case DW_CFA_def_cfa_offset: - Function.addCFIInstruction( - Offset, - MCCFIInstruction::cfiDefCfaOffset(nullptr, Instr.Ops[0])); - break; - case DW_CFA_def_cfa_offset_sf: - Function.addCFIInstruction( - Offset, MCCFIInstruction::cfiDefCfaOffset( - nullptr, DataAlignment * int64_t(Instr.Ops[0]))); - break; - case DW_CFA_GNU_args_size: - Function.addCFIInstruction( - Offset, - MCCFIInstruction::createGnuArgsSize(nullptr, Instr.Ops[0])); - Function.setUsesGnuArgsSize(); - break; - case DW_CFA_val_offset_sf: - case DW_CFA_val_offset: - if (opts::Verbosity >= 1) { - errs() << "BOLT-WARNING: DWARF val_offset() unimplemented\n"; - } - return false; - case DW_CFA_def_cfa_expression: - case DW_CFA_val_expression: - case DW_CFA_expression: { - StringRef ExprBytes = Instr.Expression->getData(); - std::string Str; - raw_string_ostream OS(Str); - // Manually encode this instruction using CFI escape - OS << Opcode; - if (Opcode != DW_CFA_def_cfa_expression) { - encodeULEB128(Instr.Ops[0], OS); - } - encodeULEB128(ExprBytes.size(), OS); - OS << ExprBytes; - Function.addCFIInstruction( - Offset, MCCFIInstruction::createEscape(nullptr, OS.str())); - break; - } - case DW_CFA_MIPS_advance_loc8: - if (opts::Verbosity >= 1) { - errs() << "BOLT-WARNING: DW_CFA_MIPS_advance_loc unimplemented\n"; - } - return false; - case DW_CFA_GNU_window_save: - case DW_CFA_lo_user: - case DW_CFA_hi_user: - if (opts::Verbosity >= 1) { - errs() << "BOLT-WARNING: DW_CFA_GNU_* and DW_CFA_*_user " - "unimplemented\n"; - } - return false; - default: - if (opts::Verbosity >= 1) { - errs() << "BOLT-WARNING: Unrecognized CFI instruction: " - << Instr.Opcode << '\n'; - } - return false; - } + case DW_CFA_undefined: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createUndefined(nullptr, Instr.Ops[0])); + break; + case DW_CFA_same_value: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createSameValue(nullptr, Instr.Ops[0])); + break; + case DW_CFA_register: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createRegister(nullptr, Instr.Ops[0], + Instr.Ops[1])); + break; + case DW_CFA_remember_state: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createRememberState(nullptr)); + break; + case DW_CFA_restore_state: + Function.addCFIInstruction(Offset, + MCCFIInstruction::createRestoreState(nullptr)); + break; + case DW_CFA_def_cfa: + Function.addCFIInstruction( + Offset, + MCCFIInstruction::cfiDefCfa(nullptr, Instr.Ops[0], Instr.Ops[1])); + break; + case DW_CFA_def_cfa_sf: + Function.addCFIInstruction( + Offset, + MCCFIInstruction::cfiDefCfa(nullptr, Instr.Ops[0], + DataAlignment * int64_t(Instr.Ops[1]))); + break; + case DW_CFA_def_cfa_register: + Function.addCFIInstruction(Offset, MCCFIInstruction::createDefCfaRegister( + nullptr, Instr.Ops[0])); + break; + case DW_CFA_def_cfa_offset: + Function.addCFIInstruction( + Offset, MCCFIInstruction::cfiDefCfaOffset(nullptr, Instr.Ops[0])); + break; + case DW_CFA_def_cfa_offset_sf: + Function.addCFIInstruction( + Offset, MCCFIInstruction::cfiDefCfaOffset( + nullptr, DataAlignment * int64_t(Instr.Ops[0]))); + break; + case DW_CFA_GNU_args_size: + Function.addCFIInstruction( + Offset, MCCFIInstruction::createGnuArgsSize(nullptr, Instr.Ops[0])); + Function.setUsesGnuArgsSize(); + break; + case DW_CFA_val_offset_sf: + case DW_CFA_val_offset: + if (opts::Verbosity >= 1) { + errs() << "BOLT-WARNING: DWARF val_offset() unimplemented\n"; + } + return false; + case DW_CFA_def_cfa_expression: + case DW_CFA_val_expression: + case DW_CFA_expression: { + StringRef ExprBytes = Instr.Expression->getData(); + std::string Str; + raw_string_ostream OS(Str); + // Manually encode this instruction using CFI escape + OS << Opcode; + if (Opcode != DW_CFA_def_cfa_expression) { + encodeULEB128(Instr.Ops[0], OS); + } + encodeULEB128(ExprBytes.size(), OS); + OS << ExprBytes; + Function.addCFIInstruction( + Offset, MCCFIInstruction::createEscape(nullptr, OS.str())); + break; + } + case DW_CFA_MIPS_advance_loc8: + if (opts::Verbosity >= 1) { + errs() << "BOLT-WARNING: DW_CFA_MIPS_advance_loc unimplemented\n"; + } + return false; + case DW_CFA_GNU_window_save: + case DW_CFA_lo_user: + case DW_CFA_hi_user: + if (opts::Verbosity >= 1) { + errs() << "BOLT-WARNING: DW_CFA_GNU_* and DW_CFA_*_user " + "unimplemented\n"; + } + return false; + default: + if (opts::Verbosity >= 1) { + errs() << "BOLT-WARNING: Unrecognized CFI instruction: " << Instr.Opcode + << '\n'; + } + return false; + } - return true; - }; + return true; + }; for (const CFIProgram::Instruction &Instr : CurFDE.getLinkedCIE()->cfis()) { if (!decodeFrameInstruction(Instr)) @@ -662,8 +655,7 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const { } std::vector CFIReaderWriter::generateEHFrameHeader( - const DWARFDebugFrame &OldEHFrame, - const DWARFDebugFrame &NewEHFrame, + const DWARFDebugFrame &OldEHFrame, const DWARFDebugFrame &NewEHFrame, uint64_t EHFrameHeaderAddress, std::vector &FailedAddresses) const { // Common PC -> FDE map to be written into .eh_frame_hdr. @@ -740,7 +732,7 @@ std::vector CFIReaderWriter::generateEHFrameHeader( // Address of eh_frame. Use the new one. support::ulittle32_t::ref(EHFrameHeader.data() + 4) = - NewEHFrame.getEHFrameAddress() - (EHFrameHeaderAddress + 4); + NewEHFrame.getEHFrameAddress() - (EHFrameHeaderAddress + 4); // Number of entries in the table (FDE count). support::ulittle32_t::ref(EHFrameHeader.data() + 8) = PCToFDE.size(); @@ -828,7 +820,7 @@ Error EHFrameParser::parseCIE(uint64_t StartOffset) { } Error EHFrameParser::parseFDE(uint64_t CIEPointer, - uint64_t StartStructureOffset) { + uint64_t StartStructureOffset) { Optional LSDAAddress; CIEInfo *Cie = CIEs[StartStructureOffset - CIEPointer]; @@ -895,7 +887,7 @@ Error EHFrameParser::parse() { } Error EHFrameParser::parse(DWARFDataExtractor Data, uint64_t EHFrameAddress, - PatcherCallbackTy PatcherCallback) { + PatcherCallbackTy PatcherCallback) { EHFrameParser Parser(Data, EHFrameAddress, PatcherCallback); return Parser.parse(); } diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp index 67ac3ad097f0..18c3869b09d0 100644 --- a/bolt/lib/Core/JumpTable.cpp +++ b/bolt/lib/Core/JumpTable.cpp @@ -14,7 +14,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" using namespace llvm; @@ -25,7 +24,7 @@ using JumpTable = bolt::JumpTable; namespace opts { extern cl::opt JumpTables; extern cl::opt Verbosity; -} +} // namespace opts bolt::JumpTable::JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize, JumpTableType Type, LabelMapType &&Labels, @@ -114,12 +113,12 @@ void bolt::JumpTable::print(raw_ostream &OS) const { if (Offset && LI != Labels.end()) { OS << "Jump Table " << LI->second->getName() << " at 0x" << Twine::utohexstr(getAddress() + Offset) - << " (possibly part of larger jump table):\n"; + << " (possibly part of larger jump table):\n"; } OS << format(" 0x%04" PRIx64 " : ", Offset) << Entry->getName(); if (!Counts.empty()) { - OS << " : " << Counts[Offset / EntrySize].Mispreds - << "/" << Counts[Offset / EntrySize].Count; + OS << " : " << Counts[Offset / EntrySize].Mispreds << "/" + << Counts[Offset / EntrySize].Count; } OS << '\n'; Offset += EntrySize; diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index 452c966c8693..3c3150ae4424 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -117,7 +117,7 @@ bool MCPlusBuilder::equals(const MCExpr &A, const MCExpr &B, bool MCPlusBuilder::equals(const MCTargetExpr &A, const MCTargetExpr &B, CompFuncTy Comp) const { - llvm_unreachable("target-specific expressions are unsupported"); + llvm_unreachable("target-specific expressions are unsupported"); } void MCPlusBuilder::setTailCall(MCInst &Inst) { @@ -214,8 +214,7 @@ MCPlusBuilder::getConditionalTailCall(const MCInst &Inst) const { return static_cast(*Value); } -bool -MCPlusBuilder::setConditionalTailCall(MCInst &Inst, uint64_t Dest) { +bool MCPlusBuilder::setConditionalTailCall(MCInst &Inst, uint64_t Dest) { if (!isConditionalBranch(Inst)) return false; @@ -265,8 +264,8 @@ void MCPlusBuilder::stripAnnotations(MCInst &Inst, bool KeepTC) { setTailCall(Inst); } -void -MCPlusBuilder::printAnnotations(const MCInst &Inst, raw_ostream &OS) const { +void MCPlusBuilder::printAnnotations(const MCInst &Inst, + raw_ostream &OS) const { const MCInst *AnnotationInst = getAnnotationInst(Inst); if (!AnnotationInst) return; @@ -275,11 +274,9 @@ MCPlusBuilder::printAnnotations(const MCInst &Inst, raw_ostream &OS) const { const int64_t Imm = AnnotationInst->getOperand(I).getImm(); const unsigned Index = extractAnnotationIndex(Imm); const int64_t Value = extractAnnotationValue(Imm); - const auto *Annotation = - reinterpret_cast(Value); + const auto *Annotation = reinterpret_cast(Value); if (Index >= MCAnnotation::kGeneric) { - OS << " # " << AnnotationNames[Index - MCAnnotation::kGeneric] - << ": "; + OS << " # " << AnnotationNames[Index - MCAnnotation::kGeneric] << ": "; Annotation->print(OS); } } @@ -309,8 +306,7 @@ void MCPlusBuilder::getClobberedRegs(const MCInst &Inst, } } -void MCPlusBuilder::getTouchedRegs(const MCInst &Inst, - BitVector &Regs) const { +void MCPlusBuilder::getTouchedRegs(const MCInst &Inst, BitVector &Regs) const { if (isPrefix(Inst) || isCFI(Inst)) return; @@ -332,8 +328,7 @@ void MCPlusBuilder::getTouchedRegs(const MCInst &Inst, } } -void MCPlusBuilder::getWrittenRegs(const MCInst &Inst, - BitVector &Regs) const { +void MCPlusBuilder::getWrittenRegs(const MCInst &Inst, BitVector &Regs) const { if (isPrefix(Inst) || isCFI(Inst)) return; @@ -423,9 +418,8 @@ bool MCPlusBuilder::hasUseOfPhysReg(const MCInst &MI, unsigned Reg) const { return false; } -const BitVector & -MCPlusBuilder::getAliases(MCPhysReg Reg, - bool OnlySmaller) const { +const BitVector &MCPlusBuilder::getAliases(MCPhysReg Reg, + bool OnlySmaller) const { // AliasMap caches a mapping of registers to the set of registers that // alias (are sub or superregs of itself, including itself). static std::vector AliasMap; @@ -490,8 +484,7 @@ MCPlusBuilder::getAliases(MCPhysReg Reg, return AliasMap[SuperReg[Reg]]; } -uint8_t -MCPlusBuilder::getRegSize(MCPhysReg Reg) const { +uint8_t MCPlusBuilder::getRegSize(MCPhysReg Reg) const { // SizeMap caches a mapping of registers to their sizes static std::vector SizeMap; diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp index 28d702838a94..5821a220976c 100644 --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -176,9 +176,7 @@ bool skipRelocationProcessAArch64(uint64_t Type, uint64_t Contents) { return false; }; - auto IsNop = [](uint64_t Contents) -> bool { - return Contents == 0xd503201f; - }; + auto IsNop = [](uint64_t Contents) -> bool { return Contents == 0xd503201f; }; // The linker might eliminate the instruction and replace it with NOP, ignore if (IsNop(Contents)) @@ -550,25 +548,23 @@ size_t Relocation::emit(MCStreamer *Streamer) const { if (Symbol) { Value = MCSymbolRefExpr::create(Symbol, Ctx); if (Addend) { - Value = MCBinaryExpr::createAdd(Value, - MCConstantExpr::create(Addend, Ctx), - Ctx); + Value = MCBinaryExpr::createAdd( + Value, MCConstantExpr::create(Addend, Ctx), Ctx); } } else { Value = MCConstantExpr::create(Addend, Ctx); } - Value = MCBinaryExpr::createSub(Value, - MCSymbolRefExpr::create(TempLabel, Ctx), - Ctx); + Value = MCBinaryExpr::createSub( + Value, MCSymbolRefExpr::create(TempLabel, Ctx), Ctx); Streamer->emitValue(Value, Size); return Size; } if (Symbol && Addend) { - auto Value = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, Ctx), - MCConstantExpr::create(Addend, Ctx), - Ctx); + auto Value = + MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, Ctx), + MCConstantExpr::create(Addend, Ctx), Ctx); Streamer->emitValue(Value, Size); } else if (Symbol) { Streamer->emitSymbolValue(Symbol, Size); diff --git a/bolt/lib/Passes/BinaryFunctionCallGraph.cpp b/bolt/lib/Passes/BinaryFunctionCallGraph.cpp index 6f8b4577de86..ab903102febe 100644 --- a/bolt/lib/Passes/BinaryFunctionCallGraph.cpp +++ b/bolt/lib/Passes/BinaryFunctionCallGraph.cpp @@ -20,7 +20,7 @@ namespace opts { extern llvm::cl::opt TimeOpts; extern llvm::cl::opt Verbosity; -} +} // namespace opts namespace llvm { namespace bolt { @@ -76,14 +76,11 @@ std::deque BinaryFunctionCallGraph::buildTraversalOrder() { return TopologicalOrder; } -BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, - CgFilterFunction Filter, - bool CgFromPerfData, - bool IncludeColdCalls, - bool UseFunctionHotSize, - bool UseSplitHotSize, - bool UseEdgeCounts, - bool IgnoreRecursiveCalls) { +BinaryFunctionCallGraph +buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData, + bool IncludeColdCalls, bool UseFunctionHotSize, + bool UseSplitHotSize, bool UseEdgeCounts, + bool IgnoreRecursiveCalls) { NamedRegionTimer T1("buildcg", "Callgraph construction", "CG breakdown", "CG breakdown", opts::TimeOpts); BinaryFunctionCallGraph Cg; @@ -93,8 +90,8 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, // Compute function size auto functionSize = [&](const BinaryFunction *Function) { return UseFunctionHotSize && Function->isSplit() - ? Function->estimateHotSize(UseSplitHotSize) - : Function->estimateSize(); + ? Function->estimateHotSize(UseSplitHotSize) + : Function->estimateSize(); }; // Add call graph nodes. @@ -156,11 +153,10 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, if (!IsValidCount) ++NoProfileCallsites; Cg.incArcWeight(SrcId, DstId, AdjCount, Offset); - LLVM_DEBUG( - if (opts::Verbosity > 1) { - dbgs() << "BOLT-DEBUG: buildCallGraph: call " << *Function - << " -> " << *DstFunc << " @ " << Offset << "\n"; - }); + LLVM_DEBUG(if (opts::Verbosity > 1) { + dbgs() << "BOLT-DEBUG: buildCallGraph: call " << *Function << " -> " + << *DstFunc << " @ " << Offset << "\n"; + }); return true; } @@ -180,8 +176,8 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, // If this is an indirect call use perf data directly. if (!DstSym && BC.MIB->hasAnnotation(Inst, "CallProfile")) { - const auto &ICSP = - BC.MIB->getAnnotationAs(Inst, "CallProfile"); + const auto &ICSP = BC.MIB->getAnnotationAs( + Inst, "CallProfile"); for (const IndirectCallProfile &CSI : ICSP) { if (CSI.Symbol) Counts.emplace_back(CSI.Symbol, CSI.Count); @@ -281,5 +277,5 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, return Cg; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp index 8664d962cc53..d8bc954f27d5 100644 --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -24,13 +24,13 @@ using namespace bolt; namespace { -const char* dynoStatsOptName(const bolt::DynoStats::Category C) { +const char *dynoStatsOptName(const bolt::DynoStats::Category C) { if (C == bolt::DynoStats::FIRST_DYNO_STAT) return "none"; else if (C == bolt::DynoStats::LAST_DYNO_STAT) return "all"; - static std::string OptNames[bolt::DynoStats::LAST_DYNO_STAT+1]; + static std::string OptNames[bolt::DynoStats::LAST_DYNO_STAT + 1]; OptNames[C] = bolt::DynoStats::Description(C); @@ -39,7 +39,7 @@ const char* dynoStatsOptName(const bolt::DynoStats::Category C) { return OptNames[C].c_str(); } -const char* dynoStatsOptDesc(const bolt::DynoStats::Category C) { +const char *dynoStatsOptDesc(const bolt::DynoStats::Category C) { if (C == bolt::DynoStats::FIRST_DYNO_STAT) return "unsorted"; else if (C == bolt::DynoStats::LAST_DYNO_STAT) @@ -263,8 +263,7 @@ namespace llvm { namespace bolt { bool BinaryFunctionPass::shouldOptimize(const BinaryFunction &BF) const { - return BF.isSimple() && - BF.getState() == BinaryFunction::State::CFG && + return BF.isSimple() && BF.getState() == BinaryFunction::State::CFG && !BF.isIgnored(); } @@ -356,7 +355,7 @@ void NormalizeCFG::runOnFunctions(BinaryContext &BC) { << '\n'; } -void EliminateUnreachableBlocks::runOnFunction(BinaryFunction& Function) { +void EliminateUnreachableBlocks::runOnFunction(BinaryFunction &Function) { if (Function.layout_size() > 0) { unsigned Count; uint64_t Bytes; @@ -453,8 +452,8 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) { << opts::PrintFuncStat << " functions. (Ranked by function score)" << "\n\n"; uint64_t I = 0; - for (std::map::reverse_iterator - Rit = ScoreMap.rbegin(); + for (std::map::reverse_iterator Rit = + ScoreMap.rbegin(); Rit != ScoreMap.rend() && I < opts::PrintFuncStat; ++Rit, ++I) { BinaryFunction &Function = Rit->second; @@ -472,7 +471,8 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) { } void ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF, - LayoutType Type, bool MinBranchClusters) const { + LayoutType Type, + bool MinBranchClusters) const { if (BF.size() == 0 || Type == LT_NONE) return; @@ -498,7 +498,7 @@ void ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF, else CAlgo.reset(new PHGreedyClusterAlgorithm()); - switch(Type) { + switch (Type) { case LT_OPTIMIZE: Algo.reset(new OptimizeReorderAlgorithm(std::move(CAlgo))); break; @@ -622,7 +622,8 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) { const int64_t NewGnuArgsSize = BC.MIB->getGnuArgsSize(*II); assert(NewGnuArgsSize >= 0 && "expected non-negative GNU_args_size"); if (NewGnuArgsSize != CurrentGnuArgsSize) { - auto InsertII = BF.addCFIInstruction(BB, II, + auto InsertII = BF.addCFIInstruction( + BB, II, MCCFIInstruction::createGnuArgsSize(nullptr, NewGnuArgsSize)); CurrentGnuArgsSize = NewGnuArgsSize; II = std::next(InsertII); @@ -678,8 +679,7 @@ uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) { MCContext *Ctx = Function.getBinaryContext().Ctx.get(); MCPlusBuilder *MIB = Function.getBinaryContext().MIB.get(); for (BinaryBasicBlock &BB : Function) { - auto checkAndPatch = [&](BinaryBasicBlock *Pred, - BinaryBasicBlock *Succ, + auto checkAndPatch = [&](BinaryBasicBlock *Pred, BinaryBasicBlock *Succ, const MCSymbol *SuccSym) { // Ignore infinite loop jumps or fallthrough tail jumps. if (Pred == Succ || Succ == &BB) @@ -691,7 +691,7 @@ uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) { MCInst *CondBranch = nullptr; MCInst *UncondBranch = nullptr; bool Res = Pred->analyzeBranch(TBB, FBB, CondBranch, UncondBranch); - if(!Res) { + if (!Res) { LLVM_DEBUG(dbgs() << "analyzeBranch failed in peepholes in block:\n"; Pred->dump()); return false; @@ -764,8 +764,7 @@ uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) { (Pred->getConditionalSuccessor(true) == &BB && !IsTailCall) || Pred->getConditionalSuccessor(false) == &BB) { if (checkAndPatch(Pred, Succ, SuccSym) && MarkInvalid) { - BB.markValid(BB.pred_size() != 0 || - BB.isLandingPad() || + BB.markValid(BB.pred_size() != 0 || BB.isLandingPad() || BB.isEntryPoint()); } } @@ -774,14 +773,11 @@ uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) { return NumDoubleJumps; } -} +} // namespace bool SimplifyConditionalTailCalls::shouldRewriteBranch( - const BinaryBasicBlock *PredBB, - const MCInst &CondBranch, - const BinaryBasicBlock *BB, - const bool DirectionFlag -) { + const BinaryBasicBlock *PredBB, const MCInst &CondBranch, + const BinaryBasicBlock *BB, const bool DirectionFlag) { if (BeenOptimized.count(PredBB)) return false; @@ -810,9 +806,8 @@ bool SimplifyConditionalTailCalls::shouldRewriteBranch( return true; // TODO: should this use misprediction frequency instead? - const bool Result = - (IsForward && Frequency.get().first >= 0.5) || - (!IsForward && Frequency.get().first <= 0.5); + const bool Result = (IsForward && Frequency.get().first >= 0.5) || + (!IsForward && Frequency.get().first <= 0.5); return Result == DirectionFlag; } @@ -828,14 +823,12 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { uint64_t NumLocalCTCs = 0; uint64_t LocalCTCTakenCount = 0; uint64_t LocalCTCExecCount = 0; - std::vector> NeedsUncondBranch; + std::vector> + NeedsUncondBranch; // Will block be deleted by UCE? auto isValid = [](const BinaryBasicBlock *BB) { - return (BB->pred_size() != 0 || - BB->isLandingPad() || - BB->isEntryPoint()); + return (BB->pred_size() != 0 || BB->isLandingPad() || BB->isEntryPoint()); }; for (BinaryBasicBlock *BB : BF.layout()) { @@ -913,7 +906,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { Count = PredBB->getTakenBranchInfo().Count; } const uint64_t CTCTakenFreq = - Count == BinaryBasicBlock::COUNT_NO_PROFILE ? 0 : Count; + Count == BinaryBasicBlock::COUNT_NO_PROFILE ? 0 : Count; // Annotate it, so "isCall" returns true for this jcc MIB->setConditionalTailCall(*CondBranch); @@ -1097,13 +1090,9 @@ void Peepholes::removeUselessCondBranches(BinaryFunction &Function) { } void Peepholes::runOnFunctions(BinaryContext &BC) { - const char Opts = - std::accumulate(opts::Peepholes.begin(), - opts::Peepholes.end(), - 0, - [](const char A, const opts::PeepholeOpts B) { - return A | B; - }); + const char Opts = std::accumulate( + opts::Peepholes.begin(), opts::Peepholes.end(), 0, + [](const char A, const opts::PeepholeOpts B) { return A | B; }); if (Opts == opts::PEEP_NONE || !BC.isX86()) return; @@ -1157,7 +1146,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads(BinaryFunction &BF) { MCOperand *DispOpI = MIB->getMemOperandDisp(Inst); assert(DispOpI != Inst.end() && "expected PC-relative displacement"); assert(DispOpI->isExpr() && - "found PC-relative with non-symbolic displacement"); + "found PC-relative with non-symbolic displacement"); // Get displacement symbol. const MCSymbol *DisplSymbol; @@ -1249,8 +1238,7 @@ void AssignSections::runOnFunctions(BinaryContext &BC) { continue; } - if (!UseColdSection || - Function.hasValidIndex() || + if (!UseColdSection || Function.hasValidIndex() || Function.hasValidProfile()) { Function.setCodeSectionName(BC.getMainCodeSectionName()); } else { @@ -1355,8 +1343,7 @@ void PrintProfileStats::runOnFunctions(BinaryContext &BC) { } } -void -PrintProgramStats::runOnFunctions(BinaryContext &BC) { +void PrintProgramStats::runOnFunctions(BinaryContext &BC) { uint64_t NumRegularFunctions = 0; uint64_t NumStaleProfileFunctions = 0; uint64_t NumNonSimpleProfiledFunctions = 0; @@ -1385,7 +1372,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { if (opts::PrintUnknownCFG) { Function.dump(); } else if (opts::PrintUnknown) { - errs() << "function with unknown control flow: " << Function <<'\n'; + errs() << "function with unknown control flow: " << Function << '\n'; } ++NumUnknownControlFlowFunctions; } @@ -1412,19 +1399,19 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { const size_t NumAllProfiledFunctions = ProfiledFunctions.size() + NumStaleProfileFunctions; - outs() << "BOLT-INFO: " << NumAllProfiledFunctions - << " out of " << NumRegularFunctions << " functions in the binary (" + outs() << "BOLT-INFO: " << NumAllProfiledFunctions << " out of " + << NumRegularFunctions << " functions in the binary (" << format("%.1f", NumAllProfiledFunctions / - (float) NumRegularFunctions * 100.0f) + (float)NumRegularFunctions * 100.0f) << "%) have non-empty execution profile\n"; if (NumNonSimpleProfiledFunctions) { - outs() << "BOLT-INFO: " << NumNonSimpleProfiledFunctions - << " function" << (NumNonSimpleProfiledFunctions == 1 ? "" : "s") + outs() << "BOLT-INFO: " << NumNonSimpleProfiledFunctions << " function" + << (NumNonSimpleProfiledFunctions == 1 ? "" : "s") << " with profile could not be optimized\n"; } if (NumStaleProfileFunctions) { const float PctStale = - NumStaleProfileFunctions / (float) NumAllProfiledFunctions * 100.0f; + NumStaleProfileFunctions / (float)NumAllProfiledFunctions * 100.0f; auto printErrorOrWarning = [&]() { if (PctStale > opts::StaleThreshold) { errs() << "BOLT-ERROR: "; @@ -1434,8 +1421,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { }; printErrorOrWarning(); errs() << NumStaleProfileFunctions - << format(" (%.1f%% of all profiled)", PctStale) - << " function" << (NumStaleProfileFunctions == 1 ? "" : "s") + << format(" (%.1f%% of all profiled)", PctStale) << " function" + << (NumStaleProfileFunctions == 1 ? "" : "s") << " have invalid (possibly stale) profile." " Use -report-stale to see the list.\n"; if (TotalSampleCount > 0) { @@ -1464,21 +1451,18 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { std::sort(ProfiledFunctions.begin(), ProfiledFunctions.end(), [](const BinaryFunction *A, const BinaryFunction *B) { return B->getExecutionCount() < A->getExecutionCount(); - } - ); + }); auto SFI = ProfiledFunctions.begin(); auto SFIend = ProfiledFunctions.end(); for (unsigned I = 0u; I < opts::TopCalledLimit && SFI != SFIend; ++SFI, ++I) { - outs() << " " << **SFI << " : " - << (*SFI)->getExecutionCount() << '\n'; + outs() << " " << **SFI << " : " << (*SFI)->getExecutionCount() << '\n'; } } } if (!opts::PrintSortedBy.empty() && - std::find(opts::PrintSortedBy.begin(), - opts::PrintSortedBy.end(), + std::find(opts::PrintSortedBy.begin(), opts::PrintSortedBy.end(), DynoStats::FIRST_DYNO_STAT) == opts::PrintSortedBy.end()) { std::vector Functions; @@ -1493,34 +1477,29 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { } const bool SortAll = - std::find(opts::PrintSortedBy.begin(), - opts::PrintSortedBy.end(), - DynoStats::LAST_DYNO_STAT) != opts::PrintSortedBy.end(); + std::find(opts::PrintSortedBy.begin(), opts::PrintSortedBy.end(), + DynoStats::LAST_DYNO_STAT) != opts::PrintSortedBy.end(); const bool Ascending = - opts::DynoStatsSortOrderOpt == opts::DynoStatsSortOrder::Ascending; + opts::DynoStatsSortOrderOpt == opts::DynoStatsSortOrder::Ascending; if (SortAll) { - std::stable_sort( - Functions.begin(), - Functions.end(), - [Ascending,&Stats](const BinaryFunction *A, const BinaryFunction *B) { - return Ascending ? - Stats.at(A) < Stats.at(B) : Stats.at(B) < Stats.at(A); - } - ); + std::stable_sort(Functions.begin(), Functions.end(), + [Ascending, &Stats](const BinaryFunction *A, + const BinaryFunction *B) { + return Ascending ? Stats.at(A) < Stats.at(B) + : Stats.at(B) < Stats.at(A); + }); } else { std::stable_sort( - Functions.begin(), - Functions.end(), - [Ascending,&Stats](const BinaryFunction *A, const BinaryFunction *B) { - const DynoStats &StatsA = Stats.at(A); - const DynoStats &StatsB = Stats.at(B); - return Ascending - ? StatsA.lessThan(StatsB, opts::PrintSortedBy) - : StatsB.lessThan(StatsA, opts::PrintSortedBy); - } - ); + Functions.begin(), Functions.end(), + [Ascending, &Stats](const BinaryFunction *A, + const BinaryFunction *B) { + const DynoStats &StatsA = Stats.at(A); + const DynoStats &StatsB = Stats.at(B); + return Ascending ? StatsA.lessThan(StatsB, opts::PrintSortedBy) + : StatsB.lessThan(StatsA, opts::PrintSortedBy); + }); } outs() << "BOLT-INFO: top functions sorted by "; @@ -1530,7 +1509,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { outs() << "("; bool PrintComma = false; for (const DynoStats::Category Category : opts::PrintSortedBy) { - if (PrintComma) outs() << ", "; + if (PrintComma) + outs() << ", "; outs() << DynoStats::Description(Category); PrintComma = true; } @@ -1546,7 +1526,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { outs() << " ("; bool PrintComma = false; for (const DynoStats::Category Category : opts::PrintSortedBy) { - if (PrintComma) outs() << ", "; + if (PrintComma) + outs() << ", "; outs() << dynoStatsOptName(Category) << "=" << Stats[Category]; PrintComma = true; } @@ -1557,8 +1538,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { } if (!BC.TrappedFunctions.empty()) { - errs() << "BOLT-WARNING: " << BC.TrappedFunctions.size() - << " function" << (BC.TrappedFunctions.size() > 1 ? "s" : "") + errs() << "BOLT-WARNING: " << BC.TrappedFunctions.size() << " function" + << (BC.TrappedFunctions.size() > 1 ? "s" : "") << " will trap on entry. Use -trap-avx512=0 to disable" " traps."; if (opts::Verbosity >= 1 || BC.TrappedFunctions.size() <= 5) { @@ -1572,9 +1553,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { // Print information on missed macro-fusion opportunities seen on input. if (BC.MissedMacroFusionPairs) { - outs() << "BOLT-INFO: the input contains " - << BC.MissedMacroFusionPairs << " (dynamic count : " - << BC.MissedMacroFusionExecCount + outs() << "BOLT-INFO: the input contains " << BC.MissedMacroFusionPairs + << " (dynamic count : " << BC.MissedMacroFusionExecCount << ") opportunities for macro-fusion optimization"; switch (opts::AlignMacroOpFusion) { case MFT_NONE: @@ -1614,12 +1594,13 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) { if (!SuboptimalFuncs.empty()) { std::sort(SuboptimalFuncs.begin(), SuboptimalFuncs.end(), - [](const BinaryFunction *A, const BinaryFunction *B) { - return A->getKnownExecutionCount() / A->getSize() > - B->getKnownExecutionCount() / B->getSize(); - }); + [](const BinaryFunction *A, const BinaryFunction *B) { + return A->getKnownExecutionCount() / A->getSize() > + B->getKnownExecutionCount() / B->getSize(); + }); - outs() << "BOLT-INFO: " << SuboptimalFuncs.size() << " functions have " + outs() << "BOLT-INFO: " << SuboptimalFuncs.size() + << " functions have " "cold code in the middle of hot code. Top functions are:\n"; for (unsigned I = 0; I < std::min(static_cast(opts::ReportBadLayout), @@ -1656,8 +1637,7 @@ void StripRepRet::runOnFunctions(BinaryContext &BC) { for (auto &BFI : BC.getBinaryFunctions()) { for (BinaryBasicBlock &BB : BFI.second) { auto LastInstRIter = BB.getLastNonPseudo(); - if (LastInstRIter == BB.rend() || - !BC.MIB->isReturn(*LastInstRIter) || + if (LastInstRIter == BB.rend() || !BC.MIB->isReturn(*LastInstRIter) || !BC.MIB->deleteREPPrefix(*LastInstRIter)) continue; @@ -1667,9 +1647,10 @@ void StripRepRet::runOnFunctions(BinaryContext &BC) { } if (NumBytesSaved) { - outs() << "BOLT-INFO: removed " << NumBytesSaved << " 'repz' prefixes" - " with estimated execution count of " << NumPrefixesRemoved - << " times.\n"; + outs() << "BOLT-INFO: removed " << NumBytesSaved + << " 'repz' prefixes" + " with estimated execution count of " + << NumPrefixesRemoved << " times.\n"; } } @@ -1735,8 +1716,8 @@ bool SpecializeMemcpy1::shouldOptimize(const BinaryFunction &Function) const { return false; } -std::set -SpecializeMemcpy1::getCallSitesToOptimize(const BinaryFunction &Function) const{ +std::set SpecializeMemcpy1::getCallSitesToOptimize( + const BinaryFunction &Function) const { StringRef SitesString; for (const std::string &FunctionSpec : Spec) { StringRef FunctionName; diff --git a/bolt/lib/Passes/CacheMetrics.cpp b/bolt/lib/Passes/CacheMetrics.cpp index 21e2657d3984..ebaf09d7736c 100644 --- a/bolt/lib/Passes/CacheMetrics.cpp +++ b/bolt/lib/Passes/CacheMetrics.cpp @@ -30,15 +30,15 @@ extern cl::opt BackwardDistance; extern cl::opt ITLBPageSize; extern cl::opt ITLBEntries; -} +} // namespace opts namespace { /// Initialize and return a position map for binary basic blocks void extractBasicBlockInfo( - const std::vector &BinaryFunctions, - std::unordered_map &BBAddr, - std::unordered_map &BBSize) { + const std::vector &BinaryFunctions, + std::unordered_map &BBAddr, + std::unordered_map &BBSize) { for (BinaryFunction *BF : BinaryFunctions) { const BinaryContext &BC = BF->getBinaryContext(); @@ -58,10 +58,10 @@ void extractBasicBlockInfo( /// Calculate TSP metric, which quantifies the number of fallthrough jumps in /// the ordering of basic blocks -double calcTSPScore( - const std::vector &BinaryFunctions, - const std::unordered_map &BBAddr, - const std::unordered_map &BBSize) { +double +calcTSPScore(const std::vector &BinaryFunctions, + const std::unordered_map &BBAddr, + const std::unordered_map &BBSize) { double Score = 0; for (BinaryFunction *BF : BinaryFunctions) { @@ -83,9 +83,9 @@ double calcTSPScore( /// Calculate Ext-TSP metric, which quantifies the expected number of i-cache /// misses for a given ordering of basic blocks double calcExtTSPScore( - const std::vector &BinaryFunctions, - const std::unordered_map &BBAddr, - const std::unordered_map &BBSize) { + const std::vector &BinaryFunctions, + const std::unordered_map &BBAddr, + const std::unordered_map &BBSize) { double Score = 0.0; for (BinaryFunction *BF : BinaryFunctions) { @@ -95,10 +95,8 @@ double calcExtTSPScore( auto BI = SrcBB->branch_info_begin(); for (BinaryBasicBlock *DstBB : SrcBB->successors()) { if (DstBB != SrcBB) { - Score += CacheMetrics::extTSPScore(BBAddr.at(SrcBB), - BBSize.at(SrcBB), - BBAddr.at(DstBB), - BI->Count); + Score += CacheMetrics::extTSPScore(BBAddr.at(SrcBB), BBSize.at(SrcBB), + BBAddr.at(DstBB), BI->Count); } ++BI; } @@ -124,7 +122,7 @@ extractFunctionCalls(const std::vector &BinaryFunctions) { continue; // Call info - const MCSymbol* DstSym = BC.MIB->getTargetSymbol(Inst); + const MCSymbol *DstSym = BC.MIB->getTargetSymbol(Inst); uint64_t Count = BB->getKnownExecutionCount(); // Ignore calls w/o information if (DstSym == nullptr || Count == 0) @@ -132,8 +130,7 @@ extractFunctionCalls(const std::vector &BinaryFunctions) { const BinaryFunction *DstFunction = BC.getFunctionForSymbol(DstSym); // Ignore recursive calls - if (DstFunction == nullptr || - DstFunction->layout_empty() || + if (DstFunction == nullptr || DstFunction->layout_empty() || DstFunction == SrcFunction) continue; @@ -150,17 +147,17 @@ extractFunctionCalls(const std::vector &BinaryFunctions) { /// functions calls into two categories: /// - 'short' ones that have a caller-callee distance less than a page; /// - 'long' ones where the distance exceeds a page. -/// The short calls are likely to result in a i-TLB cache hit. For the long ones, -/// the hit/miss result depends on the 'hotness' of the page (i.e., how often -/// the page is accessed). Assuming that functions are sent to the i-TLB cache -/// in a random order, the probability that a page is present in the cache is -/// proportional to the number of samples corresponding to the functions on the -/// page. The following procedure detects short and long calls, and estimates -/// the expected number of cache misses for the long ones. +/// The short calls are likely to result in a i-TLB cache hit. For the long +/// ones, the hit/miss result depends on the 'hotness' of the page (i.e., how +/// often the page is accessed). Assuming that functions are sent to the i-TLB +/// cache in a random order, the probability that a page is present in the cache +/// is proportional to the number of samples corresponding to the functions on +/// the page. The following procedure detects short and long calls, and +/// estimates the expected number of cache misses for the long ones. double expectedCacheHitRatio( - const std::vector &BinaryFunctions, - const std::unordered_map &BBAddr, - const std::unordered_map &BBSize) { + const std::vector &BinaryFunctions, + const std::unordered_map &BBAddr, + const std::unordered_map &BBSize) { const double PageSize = opts::ITLBPageSize; const uint64_t CacheEntries = opts::ITLBEntries; @@ -218,12 +215,10 @@ double expectedCacheHitRatio( return 100.0 * (1.0 - Misses / TotalSamples); } -} // end namespace anonymous +} // namespace -double CacheMetrics::extTSPScore(uint64_t SrcAddr, - uint64_t SrcSize, - uint64_t DstAddr, - uint64_t Count) { +double CacheMetrics::extTSPScore(uint64_t SrcAddr, uint64_t SrcSize, + uint64_t DstAddr, uint64_t Count) { assert(Count != BinaryBasicBlock::COUNT_NO_PROFILE); // Fallthrough @@ -283,22 +278,23 @@ void CacheMetrics::printAll(const std::vector &BFs) { } outs() << format(" There are %zu functions;", NumFunctions) - << format(" %zu (%.2lf%%) are in the hot section,", - NumHotFunctions, 100.0 * NumHotFunctions / NumFunctions) - << format(" %zu (%.2lf%%) have profile\n", - NumProfiledFunctions, 100.0 * NumProfiledFunctions / NumFunctions); + << format(" %zu (%.2lf%%) are in the hot section,", NumHotFunctions, + 100.0 * NumHotFunctions / NumFunctions) + << format(" %zu (%.2lf%%) have profile\n", NumProfiledFunctions, + 100.0 * NumProfiledFunctions / NumFunctions); outs() << format(" There are %zu basic blocks;", NumBlocks) - << format(" %zu (%.2lf%%) are in the hot section\n", - NumHotBlocks, 100.0 * NumHotBlocks / NumBlocks); + << format(" %zu (%.2lf%%) are in the hot section\n", NumHotBlocks, + 100.0 * NumHotBlocks / NumBlocks); assert(TotalCodeMinAddr <= TotalCodeMaxAddr && "incorrect output addresses"); size_t HotCodeSize = HotCodeMaxAddr - HotCodeMinAddr; size_t TotalCodeSize = TotalCodeMaxAddr - TotalCodeMinAddr; size_t HugePage2MB = 2 << 20; - outs() << format(" Hot code takes %.2lf%% of binary (%zu bytes out of %zu, %.2lf huge pages)\n", - 100.0 * HotCodeSize / TotalCodeSize, HotCodeSize, TotalCodeSize, - double(HotCodeSize) / HugePage2MB); + outs() << format(" Hot code takes %.2lf%% of binary (%zu bytes out of %zu, " + "%.2lf huge pages)\n", + 100.0 * HotCodeSize / TotalCodeSize, HotCodeSize, + TotalCodeSize, double(HotCodeSize) / HugePage2MB); // Stats related to expected cache performance std::unordered_map BBAddr; diff --git a/bolt/lib/Passes/CallGraphWalker.cpp b/bolt/lib/Passes/CallGraphWalker.cpp index 1033097293b6..7bdd9b6c496a 100644 --- a/bolt/lib/Passes/CallGraphWalker.cpp +++ b/bolt/lib/Passes/CallGraphWalker.cpp @@ -61,5 +61,5 @@ void CallGraphWalker::walk() { traverseCG(); } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/DataflowInfoManager.cpp b/bolt/lib/Passes/DataflowInfoManager.cpp index 6c36ea92cf55..0d33be887b7c 100644 --- a/bolt/lib/Passes/DataflowInfoManager.cpp +++ b/bolt/lib/Passes/DataflowInfoManager.cpp @@ -22,9 +22,7 @@ ReachingDefOrUse &DataflowInfoManager::getReachingDefs() { return *RD; } -void DataflowInfoManager::invalidateReachingDefs() { - RD.reset(nullptr); -} +void DataflowInfoManager::invalidateReachingDefs() { RD.reset(nullptr); } ReachingDefOrUse &DataflowInfoManager::getReachingUses() { if (RU) @@ -35,9 +33,7 @@ ReachingDefOrUse &DataflowInfoManager::getReachingUses() { return *RU; } -void DataflowInfoManager::invalidateReachingUses() { - RU.reset(nullptr); -} +void DataflowInfoManager::invalidateReachingUses() { RU.reset(nullptr); } LivenessAnalysis &DataflowInfoManager::getLivenessAnalysis() { if (LA) @@ -48,9 +44,7 @@ LivenessAnalysis &DataflowInfoManager::getLivenessAnalysis() { return *LA; } -void DataflowInfoManager::invalidateLivenessAnalysis() { - LA.reset(nullptr); -} +void DataflowInfoManager::invalidateLivenessAnalysis() { LA.reset(nullptr); } StackReachingUses &DataflowInfoManager::getStackReachingUses() { if (SRU) @@ -61,9 +55,7 @@ StackReachingUses &DataflowInfoManager::getStackReachingUses() { return *SRU; } -void DataflowInfoManager::invalidateStackReachingUses() { - SRU.reset(nullptr); -} +void DataflowInfoManager::invalidateStackReachingUses() { SRU.reset(nullptr); } DominatorAnalysis &DataflowInfoManager::getDominatorAnalysis() { if (DA) @@ -73,9 +65,7 @@ DominatorAnalysis &DataflowInfoManager::getDominatorAnalysis() { return *DA; } -void DataflowInfoManager::invalidateDominatorAnalysis() { - DA.reset(nullptr); -} +void DataflowInfoManager::invalidateDominatorAnalysis() { DA.reset(nullptr); } DominatorAnalysis &DataflowInfoManager::getPostDominatorAnalysis() { if (PDA) @@ -110,9 +100,7 @@ ReachingInsns &DataflowInfoManager::getReachingInsns() { return *RI; } -void DataflowInfoManager::invalidateReachingInsns() { - RI.reset(nullptr); -} +void DataflowInfoManager::invalidateReachingInsns() { RI.reset(nullptr); } ReachingInsns &DataflowInfoManager::getReachingInsnsBackwards() { if (RIB) @@ -151,9 +139,7 @@ DataflowInfoManager::getInsnToBBMap() { return *InsnToBB; } -void DataflowInfoManager::invalidateInsnToBBMap() { - InsnToBB.reset(nullptr); -} +void DataflowInfoManager::invalidateInsnToBBMap() { InsnToBB.reset(nullptr); } void DataflowInfoManager::invalidateAll() { invalidateReachingDefs(); diff --git a/bolt/lib/Passes/ExtTSPReorderAlgorithm.cpp b/bolt/lib/Passes/ExtTSPReorderAlgorithm.cpp index 25d855ea3971..08bef3d89dd5 100644 --- a/bolt/lib/Passes/ExtTSPReorderAlgorithm.cpp +++ b/bolt/lib/Passes/ExtTSPReorderAlgorithm.cpp @@ -95,9 +95,7 @@ class Edge; // Calculate Ext-TSP value, which quantifies the expected number of i-cache // misses for a given ordering of basic blocks -double extTSPScore(uint64_t SrcAddr, - uint64_t SrcSize, - uint64_t DstAddr, +double extTSPScore(uint64_t SrcAddr, uint64_t SrcSize, uint64_t DstAddr, uint64_t Count) { assert(Count != BinaryBasicBlock::COUNT_NO_PROFILE); @@ -139,24 +137,16 @@ class MergeGainTy { public: explicit MergeGainTy() {} explicit MergeGainTy(double Score, size_t MergeOffset, MergeTypeTy MergeType) - : Score(Score), - MergeOffset(MergeOffset), - MergeType(MergeType) {} + : Score(Score), MergeOffset(MergeOffset), MergeType(MergeType) {} - double score() const { - return Score; - } + double score() const { return Score; } - size_t mergeOffset() const { - return MergeOffset; - } + size_t mergeOffset() const { return MergeOffset; } - MergeTypeTy mergeType() const { - return MergeType; - } + MergeTypeTy mergeType() const { return MergeType; } // returns 'true' iff Other is preferred over this - bool operator < (const MergeGainTy& Other) const { + bool operator<(const MergeGainTy &Other) const { return (Other.Score > EPS && Other.Score > Score + EPS); } @@ -170,10 +160,10 @@ private: // The class wraps several mutable fields utilized in the ExtTSP algorithm class Block { public: - Block(const Block&) = delete; - Block(Block&&) = default; - Block& operator=(const Block&) = delete; - Block& operator=(Block&&) = default; + Block(const Block &) = delete; + Block(Block &&) = default; + Block &operator=(const Block &) = delete; + Block &operator=(Block &&) = default; // Corresponding basic block BinaryBasicBlock *BB{nullptr}; @@ -204,10 +194,8 @@ public: public: explicit Block(BinaryBasicBlock *BB_, uint64_t Size_) - : BB(BB_), - Size(Size_), - ExecutionCount(BB_->getKnownExecutionCount()), - Index(BB->getLayoutIndex()) {} + : BB(BB_), Size(Size_), ExecutionCount(BB_->getKnownExecutionCount()), + Index(BB->getLayoutIndex()) {} bool adjacent(const Block *Other) const { return hasOutJump(Other) || hasInJump(Other); @@ -233,54 +221,33 @@ public: // A chain (ordered sequence) of CFG nodes (basic blocks) class Chain { public: - Chain(const Chain&) = delete; - Chain(Chain&&) = default; - Chain& operator=(const Chain&) = delete; - Chain& operator=(Chain&&) = default; + Chain(const Chain &) = delete; + Chain(Chain &&) = default; + Chain &operator=(const Chain &) = delete; + Chain &operator=(Chain &&) = default; explicit Chain(size_t Id, Block *Block) - : Id(Id), - IsEntry(Block->Index == 0), - ExecutionCount(Block->ExecutionCount), - Size(Block->Size), - Score(0), - Blocks(1, Block) {} + : Id(Id), IsEntry(Block->Index == 0), + ExecutionCount(Block->ExecutionCount), Size(Block->Size), Score(0), + Blocks(1, Block) {} - size_t id() const { - return Id; - } + size_t id() const { return Id; } - uint64_t size() const { - return Size; - } + uint64_t size() const { return Size; } - double density() const { - return static_cast(ExecutionCount) / Size; - } + double density() const { return static_cast(ExecutionCount) / Size; } - uint64_t executionCount() const { - return ExecutionCount; - } + uint64_t executionCount() const { return ExecutionCount; } - bool isEntryPoint() const { - return IsEntry; - } + bool isEntryPoint() const { return IsEntry; } - double score() const { - return Score; - } + double score() const { return Score; } - void setScore(double NewScore) { - Score = NewScore; - } + void setScore(double NewScore) { Score = NewScore; } - const std::vector &blocks() const { - return Blocks; - } + const std::vector &blocks() const { return Blocks; } - const std::vector> &edges() const { - return Edges; - } + const std::vector> &edges() const { return Edges; } Edge *getEdge(Chain *Other) const { for (std::pair It : Edges) { @@ -340,19 +307,16 @@ private: // there is always at most one edge between a pair of chains class Edge { public: - Edge(const Edge&) = delete; - Edge(Edge&&) = default; - Edge& operator=(const Edge&) = delete; - Edge& operator=(Edge&&) = default; + Edge(const Edge &) = delete; + Edge(Edge &&) = default; + Edge &operator=(const Edge &) = delete; + Edge &operator=(Edge &&) = default; explicit Edge(Block *SrcBlock, Block *DstBlock, uint64_t EC) - : SrcChain(SrcBlock->CurChain), - DstChain(DstBlock->CurChain), - Jumps(1, std::make_pair(std::make_pair(SrcBlock, DstBlock), EC)) {} + : SrcChain(SrcBlock->CurChain), DstChain(DstBlock->CurChain), + Jumps(1, std::make_pair(std::make_pair(SrcBlock, DstBlock), EC)) {} - const JumpList &jumps() const { - return Jumps; - } + const JumpList &jumps() const { return Jumps; } void changeEndpoint(Chain *From, Chain *To) { if (From == SrcChain) @@ -439,21 +403,13 @@ void Chain::mergeEdges(Chain *Other) { // instantiation of the vectors. class MergedChain { public: - MergedChain(BlockIter Begin1, - BlockIter End1, - BlockIter Begin2 = BlockIter(), - BlockIter End2 = BlockIter(), - BlockIter Begin3 = BlockIter(), + MergedChain(BlockIter Begin1, BlockIter End1, BlockIter Begin2 = BlockIter(), + BlockIter End2 = BlockIter(), BlockIter Begin3 = BlockIter(), BlockIter End3 = BlockIter()) - : Begin1(Begin1), - End1(End1), - Begin2(Begin2), - End2(End2), - Begin3(Begin3), - End3(End3) {} + : Begin1(Begin1), End1(End1), Begin2(Begin2), End2(End2), Begin3(Begin3), + End3(End3) {} - template - void forEach(const F &Func) const { + template void forEach(const F &Func) const { for (auto It = Begin1; It != End1; It++) Func(*It); for (auto It = Begin2; It != End2; It++) @@ -464,8 +420,7 @@ public: std::vector getBlocks() const { std::vector Result; - Result.reserve(std::distance(Begin1, End1) + - std::distance(Begin2, End2) + + Result.reserve(std::distance(Begin1, End1) + std::distance(Begin2, End2) + std::distance(Begin3, End3)); Result.insert(Result.end(), Begin1, End1); Result.insert(Result.end(), Begin2, End2); @@ -473,9 +428,7 @@ public: return Result; } - const Block *getFirstBlock() const { - return *Begin1; - } + const Block *getFirstBlock() const { return *Begin1; } private: BlockIter Begin1; @@ -487,8 +440,8 @@ private: }; /// Deterministically compare pairs of chains -bool compareChainPairs(const Chain *A1, const Chain *B1, - const Chain *A2, const Chain *B2) { +bool compareChainPairs(const Chain *A1, const Chain *B1, const Chain *A2, + const Chain *B2) { const uint64_t Samples1 = A1->executionCount() + B1->executionCount(); const uint64_t Samples2 = A2->executionCount() + B2->executionCount(); if (Samples1 != Samples2) @@ -501,9 +454,7 @@ bool compareChainPairs(const Chain *A1, const Chain *B1, } class ExtTSP { public: - ExtTSP(const BinaryFunction &BF) : BF(BF) { - initialize(); - } + ExtTSP(const BinaryFunction &BF) : BF(BF) { initialize(); } /// Run the algorithm and return an ordering of basic block void run(BinaryFunction::BasicBlockOrderType &Order) { @@ -624,8 +575,7 @@ private: class Block *const SuccBlock = Edge.first; // Successor cannot be the first BB, which is pinned if (Block.OutWeight == Edge.second && - SuccBlock->InWeight == Edge.second && - SuccBlock->Index != 0) { + SuccBlock->InWeight == Edge.second && SuccBlock->Index != 0) { Block.FallthroughSucc = SuccBlock; SuccBlock->FallthroughPred = &Block; break; @@ -688,9 +638,7 @@ private: if (BestGain < CurGain || (std::abs(CurGain.score() - BestGain.score()) < EPS && - compareChainPairs(ChainPred, - ChainSucc, - BestChainPred, + compareChainPairs(ChainPred, ChainSucc, BestChainPred, BestChainSucc))) { BestGain = CurGain; BestChainPred = ChainPred; @@ -704,9 +652,7 @@ private: break; // Merge the best pair of chains - mergeChains(BestChainPred, - BestChainSucc, - BestGain.mergeOffset(), + mergeChains(BestChainPred, BestChainSucc, BestGain.mergeOffset(), BestGain.mergeType()); } } @@ -747,10 +693,8 @@ private: for (const std::pair, uint64_t> &Jump : Jumps) { const Block *SrcBlock = Jump.first.first; const Block *DstBlock = Jump.first.second; - Score += extTSPScore(SrcBlock->EstimatedAddr, - SrcBlock->Size, - DstBlock->EstimatedAddr, - Jump.second); + Score += extTSPScore(SrcBlock->EstimatedAddr, SrcBlock->Size, + DstBlock->EstimatedAddr, Jump.second); } return Score; } @@ -775,8 +719,8 @@ private: MergeGainTy Gain = MergeGainTy(); // Try to concatenate two chains w/o splitting - Gain = computeMergeGain( - Gain, ChainPred, ChainSucc, Jumps, 0, MergeTypeTy::X_Y); + Gain = computeMergeGain(Gain, ChainPred, ChainSucc, Jumps, 0, + MergeTypeTy::X_Y); // Try to break ChainPred in various ways and concatenate with ChainSucc if (ChainPred->blocks().size() <= opts::ChainSplitThreshold) { @@ -790,12 +734,12 @@ private: continue; } - Gain = computeMergeGain( - Gain, ChainPred, ChainSucc, Jumps, Offset, MergeTypeTy::X1_Y_X2); - Gain = computeMergeGain( - Gain, ChainPred, ChainSucc, Jumps, Offset, MergeTypeTy::Y_X2_X1); - Gain = computeMergeGain( - Gain, ChainPred, ChainSucc, Jumps, Offset, MergeTypeTy::X2_X1_Y); + Gain = computeMergeGain(Gain, ChainPred, ChainSucc, Jumps, Offset, + MergeTypeTy::X1_Y_X2); + Gain = computeMergeGain(Gain, ChainPred, ChainSucc, Jumps, Offset, + MergeTypeTy::Y_X2_X1); + Gain = computeMergeGain(Gain, ChainPred, ChainSucc, Jumps, Offset, + MergeTypeTy::X2_X1_Y); } } @@ -805,10 +749,8 @@ private: /// Merge two chains and update the best Gain MergeGainTy computeMergeGain(const MergeGainTy &CurGain, - const Chain *ChainPred, - const Chain *ChainSucc, - const JumpList &Jumps, - size_t MergeOffset, + const Chain *ChainPred, const Chain *ChainSucc, + const JumpList &Jumps, size_t MergeOffset, MergeTypeTy MergeType) const { MergedChain MergedBlocks = mergeBlocks( ChainPred->blocks(), ChainSucc->blocks(), MergeOffset, MergeType); @@ -830,8 +772,7 @@ private: /// Otherwise, the first chain is cut into two sub-chains at the offset, /// and merged using all possible ways of concatenating three chains. MergedChain mergeBlocks(const std::vector &X, - const std::vector &Y, - size_t MergeOffset, + const std::vector &Y, size_t MergeOffset, MergeTypeTy MergeType) const { // Split the first chain, X, into X1 and X2 BlockIter BeginX1 = X.begin(); @@ -842,15 +783,15 @@ private: BlockIter EndY = Y.end(); // Construct a new chain from the three existing ones - switch(MergeType) { - case MergeTypeTy::X_Y: - return MergedChain(BeginX1, EndX2, BeginY, EndY); - case MergeTypeTy::X1_Y_X2: - return MergedChain(BeginX1, EndX1, BeginY, EndY, BeginX2, EndX2); - case MergeTypeTy::Y_X2_X1: - return MergedChain(BeginY, EndY, BeginX2, EndX2, BeginX1, EndX1); - case MergeTypeTy::X2_X1_Y: - return MergedChain(BeginX2, EndX2, BeginX1, EndX1, BeginY, EndY); + switch (MergeType) { + case MergeTypeTy::X_Y: + return MergedChain(BeginX1, EndX2, BeginY, EndY); + case MergeTypeTy::X1_Y_X2: + return MergedChain(BeginX1, EndX1, BeginY, EndY, BeginX2, EndX2); + case MergeTypeTy::Y_X2_X1: + return MergedChain(BeginY, EndY, BeginX2, EndX2, BeginX1, EndX1); + case MergeTypeTy::X2_X1_Y: + return MergedChain(BeginX2, EndX2, BeginX1, EndX1, BeginY, EndY); } llvm_unreachable("unexpected merge type"); @@ -858,9 +799,7 @@ private: /// Merge chain From into chain Into, update the list of active chains, /// adjacency information, and the corresponding cached values - void mergeChains(Chain *Into, - Chain *From, - size_t MergeOffset, + void mergeChains(Chain *Into, Chain *From, size_t MergeOffset, MergeTypeTy MergeType) { assert(Into != From && "a chain cannot be merged with itself"); @@ -946,8 +885,8 @@ private: std::vector AllEdges; }; -void ExtTSPReorderAlgorithm::reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const { +void ExtTSPReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const { if (BF.layout_empty()) return; diff --git a/bolt/lib/Passes/FrameAnalysis.cpp b/bolt/lib/Passes/FrameAnalysis.cpp index ebd0a193a682..ab10e43169bd 100644 --- a/bolt/lib/Passes/FrameAnalysis.cpp +++ b/bolt/lib/Passes/FrameAnalysis.cpp @@ -260,8 +260,7 @@ void FrameAnalysis::addArgInStackAccessFor(MCInst &Inst, } void FrameAnalysis::addFIEFor(MCInst &Inst, const FrameIndexEntry &FIE) { - BC.MIB->addAnnotation(Inst, "FrameAccessEntry", - (unsigned)FIEVector.size()); + BC.MIB->addAnnotation(Inst, "FrameAccessEntry", (unsigned)FIEVector.size()); FIEVector.emplace_back(FIE); } @@ -295,9 +294,8 @@ FrameAnalysis::getFIEFor(const MCInst &Inst) const { void FrameAnalysis::traverseCG(BinaryFunctionCallGraph &CG) { CallGraphWalker CGWalker(CG); - CGWalker.registerVisitor([&](BinaryFunction *Func) -> bool { - return computeArgsAccessed(*Func); - }); + CGWalker.registerVisitor( + [&](BinaryFunction *Func) -> bool { return computeArgsAccessed(*Func); }); CGWalker.walk(); diff --git a/bolt/lib/Passes/FrameOptimizer.cpp b/bolt/lib/Passes/FrameOptimizer.cpp index c32b1579342a..8017f5d185cd 100644 --- a/bolt/lib/Passes/FrameOptimizer.cpp +++ b/bolt/lib/Passes/FrameOptimizer.cpp @@ -66,7 +66,7 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA, const auto ExprEnd = SAE.expr_end(); MCPlusBuilder *MIB = BF.getBinaryContext().MIB.get(); for (BinaryBasicBlock &BB : BF) { - LLVM_DEBUG(dbgs() <<"\tNow at BB " << BB.getName() << "\n"); + LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n"); const MCInst *Prev = nullptr; for (MCInst &Inst : BB) { LLVM_DEBUG({ @@ -168,7 +168,7 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA, std::vector> ToErase; bool Changed = false; for (BinaryBasicBlock &BB : BF) { - LLVM_DEBUG(dbgs() <<"\tNow at BB " << BB.getName() << "\n"); + LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n"); const MCInst *Prev = nullptr; for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) { MCInst &Inst = *I; @@ -206,7 +206,7 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA, LLVM_DEBUG(Inst.dump()); LLVM_DEBUG(dbgs() << "@BB: " << BB.getName() << "\n"); LLVM_DEBUG(dbgs() << "FIE offset = " << FIEX->StackOffset - << " size = " << (int)FIEX->Size << "\n"); + << " size = " << (int)FIEX->Size << "\n"); // Delete it! ToErase.emplace_back(&BB, &Inst); Prev = &Inst; @@ -242,8 +242,8 @@ void FrameOptimizerPass::runOnFunctions(BinaryContext &BC) { } { - NamedRegionTimer T1("reganalysis", "reg analysis", "FOP", - "FOP breakdown", opts::TimeOpts); + NamedRegionTimer T1("reganalysis", "reg analysis", "FOP", "FOP breakdown", + opts::TimeOpts); RA = std::make_unique(BC, &BC.getBinaryFunctions(), CG.get()); } @@ -278,7 +278,6 @@ void FrameOptimizerPass::runOnFunctions(BinaryContext &BC) { // Don't even start shrink wrapping if no profiling info is available if (I.second.getKnownExecutionCount() == 0) continue; - } { diff --git a/bolt/lib/Passes/HFSort.cpp b/bolt/lib/Passes/HFSort.cpp index cca94af9f944..6e57aaf30ae6 100644 --- a/bolt/lib/Passes/HFSort.cpp +++ b/bolt/lib/Passes/HFSort.cpp @@ -18,7 +18,6 @@ #include "llvm/Support/raw_ostream.h" #include -#undef DEBUG_TYPE #define DEBUG_TYPE "hfsort" namespace opts { @@ -47,14 +46,13 @@ constexpr double MinArcProbability = 0.1; // willing to degrade it's density by merging a callee. constexpr int CallerDegradeFactor = 8; -} +} // namespace //////////////////////////////////////////////////////////////////////////////// Cluster::Cluster(NodeId Id, const Node &Func) -: Samples(Func.samples()), - Size(Func.size()), - Density((double)Samples / Size) { + : Samples(Func.samples()), Size(Func.size()), + Density((double)Samples / Size) { Targets.push_back(Id); } @@ -75,7 +73,8 @@ std::string Cluster::toString() const { bool PrintComma = false; CS << "funcs = ["; for (const NodeId &Target : Targets) { - if (PrintComma) CS << ", "; + if (PrintComma) + CS << ", "; CS << Target; PrintComma = true; } @@ -90,33 +89,30 @@ void freezeClusters(const CallGraph &Cg, std::vector &Clusters) { std::sort(Clusters.begin(), Clusters.end(), compareClustersDensity); for (Cluster &C : Clusters) { uint32_t NewSize = TotalSize + C.size(); - if (NewSize > FrozenPages * HugePageSize) break; + if (NewSize > FrozenPages * HugePageSize) + break; C.freeze(); TotalSize = NewSize; - LLVM_DEBUG( - NodeId Fid = C.target(0); - dbgs() << - format("freezing cluster for func %d, size = %u, samples = %lu)\n", - Fid, Cg.size(Fid), Cg.samples(Fid));); + LLVM_DEBUG(NodeId Fid = C.target(0); + dbgs() << format( + "freezing cluster for func %d, size = %u, samples = %lu)\n", + Fid, Cg.size(Fid), Cg.samples(Fid));); } } -} +} // namespace -void Cluster::reverseTargets() { - std::reverse(Targets.begin(), Targets.end()); -} +void Cluster::reverseTargets() { std::reverse(Targets.begin(), Targets.end()); } void Cluster::merge(const Cluster &Other, const double Aw) { - Targets.insert(Targets.end(), - Other.Targets.begin(), - Other.Targets.end()); + Targets.insert(Targets.end(), Other.Targets.begin(), Other.Targets.end()); Size += Other.Size; Samples += Other.Samples; Density = (double)Samples / Size; } -void Cluster::merge(const Cluster &Other, const std::vector &Targets_) { +void Cluster::merge(const Cluster &Other, + const std::vector &Targets_) { Targets = Targets_; Size += Other.Size; Samples += Other.Samples; @@ -136,12 +132,13 @@ std::vector clusterize(const CallGraph &Cg) { std::vector SortedFuncs; // indexed by NodeId, keeps it's current cluster - std::vector FuncCluster(Cg.numNodes(), nullptr); + std::vector FuncCluster(Cg.numNodes(), nullptr); std::vector Clusters; Clusters.reserve(Cg.numNodes()); for (NodeId F = 0; F < Cg.numNodes(); F++) { - if (Cg.samples(F) == 0) continue; + if (Cg.samples(F) == 0) + continue; Clusters.emplace_back(F, Cg.getNode(F)); SortedFuncs.push_back(F); } @@ -154,23 +151,20 @@ std::vector clusterize(const CallGraph &Cg) { FuncCluster[Cluster.targets().front()] = &Cluster; } - std::sort( - SortedFuncs.begin(), - SortedFuncs.end(), - [&] (const NodeId F1, const NodeId F2) { - const CallGraph::Node &Func1 = Cg.getNode(F1); - const CallGraph::Node &Func2 = Cg.getNode(F2); - return - Func1.samples() * Func2.size() > // TODO: is this correct? - Func2.samples() * Func1.size(); - } - ); + std::sort(SortedFuncs.begin(), SortedFuncs.end(), + [&](const NodeId F1, const NodeId F2) { + const CallGraph::Node &Func1 = Cg.getNode(F1); + const CallGraph::Node &Func2 = Cg.getNode(F2); + return Func1.samples() * Func2.size() > // TODO: is this correct? + Func2.samples() * Func1.size(); + }); // Process each function, and consider merging its cluster with the // one containing its most likely predecessor. for (const NodeId Fid : SortedFuncs) { Cluster *Cluster = FuncCluster[Fid]; - if (Cluster->frozen()) continue; + if (Cluster->frozen()) + continue; // Find best predecessor. NodeId BestPred = CallGraph::InvalidId; @@ -178,7 +172,8 @@ std::vector clusterize(const CallGraph &Cg) { for (const NodeId Src : Cg.predecessors(Fid)) { const Arc &Arc = *Cg.findArc(Src, Fid); - if (BestPred == CallGraph::InvalidId || Arc.normalizedWeight() > BestProb) { + if (BestPred == CallGraph::InvalidId || + Arc.normalizedWeight() > BestProb) { BestPred = Arc.src(); BestProb = Arc.normalizedWeight(); } @@ -187,7 +182,8 @@ std::vector clusterize(const CallGraph &Cg) { // Check if the merge is good for the callee. // Don't merge if the probability of getting to the callee from the // caller is too low. - if (BestProb < MinArcProbability) continue; + if (BestProb < MinArcProbability) + continue; assert(BestPred != CallGraph::InvalidId); @@ -201,25 +197,24 @@ std::vector clusterize(const CallGraph &Cg) { } // Skip if merged cluster would be bigger than the threshold. - if (Cluster->size() + PredCluster->size() > MaxClusterSize) continue; + if (Cluster->size() + PredCluster->size() > MaxClusterSize) + continue; // Check if the merge is good for the caller. // Don't merge if the caller's density is significantly better // than the density resulting from the merge. const double NewDensity = - ((double)PredCluster->samples() + Cluster->samples()) / - (PredCluster->size() + Cluster->size()); + ((double)PredCluster->samples() + Cluster->samples()) / + (PredCluster->size() + Cluster->size()); if (PredCluster->density() > NewDensity * CallerDegradeFactor) { continue; } - LLVM_DEBUG( - if (opts::Verbosity > 1) { - dbgs() << format("merging %s -> %s: %u\n", - PredCluster->toString().c_str(), - Cluster->toString().c_str(), - Cg.samples(Fid)); - }); + LLVM_DEBUG(if (opts::Verbosity > 1) { + dbgs() << format("merging %s -> %s: %u\n", + PredCluster->toString().c_str(), + Cluster->toString().c_str(), Cg.samples(Fid)); + }); for (NodeId F : Cluster->targets()) { FuncCluster[F] = PredCluster; @@ -235,17 +230,14 @@ std::vector clusterize(const CallGraph &Cg) { std::unordered_set Visited; for (const NodeId Func : SortedFuncs) { Cluster *Cluster = FuncCluster[Func]; - if (!Cluster || - Visited.count(Cluster) == 1 || - Cluster->target(0) != Func) { + if (!Cluster || Visited.count(Cluster) == 1 || Cluster->target(0) != Func) { continue; } SortedClusters.emplace_back(std::move(*Cluster)); Visited.insert(Cluster); } - std::sort(SortedClusters.begin(), - SortedClusters.end(), + std::sort(SortedClusters.begin(), SortedClusters.end(), compareClustersDensity); return SortedClusters; @@ -257,15 +249,14 @@ std::vector randomClusters(const CallGraph &Cg) { Clusters.reserve(Cg.numNodes()); for (NodeId F = 0; F < Cg.numNodes(); F++) { - if (Cg.samples(F) == 0) continue; + if (Cg.samples(F) == 0) + continue; Clusters.emplace_back(F, Cg.getNode(F)); } - std::sort(Clusters.begin(), - Clusters.end(), - [](const Cluster &A, const Cluster &B) { - return A.size() < B.size(); - }); + std::sort( + Clusters.begin(), Clusters.end(), + [](const Cluster &A, const Cluster &B) { return A.size() < B.size(); }); auto pickMergeCluster = [&Clusters](const size_t Idx) { size_t MaxIdx = Idx + 1; @@ -277,7 +268,8 @@ std::vector randomClusters(const CallGraph &Cg) { if (MaxIdx - Idx > 1) { size_t MergeIdx = (std::rand() % (MaxIdx - Idx - 1)) + Idx + 1; - assert(Clusters[MergeIdx].size() + Clusters[Idx].size() <= MaxClusterSize); + assert(Clusters[MergeIdx].size() + Clusters[Idx].size() <= + MaxClusterSize); return MergeIdx; } return Clusters.size(); @@ -297,5 +289,5 @@ std::vector randomClusters(const CallGraph &Cg) { return Clusters; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/HFSortPlus.cpp b/bolt/lib/Passes/HFSortPlus.cpp index 57a0a756f96e..7abbd3b4c12f 100644 --- a/bolt/lib/Passes/HFSortPlus.cpp +++ b/bolt/lib/Passes/HFSortPlus.cpp @@ -28,7 +28,6 @@ #include #include -#undef DEBUG_TYPE #define DEBUG_TYPE "hfsort" using namespace llvm; diff --git a/bolt/lib/Passes/IdenticalCodeFolding.cpp b/bolt/lib/Passes/IdenticalCodeFolding.cpp index fb0fc63f6f3a..d05c11d4e905 100644 --- a/bolt/lib/Passes/IdenticalCodeFolding.cpp +++ b/bolt/lib/Passes/IdenticalCodeFolding.cpp @@ -47,8 +47,7 @@ using JumpTable = bolt::JumpTable; /// Compare two jump tables in 2 functions. The function relies on consistent /// ordering of basic blocks in both binary functions (e.g. DFS). -bool equalJumpTables(const JumpTable &JumpTableA, - const JumpTable &JumpTableB, +bool equalJumpTables(const JumpTable &JumpTableA, const JumpTable &JumpTableB, const BinaryFunction &FunctionA, const BinaryFunction &FunctionB) { if (JumpTableA.EntrySize != JumpTableB.EntrySize) @@ -143,7 +142,6 @@ bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA, return BC.MIB->equals(InstA, InstB, Comp); } - /// Returns true if this function has identical code and CFG with /// the given function \p BF. /// @@ -199,8 +197,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B, auto OtherI = OtherBB->begin(), OtherE = OtherBB->end(); while (I != E && OtherI != OtherE) { // Compare symbols. - auto AreSymbolsIdentical = [&] (const MCSymbol *SymbolA, - const MCSymbol *SymbolB) { + auto AreSymbolsIdentical = [&](const MCSymbol *SymbolA, + const MCSymbol *SymbolB) { if (SymbolA == SymbolB) return true; @@ -269,12 +267,13 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B, return equalJumpTables(*JumpTableA, *JumpTableB, A, B); }; - if(!isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB, - AreSymbolsIdentical)) { + if (!isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB, + AreSymbolsIdentical)) { return false; } - ++I; ++OtherI; + ++I; + ++OtherI; } // One of the identical blocks may have a trailing unconditional jump that @@ -301,9 +300,7 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B, // This hash table is used to identify identical functions. It maps // a function to a bucket of functions identical to it. struct KeyHash { - size_t operator()(const BinaryFunction *F) const { - return F->getHash(); - } + size_t operator()(const BinaryFunction *F) const { return F->getHash(); } }; /// Identify two congruent functions. Two functions are considered congruent, @@ -429,10 +426,9 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) { // Pre-compute hash before pushing into hashtable. // Hash instruction operands to minimize hash collisions. - BF.computeHash(opts::UseDFS, - [&BC] (const MCOperand &Op) { - return hashInstOperand(BC, Op); - }); + BF.computeHash(opts::UseDFS, [&BC](const MCOperand &Op) { + return hashInstOperand(BC, Op); + }); }; ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) { @@ -578,12 +574,12 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) { }); if (NumFunctionsFolded) { - outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded - << " out of " << OriginalFunctionCount << " functions in " - << Iteration << " passes. " - << NumJTFunctionsFolded << " functions had jump tables.\n" + outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded << " out of " + << OriginalFunctionCount << " functions in " << Iteration + << " passes. " << NumJTFunctionsFolded + << " functions had jump tables.\n" << "BOLT-INFO: Removing all identical functions will save " - << format("%.2lf", (double) BytesSavedEstimate / 1024) + << format("%.2lf", (double)BytesSavedEstimate / 1024) << " KB of code space. Folded functions were called " << CallsSavedEstimate << " times based on profile.\n"; } diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp index 7e39b92320e5..97a554bdc9b3 100644 --- a/bolt/lib/Passes/IndirectCallPromotion.cpp +++ b/bolt/lib/Passes/IndirectCallPromotion.cpp @@ -15,7 +15,10 @@ #include "llvm/Support/CommandLine.h" #define DEBUG_TYPE "ICP" -#define DEBUG_VERBOSE(Level, X) if (opts::Verbosity >= (Level)) { X; } +#define DEBUG_VERBOSE(Level, X) \ + if (opts::Verbosity >= (Level)) { \ + X; \ + } using namespace llvm; using namespace bolt; @@ -28,127 +31,87 @@ extern cl::opt IndirectCallPromotion; extern cl::opt Verbosity; extern cl::opt ExecutionCountThreshold; -static cl::opt -ICPJTRemainingPercentThreshold( +static cl::opt ICPJTRemainingPercentThreshold( "icp-jt-remaining-percent-threshold", cl::desc("The percentage threshold against remaining unpromoted indirect " "call count for the promotion for jump tables"), - cl::init(30), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltOptCategory)); + cl::init(30), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory)); -static cl::opt -ICPJTTotalPercentThreshold( +static cl::opt ICPJTTotalPercentThreshold( "icp-jt-total-percent-threshold", - cl::desc("The percentage threshold against total count for the promotion for " - "jump tables"), - cl::init(5), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltOptCategory)); + cl::desc( + "The percentage threshold against total count for the promotion for " + "jump tables"), + cl::init(5), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory)); -static cl::opt -ICPCallsRemainingPercentThreshold( +static cl::opt ICPCallsRemainingPercentThreshold( "icp-calls-remaining-percent-threshold", cl::desc("The percentage threshold against remaining unpromoted indirect " "call count for the promotion for calls"), - cl::init(50), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltOptCategory)); + cl::init(50), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory)); -static cl::opt -ICPCallsTotalPercentThreshold( +static cl::opt ICPCallsTotalPercentThreshold( "icp-calls-total-percent-threshold", - cl::desc("The percentage threshold against total count for the promotion for " - "calls"), - cl::init(30), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltOptCategory)); + cl::desc( + "The percentage threshold against total count for the promotion for " + "calls"), + cl::init(30), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory)); -static cl::opt -IndirectCallPromotionMispredictThreshold( +static cl::opt IndirectCallPromotionMispredictThreshold( "indirect-call-promotion-mispredict-threshold", cl::desc("misprediction threshold for skipping ICP on an " "indirect call"), - cl::init(0), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(0), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -IndirectCallPromotionUseMispredicts( +static cl::opt IndirectCallPromotionUseMispredicts( "indirect-call-promotion-use-mispredicts", cl::desc("use misprediction frequency for determining whether or not ICP " "should be applied at a callsite. The " "-indirect-call-promotion-mispredict-threshold value will be used " "by this heuristic"), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -IndirectCallPromotionTopN( +static cl::opt IndirectCallPromotionTopN( "indirect-call-promotion-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion. 0 = no limit"), - cl::init(3), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(3), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -IndirectCallPromotionCallsTopN( +static cl::opt IndirectCallPromotionCallsTopN( "indirect-call-promotion-calls-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion on calls. 0 = no limit"), - cl::init(0), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(0), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -IndirectCallPromotionJumpTablesTopN( +static cl::opt IndirectCallPromotionJumpTablesTopN( "indirect-call-promotion-jump-tables-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion on jump tables. 0 = no limit"), - cl::init(0), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(0), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -EliminateLoads( +static cl::opt EliminateLoads( "icp-eliminate-loads", cl::desc("enable load elimination using memory profiling data when " "performing ICP"), - cl::init(true), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(true), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -static cl::opt -ICPTopCallsites( +static cl::opt ICPTopCallsites( "icp-top-callsites", cl::desc("optimize hottest calls until at least this percentage of all " "indirect calls frequency is covered. 0 = all callsites"), - cl::init(99), - cl::Hidden, - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); + cl::init(99), cl::Hidden, cl::ZeroOrMore, cl::cat(BoltOptCategory)); static cl::list -ICPFuncsList("icp-funcs", - cl::CommaSeparated, - cl::desc("list of functions to enable ICP for"), - cl::value_desc("func1,func2,func3,..."), - cl::Hidden, - cl::cat(BoltOptCategory)); + ICPFuncsList("icp-funcs", cl::CommaSeparated, + cl::desc("list of functions to enable ICP for"), + cl::value_desc("func1,func2,func3,..."), cl::Hidden, + cl::cat(BoltOptCategory)); static cl::opt -ICPOldCodeSequence( - "icp-old-code-sequence", - cl::desc("use old code sequence for promoted calls"), - cl::init(false), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltOptCategory)); + ICPOldCodeSequence("icp-old-code-sequence", + cl::desc("use old code sequence for promoted calls"), + cl::init(false), cl::ZeroOrMore, cl::Hidden, + cl::cat(BoltOptCategory)); static cl::opt ICPJumpTablesByTarget( "icp-jump-tables-targets", @@ -167,7 +130,8 @@ bool verifyProfile(std::map &BFs) { bool IsValid = true; for (auto &BFI : BFs) { BinaryFunction &BF = BFI.second; - if (!BF.isSimple()) continue; + if (!BF.isSimple()) + continue; for (BinaryBasicBlock *BB : BF.layout()) { auto BI = BB->branch_info_begin(); for (BinaryBasicBlock *SuccBB : BB->successors()) { @@ -175,7 +139,8 @@ bool verifyProfile(std::map &BFs) { if (BB->getKnownExecutionCount() == 0 || SuccBB->getKnownExecutionCount() == 0) { errs() << "BOLT-WARNING: profile verification failed after ICP for " - "function " << BF << '\n'; + "function " + << BF << '\n'; IsValid = false; } } @@ -186,14 +151,12 @@ bool verifyProfile(std::map &BFs) { return IsValid; } -} +} // namespace IndirectCallPromotion::Callsite::Callsite(BinaryFunction &BF, const IndirectCallProfile &ICP) - : From(BF.getSymbol()), - To(ICP.Offset), - Mispreds(ICP.Mispreds), - Branches(ICP.Count) { + : From(BF.getSymbol()), To(ICP.Offset), Mispreds(ICP.Mispreds), + Branches(ICP.Count) { if (ICP.Symbol) { To.Sym = ICP.Symbol; To.Addr = 0; @@ -269,8 +232,8 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB, continue; const Location To(Entry); const BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(Entry); - Targets.emplace_back( - From, To, BI.MispredictedCount, BI.Count, I - Range.first); + Targets.emplace_back(From, To, BI.MispredictedCount, BI.Count, + I - Range.first); } // Sort by symbol then addr. @@ -317,8 +280,8 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB, Inst.getOperand(0).getReg() == BC.MRI->getProgramCounter()) { return Targets; } - auto ICSP = - BC.MIB->tryGetAnnotationAs(Inst, "CallProfile"); + auto ICSP = BC.MIB->tryGetAnnotationAs( + Inst, "CallProfile"); if (ICSP) { for (const IndirectCallProfile &CSP : ICSP.get()) { Callsite Site(BF, CSP); @@ -342,37 +305,35 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB, }); // Remove non-symbol targets - auto Last = std::remove_if(Targets.begin(), - Targets.end(), - [](const Callsite &CS) { - return !CS.To.Sym; - }); + auto Last = std::remove_if(Targets.begin(), Targets.end(), + [](const Callsite &CS) { return !CS.To.Sym; }); Targets.erase(Last, Targets.end()); - LLVM_DEBUG( - if (BF.getJumpTable(Inst)) { - uint64_t TotalCount = 0; - uint64_t TotalMispreds = 0; - for (const Callsite &S : Targets) { - TotalCount += S.Branches; - TotalMispreds += S.Mispreds; - } - if (!TotalCount) TotalCount = 1; - if (!TotalMispreds) TotalMispreds = 1; + LLVM_DEBUG(if (BF.getJumpTable(Inst)) { + uint64_t TotalCount = 0; + uint64_t TotalMispreds = 0; + for (const Callsite &S : Targets) { + TotalCount += S.Branches; + TotalMispreds += S.Mispreds; + } + if (!TotalCount) + TotalCount = 1; + if (!TotalMispreds) + TotalMispreds = 1; - dbgs() << "BOLT-INFO: ICP: jump table size = " << Targets.size() - << ", Count = " << TotalCount - << ", Mispreds = " << TotalMispreds << "\n"; + dbgs() << "BOLT-INFO: ICP: jump table size = " << Targets.size() + << ", Count = " << TotalCount << ", Mispreds = " << TotalMispreds + << "\n"; - size_t I = 0; - for (const Callsite &S : Targets) { - dbgs () << "Count[" << I << "] = " << S.Branches << ", " - << format("%.1f", (100.0*S.Branches)/TotalCount) << ", " - << "Mispreds[" << I << "] = " << S.Mispreds << ", " - << format("%.1f", (100.0*S.Mispreds)/TotalMispreds) << "\n"; - ++I; - } - }); + size_t I = 0; + for (const Callsite &S : Targets) { + dbgs() << "Count[" << I << "] = " << S.Branches << ", " + << format("%.1f", (100.0 * S.Branches) / TotalCount) << ", " + << "Mispreds[" << I << "] = " << S.Mispreds << ", " + << format("%.1f", (100.0 * S.Mispreds) / TotalMispreds) << "\n"; + ++I; + } + }); return Targets; } @@ -417,23 +378,24 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB, }); DEBUG_VERBOSE(1, { - dbgs() << "Jmp info: Type = " << (unsigned)Type << ", " - << "BaseReg = " << BC.MRI->getName(BaseReg) << ", " - << "IndexReg = " << BC.MRI->getName(IndexReg) << ", " - << "DispValue = " << Twine::utohexstr(DispValue) << ", " - << "DispExpr = " << DispExpr << ", " - << "MemLocInstr = "; - BC.printInstruction(dbgs(), *MemLocInstr, 0, &Function); - dbgs() << "\n"; - }); + dbgs() << "Jmp info: Type = " << (unsigned)Type << ", " + << "BaseReg = " << BC.MRI->getName(BaseReg) << ", " + << "IndexReg = " << BC.MRI->getName(IndexReg) << ", " + << "DispValue = " << Twine::utohexstr(DispValue) << ", " + << "DispExpr = " << DispExpr << ", " + << "MemLocInstr = "; + BC.printInstruction(dbgs(), *MemLocInstr, 0, &Function); + dbgs() << "\n"; + }); ++TotalIndexBasedCandidates; auto ErrorOrMemAccesssProfile = - BC.MIB->tryGetAnnotationAs(*MemLocInstr, - "MemoryAccessProfile"); + BC.MIB->tryGetAnnotationAs(*MemLocInstr, + "MemoryAccessProfile"); if (!ErrorOrMemAccesssProfile) { - DEBUG_VERBOSE(1, dbgs() << "BOLT-INFO: ICP no memory profiling data found\n"); + DEBUG_VERBOSE(1, dbgs() + << "BOLT-INFO: ICP no memory profiling data found\n"); return JumpTableInfoType(); } MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get(); @@ -469,11 +431,11 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB, if (AccessInfo.MemoryObject) { // Deal with bad/stale data - if (!AccessInfo.MemoryObject->getName(). - startswith("JUMP_TABLE/" + Function.getOneName().str())) + if (!AccessInfo.MemoryObject->getName().startswith( + "JUMP_TABLE/" + Function.getOneName().str())) return JumpTableInfoType(); Index = - (AccessInfo.Offset - (ArrayStart - JT->getAddress())) / JT->EntrySize; + (AccessInfo.Offset - (ArrayStart - JT->getAddress())) / JT->EntrySize; } else { Index = (AccessInfo.Offset - ArrayStart) / JT->EntrySize; } @@ -490,18 +452,18 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB, // e.g. not the end of function (unreachable) label. if (!Function.getBasicBlockForLabel(JT->Entries[Index + Range.first])) { LLVM_DEBUG({ - dbgs() << "BOLT-INFO: hot index " << Index << " pointing at bogus " - << "label " << JT->Entries[Index + Range.first]->getName() - << " in jump table:\n"; - JT->print(dbgs()); - dbgs() << "HotTargetMap:\n"; - for (std::pair> &HT : - HotTargetMap) { - dbgs() << "BOLT-INFO: " << HT.first->getName() - << " = (count=" << HT.first << ", index=" << HT.second - << ")\n"; - } - }); + dbgs() << "BOLT-INFO: hot index " << Index << " pointing at bogus " + << "label " << JT->Entries[Index + Range.first]->getName() + << " in jump table:\n"; + JT->print(dbgs()); + dbgs() << "HotTargetMap:\n"; + for (std::pair> &HT : + HotTargetMap) { + dbgs() << "BOLT-INFO: " << HT.first->getName() + << " = (count=" << HT.first << ", index=" << HT.second + << ")\n"; + } + }); return JumpTableInfoType(); } @@ -512,23 +474,21 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB, } std::transform( - HotTargetMap.begin(), - HotTargetMap.end(), - std::back_inserter(HotTargets), - [](const std::pair> &A) { - return A.second; - }); + HotTargetMap.begin(), HotTargetMap.end(), std::back_inserter(HotTargets), + [](const std::pair> &A) { + return A.second; + }); // Sort with highest counts first. std::sort(HotTargets.rbegin(), HotTargets.rend()); LLVM_DEBUG({ - dbgs() << "BOLT-INFO: ICP jump table hot targets:\n"; - for (const std::pair &Target : HotTargets) { - dbgs() << "BOLT-INFO: Idx = " << Target.second << ", " - << "Count = " << Target.first << "\n"; - } - }); + dbgs() << "BOLT-INFO: ICP jump table hot targets:\n"; + for (const std::pair &Target : HotTargets) { + dbgs() << "BOLT-INFO: Idx = " << Target.second << ", " + << "Count = " << Target.first << "\n"; + } + }); BC.MIB->getOrCreateAnnotationAs(CallInst, "JTIndexReg") = IndexReg; @@ -566,7 +526,8 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector &Targets, if (opts::Verbosity >= 1) { for (size_t I = 0; I < HotTargets.size(); ++I) { outs() << "BOLT-INFO: HotTarget[" << I << "] = (" - << HotTargets[I].first << ", " << HotTargets[I].second << ")\n"; + << HotTargets[I].first << ", " << HotTargets[I].second + << ")\n"; } } @@ -644,8 +605,7 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector &Targets, } } else { for (size_t I = 0; I < N; ++I) { - assert(Targets[I].To.Sym && - "All ICP targets must be to known symbols"); + assert(Targets[I].To.Sym && "All ICP targets must be to known symbols"); assert(Targets[I].JTIndices.empty() && "Can't have jump table indices for non-jump tables"); SymTargets.emplace_back(Targets[I].To.Sym, 0); @@ -672,11 +632,8 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms( return MethodInfoType(); MutableArrayRef Insts(&BB.front(), &Inst + 1); - if (!BC.MIB->analyzeVirtualMethodCall(Insts.begin(), - Insts.end(), - MethodFetchInsns, - VtableReg, - MethodReg, + if (!BC.MIB->analyzeVirtualMethodCall(Insts.begin(), Insts.end(), + MethodFetchInsns, VtableReg, MethodReg, MethodOffset)) { DEBUG_VERBOSE( 1, dbgs() << "BOLT-INFO: ICP unable to analyze method call in " @@ -700,11 +657,11 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms( // Try to get value profiling data for the method load instruction. auto ErrorOrMemAccesssProfile = - BC.MIB->tryGetAnnotationAs(*MethodFetchInsns.back(), - "MemoryAccessProfile"); + BC.MIB->tryGetAnnotationAs(*MethodFetchInsns.back(), + "MemoryAccessProfile"); if (!ErrorOrMemAccesssProfile) { - DEBUG_VERBOSE(1, - dbgs() << "BOLT-INFO: ICP no memory profiling data found\n"); + DEBUG_VERBOSE(1, dbgs() + << "BOLT-INFO: ICP no memory profiling data found\n"); return MethodInfoType(); } MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get(); @@ -725,13 +682,12 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms( const uint64_t VtableBase = Address - MethodOffset; DEBUG_VERBOSE(1, dbgs() << "BOLT-INFO: ICP vtable = " - << Twine::utohexstr(VtableBase) - << "+" << MethodOffset << "/" << AccessInfo.Count - << "\n"); + << Twine::utohexstr(VtableBase) << "+" + << MethodOffset << "/" << AccessInfo.Count << "\n"); if (ErrorOr MethodAddr = BC.getPointerAtAddress(Address)) { BinaryData *MethodBD = BC.getBinaryDataAtAddress(MethodAddr.get()); - if (!MethodBD) // skip unknown methods + if (!MethodBD) // skip unknown methods continue; MCSymbol *MethodSym = MethodBD->getSymbol(); MethodToVtable[MethodSym] = VtableBase; @@ -868,12 +824,12 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, for (size_t I = 0; I < NumEntries; ++I) { BBI.push_back( BinaryBranchInfo{(Target.Branches + NumEntries - 1) / NumEntries, - (Target.Mispreds + NumEntries - 1) / NumEntries}); - ScaledBBI.push_back(BinaryBranchInfo{ - uint64_t(TotalCount * Target.Branches / - (NumEntries * TotalIndirectBranches)), - uint64_t(TotalCount * Target.Mispreds / - (NumEntries * TotalIndirectBranches))}); + (Target.Mispreds + NumEntries - 1) / NumEntries}); + ScaledBBI.push_back( + BinaryBranchInfo{uint64_t(TotalCount * Target.Branches / + (NumEntries * TotalIndirectBranches)), + uint64_t(TotalCount * Target.Mispreds / + (NumEntries * TotalIndirectBranches))}); } } @@ -881,7 +837,7 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, BinaryBasicBlock *NewIndCallBlock = NewBBs.back().get(); IndCallBlock.moveAllSuccessorsTo(NewIndCallBlock); - std::vector SymTargets; + std::vector SymTargets; for (const Callsite &Target : Targets) { const size_t NumEntries = std::max(static_cast(1UL), Target.JTIndices.size()); @@ -922,7 +878,7 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, assert(NewBBs.size() % 2 == 1 || IsTailCall); auto ScaledBI = ScaledBBI.begin(); - auto updateCurrentBranchInfo = [&]{ + auto updateCurrentBranchInfo = [&] { assert(ScaledBI != ScaledBBI.end()); TotalCount -= ScaledBI->Count; ++ScaledBI; @@ -945,13 +901,13 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, uint64_t ExecCount = ScaledBBI[(I + 1) / 2].Count; if (I % 2 == 0) { if (MergeBlock) { - NewBBs[I]->addSuccessor(MergeBlock, ScaledBBI[(I+1)/2].Count); + NewBBs[I]->addSuccessor(MergeBlock, ScaledBBI[(I + 1) / 2].Count); } } else { assert(I + 2 < NewBBs.size()); updateCurrentBranchInfo(); - NewBBs[I]->addSuccessor(NewBBs[I+2].get(), TotalCount); - NewBBs[I]->addSuccessor(NewBBs[I+1].get(), ScaledBBI[(I+1)/2]); + NewBBs[I]->addSuccessor(NewBBs[I + 2].get(), TotalCount); + NewBBs[I]->addSuccessor(NewBBs[I + 1].get(), ScaledBBI[(I + 1) / 2]); ExecCount += TotalCount; } NewBBs[I]->setExecutionCount(ExecCount); @@ -1090,7 +1046,7 @@ size_t IndirectCallPromotion::canPromoteCallsite( // this frequency is less than the threshold, we should skip ICP at // this callsite. const double TopNMispredictFrequency = - (100.0 * TotalMispredictsTopN) / NumCalls; + (100.0 * TotalMispredictsTopN) / NumCalls; if (TopNMispredictFrequency < opts::IndirectCallPromotionMispredictThreshold) { @@ -1165,12 +1121,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { auto &BFs = BC.getBinaryFunctions(); - const bool OptimizeCalls = - (opts::IndirectCallPromotion == ICP_CALLS || - opts::IndirectCallPromotion == ICP_ALL); + const bool OptimizeCalls = (opts::IndirectCallPromotion == ICP_CALLS || + opts::IndirectCallPromotion == ICP_ALL); const bool OptimizeJumpTables = - (opts::IndirectCallPromotion == ICP_JUMP_TABLES || - opts::IndirectCallPromotion == ICP_ALL); + (opts::IndirectCallPromotion == ICP_JUMP_TABLES || + opts::IndirectCallPromotion == ICP_ALL); std::unique_ptr RA; std::unique_ptr CG; @@ -1209,9 +1164,9 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { for (MCInst &Inst : BB) { const bool IsJumpTable = Function.getJumpTable(Inst); const bool HasIndirectCallProfile = - BC.MIB->hasAnnotation(Inst, "CallProfile"); - const bool IsDirectCall = (BC.MIB->isCall(Inst) && - BC.MIB->getTargetSymbol(Inst, 0)); + BC.MIB->hasAnnotation(Inst, "CallProfile"); + const bool IsDirectCall = + (BC.MIB->isCall(Inst) && BC.MIB->getTargetSymbol(Inst, 0)); if (!IsDirectCall && ((HasIndirectCallProfile && !IsJumpTable && OptimizeCalls) || @@ -1288,7 +1243,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { const auto InstIdx = &Inst - &(*BB->begin()); const bool IsTailCall = BC.MIB->isTailCall(Inst); const bool HasIndirectCallProfile = - BC.MIB->hasAnnotation(Inst, "CallProfile"); + BC.MIB->hasAnnotation(Inst, "CallProfile"); const bool IsJumpTable = Function.getJumpTable(Inst); if (BC.MIB->isCall(Inst)) { @@ -1305,8 +1260,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { if (BC.MIB->isCall(Inst) && BC.MIB->getTargetSymbol(Inst, 0)) continue; - assert((BC.MIB->isCall(Inst) || BC.MIB->isIndirectBranch(Inst)) - && "expected a call or an indirect jump instruction"); + assert((BC.MIB->isCall(Inst) || BC.MIB->isIndirectBranch(Inst)) && + "expected a call or an indirect jump instruction"); if (IsJumpTable) ++TotalJumpTableCallsites; @@ -1422,10 +1377,9 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { for (const auto &entry : ICPcode) { const MCSymbol *const &Sym = entry.first; const InstructionListType &Insts = entry.second; - if (Sym) dbgs() << Sym->getName() << ":\n"; - Offset = BC.printInstructions(dbgs(), - Insts.begin(), - Insts.end(), + if (Sym) + dbgs() << Sym->getName() << ":\n"; + Offset = BC.printInstructions(dbgs(), Insts.begin(), Insts.end(), Offset); } dbgs() << "---------------------------------------------------\n"; @@ -1447,9 +1401,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { } if (opts::Verbosity >= 1) { - outs() << "BOLT-INFO: ICP succeeded in " - << Function << " @ " << InstIdx - << " in " << BB->getName() + outs() << "BOLT-INFO: ICP succeeded in " << Function << " @ " + << InstIdx << " in " << BB->getName() << " -> calls = " << NumCalls << "\n"; } @@ -1466,52 +1419,46 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { } outs() << "BOLT-INFO: ICP total indirect callsites with profile = " - << TotalIndirectCallsites - << "\n" + << TotalIndirectCallsites << "\n" << "BOLT-INFO: ICP total jump table callsites = " - << TotalJumpTableCallsites - << "\n" - << "BOLT-INFO: ICP total number of calls = " - << TotalCalls - << "\n" + << TotalJumpTableCallsites << "\n" + << "BOLT-INFO: ICP total number of calls = " << TotalCalls << "\n" << "BOLT-INFO: ICP percentage of calls that are indirect = " - << format("%.1f", (100.0 * TotalIndirectCalls) / TotalCalls) - << "%\n" + << format("%.1f", (100.0 * TotalIndirectCalls) / TotalCalls) << "%\n" << "BOLT-INFO: ICP percentage of indirect calls that can be " "optimized = " << format("%.1f", (100.0 * TotalNumFrequentCalls) / - std::max(TotalIndirectCalls, 1)) + std::max(TotalIndirectCalls, 1)) << "%\n" << "BOLT-INFO: ICP percentage of indirect callsites that are " "optimized = " << format("%.1f", (100.0 * TotalOptimizedIndirectCallsites) / - std::max(TotalIndirectCallsites, 1)) + std::max(TotalIndirectCallsites, 1)) << "%\n" << "BOLT-INFO: ICP number of method load elimination candidates = " - << TotalMethodLoadEliminationCandidates - << "\n" + << TotalMethodLoadEliminationCandidates << "\n" << "BOLT-INFO: ICP percentage of method calls candidates that have " "loads eliminated = " << format("%.1f", (100.0 * TotalMethodLoadsEliminated) / - std::max(TotalMethodLoadEliminationCandidates, 1)) + std::max( + TotalMethodLoadEliminationCandidates, 1)) << "%\n" << "BOLT-INFO: ICP percentage of indirect branches that are " "optimized = " << format("%.1f", (100.0 * TotalNumFrequentJmps) / - std::max(TotalIndirectJmps, 1)) + std::max(TotalIndirectJmps, 1)) << "%\n" << "BOLT-INFO: ICP percentage of jump table callsites that are " << "optimized = " << format("%.1f", (100.0 * TotalOptimizedJumpTableCallsites) / - std::max(TotalJumpTableCallsites, 1)) + std::max(TotalJumpTableCallsites, 1)) << "%\n" << "BOLT-INFO: ICP number of jump table callsites that can use hot " - << "indices = " << TotalIndexBasedCandidates - << "\n" + << "indices = " << TotalIndexBasedCandidates << "\n" << "BOLT-INFO: ICP percentage of jump table callsites that use hot " "indices = " << format("%.1f", (100.0 * TotalIndexBasedJumps) / - std::max(TotalIndexBasedCandidates, 1)) + std::max(TotalIndexBasedCandidates, 1)) << "%\n"; (void)verifyProfile; diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp index 163ef55d62c4..64cecb99e593 100644 --- a/bolt/lib/Passes/Inliner.cpp +++ b/bolt/lib/Passes/Inliner.cpp @@ -117,9 +117,7 @@ NoInline("no-inline", /// this function should reflect the change. bool inliningEnabled() { return !NoInline && - (InlineAll || - InlineSmallFunctions || - !ForceInlineFunctions.empty()); + (InlineAll || InlineSmallFunctions || !ForceInlineFunctions.empty()); } bool mustConsider(const llvm::bolt::BinaryFunction &Function) { @@ -237,8 +235,7 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const { // For a regular call the last return instruction could be removed // (or converted to a branch). const MCInst *LastInst = BF.back().getLastNonPseudoInstr(); - if (LastInst && - BC.MIB->isReturn(*LastInst) && + if (LastInst && BC.MIB->isReturn(*LastInst) && !BC.MIB->isTailCall(*LastInst)) { const uint64_t RetInstSize = BC.computeInstructionSize(*LastInst); assert(Size >= RetInstSize); @@ -249,8 +246,7 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const { return Info; } -void -Inliner::findInliningCandidates(BinaryContext &BC) { +void Inliner::findInliningCandidates(BinaryContext &BC) { for (const auto &BFI : BC.getBinaryFunctions()) { const BinaryFunction &Function = BFI.second; const InliningInfo InlInfo = getInliningInfo(Function); @@ -307,7 +303,7 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB, double ProfileRatio = 0; if (uint64_t CalleeExecCount = Callee.getKnownExecutionCount()) { ProfileRatio = - (double) FirstInlinedBB->getKnownExecutionCount() / CalleeExecCount; + (double)FirstInlinedBB->getKnownExecutionCount() / CalleeExecCount; } // Save execution count of the first block as we don't want it to change @@ -357,8 +353,8 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB, } if (CSIsTailCall || (!MIB.isCall(Inst) && !MIB.isReturn(Inst))) { - InsertII = std::next(InlinedBB->insertInstruction(InsertII, - std::move(Inst))); + InsertII = + std::next(InlinedBB->insertInstruction(InsertII, std::move(Inst))); continue; } @@ -377,30 +373,22 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB, MIB.addGnuArgsSize(Inst, CSGNUArgsSize); } - InsertII = std::next(InlinedBB->insertInstruction(InsertII, - std::move(Inst))); + InsertII = + std::next(InlinedBB->insertInstruction(InsertII, std::move(Inst))); } // Add CFG edges to the basic blocks of the inlined instance. std::vector Successors(BB.succ_size()); - std::transform( - BB.succ_begin(), - BB.succ_end(), - Successors.begin(), - [&InlinedBBMap](const BinaryBasicBlock *BB) { - return InlinedBBMap.at(BB); - }); + std::transform(BB.succ_begin(), BB.succ_end(), Successors.begin(), + [&InlinedBBMap](const BinaryBasicBlock *BB) { + return InlinedBBMap.at(BB); + }); if (CallerFunction.hasValidProfile() && Callee.hasValidProfile()) { - InlinedBB->addSuccessors( - Successors.begin(), - Successors.end(), - BB.branch_info_begin(), - BB.branch_info_end()); + InlinedBB->addSuccessors(Successors.begin(), Successors.end(), + BB.branch_info_begin(), BB.branch_info_end()); } else { - InlinedBB->addSuccessors( - Successors.begin(), - Successors.end()); + InlinedBB->addSuccessors(Successors.begin(), Successors.end()); } if (!CSIsTailCall && BB.succ_size() == 0 && NextBB) { @@ -413,8 +401,8 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB, if (opts::AdjustProfile) { InlinedBB->adjustExecutionCount(ProfileRatio); } else { - InlinedBB->setExecutionCount( - InlinedBB->getKnownExecutionCount() * ProfileRatio); + InlinedBB->setExecutionCount(InlinedBB->getKnownExecutionCount() * + ProfileRatio); } } } @@ -438,13 +426,14 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) { std::vector Blocks(Function.layout().begin(), Function.layout().end()); std::sort(Blocks.begin(), Blocks.end(), - [](const BinaryBasicBlock *BB1, const BinaryBasicBlock *BB2) { - return BB1->getKnownExecutionCount() > BB2->getKnownExecutionCount(); - }); + [](const BinaryBasicBlock *BB1, const BinaryBasicBlock *BB2) { + return BB1->getKnownExecutionCount() > + BB2->getKnownExecutionCount(); + }); bool DidInlining = false; for (BinaryBasicBlock *BB : Blocks) { - for (auto InstIt = BB->begin(); InstIt != BB->end(); ) { + for (auto InstIt = BB->begin(); InstIt != BB->end();) { MCInst &Inst = *InstIt; if (!BC.MIB->isCall(Inst) || MCPlus::getNumPrimeOperands(Inst) != 1 || !Inst.getOperand(0).isExpr()) { @@ -484,11 +473,11 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) { int64_t SizeAfterInlining; if (IsTailCall) { - SizeAfterInlining = IInfo->second.SizeAfterTailCallInlining - - getSizeOfTailCallInst(BC); + SizeAfterInlining = + IInfo->second.SizeAfterTailCallInlining - getSizeOfTailCallInst(BC); } else { - SizeAfterInlining = IInfo->second.SizeAfterInlining - - getSizeOfCallInst(BC); + SizeAfterInlining = + IInfo->second.SizeAfterInlining - getSizeOfCallInst(BC); } if (!opts::InlineAll && !opts::mustConsider(*TargetFunction)) { @@ -567,9 +556,10 @@ void Inliner::runOnFunctions(BinaryContext &BC) { ConsideredFunctions.push_back(&Function); } std::sort(ConsideredFunctions.begin(), ConsideredFunctions.end(), - [](const BinaryFunction *A, const BinaryFunction *B) { - return B->getKnownExecutionCount() < A->getKnownExecutionCount(); - }); + [](const BinaryFunction *A, const BinaryFunction *B) { + return B->getKnownExecutionCount() < + A->getKnownExecutionCount(); + }); for (BinaryFunction *Function : ConsideredFunctions) { if (opts::InlineLimit && NumInlinedCallSites >= opts::InlineLimit) break; diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp index 48d16725be33..7d8679e70fb1 100644 --- a/bolt/lib/Passes/Instrumentation.cpp +++ b/bolt/lib/Passes/Instrumentation.cpp @@ -37,9 +37,7 @@ cl::opt InstrumentationBinpath( cl::opt InstrumentationFileAppendPID( "instrumentation-file-append-pid", cl::desc("append PID to saved profile file name (default: false)"), - cl::init(false), - cl::Optional, - cl::cat(BoltInstrCategory)); + cl::init(false), cl::Optional, cl::cat(BoltInstrCategory)); cl::opt ConservativeInstrumentation( "conservative-instrumentation", @@ -79,7 +77,7 @@ cl::opt InstrumentCalls("instrument-calls", "control flow activity (default: true)"), cl::init(true), cl::Optional, cl::cat(BoltInstrCategory)); -} +} // namespace opts namespace llvm { namespace bolt { @@ -136,12 +134,12 @@ void Instrumentation::createIndCallTargetDescription( Summary->IndCallTargetDescriptions.emplace_back(ICD); } -bool Instrumentation::createEdgeDescription( - FunctionDescription &FuncDesc, - const BinaryFunction &FromFunction, uint32_t From, - uint32_t FromNodeID, - const BinaryFunction &ToFunction, uint32_t To, - uint32_t ToNodeID, bool Instrumented) { +bool Instrumentation::createEdgeDescription(FunctionDescription &FuncDesc, + const BinaryFunction &FromFunction, + uint32_t From, uint32_t FromNodeID, + const BinaryFunction &ToFunction, + uint32_t To, uint32_t ToNodeID, + bool Instrumented) { EdgeDescription ED; auto Result = FuncDesc.EdgesSet.insert(std::make_pair(FromNodeID, ToNodeID)); // Avoid creating duplicated edge descriptions. This happens in CFGs where a @@ -193,7 +191,7 @@ BinaryBasicBlock::iterator insertInstructions(InstructionListType &Instrs, } return Iter; } -} +} // namespace void Instrumentation::instrumentLeafNode(BinaryBasicBlock &BB, BinaryBasicBlock::iterator Iter, @@ -519,10 +517,9 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) { BC.registerOrUpdateSection(".bolt.instr.counters", ELF::SHT_PROGBITS, Flags, nullptr, 0, 1); - BC.registerOrUpdateNoteSection(".bolt.instr.tables", nullptr, - 0, - /*Alignment=*/1, - /*IsReadOnly=*/true, ELF::SHT_NOTE); + BC.registerOrUpdateNoteSection(".bolt.instr.tables", nullptr, 0, + /*Alignment=*/1, + /*IsReadOnly=*/true, ELF::SHT_NOTE); Summary->IndCallCounterFuncPtr = BC.Ctx->getOrCreateSymbol("__bolt_ind_call_counter_func_pointer"); @@ -580,8 +577,9 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) { MCSymbol *Target = BC.registerNameAtAddress( "__bolt_instr_fini", FiniSection->getAddress(), 0, 0); auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); }; - const auto LEA = std::find_if(std::next(std::find_if( - BB.rbegin(), BB.rend(), IsLEA)), BB.rend(), IsLEA); + const auto LEA = + std::find_if(std::next(std::find_if(BB.rbegin(), BB.rend(), IsLEA)), + BB.rend(), IsLEA); LEA->getOperand(4).setExpr( MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *BC.Ctx)); } else { @@ -708,5 +706,5 @@ void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) { assert(RtLibrary && "instrumentation runtime library object must be set"); RtLibrary->setSummary(std::move(Summary)); } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/JTFootprintReduction.cpp b/bolt/lib/Passes/JTFootprintReduction.cpp index 5b3598566865..e060ee5bde3c 100644 --- a/bolt/lib/Passes/JTFootprintReduction.cpp +++ b/bolt/lib/Passes/JTFootprintReduction.cpp @@ -26,13 +26,11 @@ extern cl::opt Verbosity; extern cl::opt JumpTables; -static cl::opt -JTFootprintOnlyPIC("jt-footprint-optimize-for-icache", - cl::desc("with jt-footprint-reduction, only process PIC jumptables and turn" - " off other transformations that increase code size"), - cl::init(false), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); +static cl::opt JTFootprintOnlyPIC( + "jt-footprint-optimize-for-icache", + cl::desc("with jt-footprint-reduction, only process PIC jumptables and turn" + " off other transformations that increase code size"), + cl::init(false), cl::ZeroOrMore, cl::cat(BoltOptCategory)); } // namespace opts @@ -128,9 +126,8 @@ void JTFootprintReduction::checkOpportunities(BinaryFunction &Function, } bool JTFootprintReduction::tryOptimizeNonPIC( - BinaryContext &BC, BinaryBasicBlock &BB, - BinaryBasicBlock::iterator Inst, uint64_t JTAddr, - JumpTable *JumpTable, DataflowInfoManager &Info) { + BinaryContext &BC, BinaryBasicBlock &BB, BinaryBasicBlock::iterator Inst, + uint64_t JTAddr, JumpTable *JumpTable, DataflowInfoManager &Info) { if (opts::JTFootprintOnlyPIC) return false; @@ -169,10 +166,11 @@ bool JTFootprintReduction::tryOptimizeNonPIC( return true; } -bool JTFootprintReduction::tryOptimizePIC( - BinaryContext &BC, BinaryBasicBlock &BB, - BinaryBasicBlock::iterator Inst, uint64_t JTAddr, - JumpTable *JumpTable, DataflowInfoManager &Info) { +bool JTFootprintReduction::tryOptimizePIC(BinaryContext &BC, + BinaryBasicBlock &BB, + BinaryBasicBlock::iterator Inst, + uint64_t JTAddr, JumpTable *JumpTable, + DataflowInfoManager &Info) { MCPhysReg BaseReg; uint64_t Scale; MCPhysReg Index; @@ -184,9 +182,9 @@ bool JTFootprintReduction::tryOptimizePIC( BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg), BC.MIB->matchImm(Scale), BC.MIB->matchReg(Index), BC.MIB->matchAnyOperand()))); - if (!PICIndJmpMatcher->match(*BC.MRI, *BC.MIB, - MutableArrayRef(&*BB.begin(), &*Inst + 1), - -1)) { + if (!PICIndJmpMatcher->match( + *BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &*Inst + 1), + -1)) { return false; } @@ -228,8 +226,8 @@ void JTFootprintReduction::optimizeFunction(BinaryFunction &Function, if (BlacklistedJTs.count(JumpTable)) continue; - if (tryOptimizeNonPIC(BC, BB, IndJmp, JTAddr, JumpTable, Info) - || tryOptimizePIC(BC, BB, IndJmp, JTAddr, JumpTable, Info)) { + if (tryOptimizeNonPIC(BC, BB, IndJmp, JTAddr, JumpTable, Info) || + tryOptimizePIC(BC, BB, IndJmp, JTAddr, JumpTable, Info)) { Modified.insert(&Function); continue; } @@ -241,7 +239,7 @@ void JTFootprintReduction::optimizeFunction(BinaryFunction &Function, return; for (BinaryBasicBlock &BB : Function) { - for (auto I = BB.begin(); I != BB.end(); ) { + for (auto I = BB.begin(); I != BB.end();) { if (BC.MIB->hasAnnotation(*I, "DeleteMe")) I = BB.eraseInstruction(I); else @@ -293,8 +291,7 @@ void JTFootprintReduction::runOnFunctions(BinaryContext &BC) { << " JTs discarded due to unsupported jump pattern.\n"; outs() << "\t " << NumJTsNoReg << " JTs discarded due to register unavailability.\n"; - outs() << "\t " << BytesSaved - << " bytes saved.\n"; + outs() << "\t " << BytesSaved << " bytes saved.\n"; } } // namespace bolt diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp index d6f126e1ea1d..cabcae326790 100644 --- a/bolt/lib/Passes/LongJmp.cpp +++ b/bolt/lib/Passes/LongJmp.cpp @@ -138,8 +138,7 @@ BinaryBasicBlock *LongJmpPass::lookupStubFromGroup( if (Cand != Candidates.begin()) { const StubTy *LeftCand = Cand; --LeftCand; - if (Cand->first - DotAddress > - DotAddress - LeftCand->first) + if (Cand->first - DotAddress > DotAddress - LeftCand->first) Cand = LeftCand; } int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1; @@ -164,8 +163,7 @@ LongJmpPass::lookupGlobalStub(const BinaryBasicBlock &SourceBB, const BinaryFunction &Func = *SourceBB.getFunction(); const StubGroupsTy &StubGroups = SourceBB.isCold() ? ColdStubGroups : HotStubGroups; - return lookupStubFromGroup(StubGroups, Func, Inst, TgtSym, - DotAddress); + return lookupStubFromGroup(StubGroups, Func, Inst, TgtSym, DotAddress); } BinaryBasicBlock *LongJmpPass::lookupLocalStub(const BinaryBasicBlock &SourceBB, @@ -189,7 +187,7 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst, const BinaryContext &BC = Func.getBinaryContext(); std::unique_ptr NewBB; const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst); - assert (TgtSym && "getTargetSymbol failed"); + assert(TgtSym && "getTargetSymbol failed"); BinaryBasicBlock::BinaryBranchInfo BI{0, 0}; BinaryBasicBlock *TgtBB = BB.getSuccessor(TgtSym, BI); @@ -239,7 +237,7 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst, StubBB->addSuccessor(TgtBB, OrigCount, OrigMispreds); StubBB->setIsCold(BB.isCold()); } - // Call / tail call + // Call / tail call } else { StubBB->setExecutionCount(StubBB->getExecutionCount() + BB.getExecutionCount()); @@ -295,8 +293,8 @@ void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) { } uint64_t LongJmpPass::tentativeLayoutRelocColdPart( - const BinaryContext &BC, std::vector &SortedFunctions, - uint64_t DotAddress) { + const BinaryContext &BC, std::vector &SortedFunctions, + uint64_t DotAddress) { for (BinaryFunction *Func : SortedFunctions) { if (!Func->isSplit()) continue; @@ -315,8 +313,8 @@ uint64_t LongJmpPass::tentativeLayoutRelocColdPart( } uint64_t LongJmpPass::tentativeLayoutRelocMode( - const BinaryContext &BC, std::vector &SortedFunctions, - uint64_t DotAddress) { + const BinaryContext &BC, std::vector &SortedFunctions, + uint64_t DotAddress) { // Compute hot cold frontier uint32_t LastHotIndex = -1u; @@ -372,8 +370,7 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode( } void LongJmpPass::tentativeLayout( - const BinaryContext &BC, - std::vector &SortedFunctions) { + const BinaryContext &BC, std::vector &SortedFunctions) { uint64_t DotAddress = BC.LayoutStartAddress; if (!BC.HasRelocations) { @@ -421,7 +418,7 @@ uint64_t LongJmpPass::getSymbolAddress(const BinaryContext &BC, const BinaryBasicBlock *TgtBB) const { if (TgtBB) { auto Iter = BBAddresses.find(TgtBB); - assert (Iter != BBAddresses.end() && "Unrecognized BB"); + assert(Iter != BBAddresses.end() && "Unrecognized BB"); return Iter->second; } uint64_t EntryID = 0; @@ -456,7 +453,7 @@ bool LongJmpPass::relaxStub(BinaryBasicBlock &StubBB) { uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB); uint64_t DotAddress = BBAddresses[&StubBB]; uint64_t PCRelTgtAddress = DotAddress > TgtAddress ? DotAddress - TgtAddress - : TgtAddress - DotAddress; + : TgtAddress - DotAddress; // If it fits in one instruction, do not relax if (!(PCRelTgtAddress & SingleInstrMask)) return false; @@ -495,7 +492,7 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst, const BinaryFunction &Func = *BB.getFunction(); const BinaryContext &BC = Func.getBinaryContext(); const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst); - assert (TgtSym && "getTargetSymbol failed"); + assert(TgtSym && "getTargetSymbol failed"); const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(TgtSym); // Check for shared stubs from foreign functions @@ -630,5 +627,5 @@ void LongJmpPass::runOnFunctions(BinaryContext &BC) { << " stubs in the cold area. Shared " << NumSharedStubs << " times, iterated " << Iterations << " times.\n"; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/MCF.cpp b/bolt/lib/Passes/MCF.cpp index 7aed526d3f6a..f478d6c873a0 100644 --- a/bolt/lib/Passes/MCF.cpp +++ b/bolt/lib/Passes/MCF.cpp @@ -93,17 +93,19 @@ void updateEdgeWeight(EdgeWeightMap &EdgeWeights, const BinaryBasicBlock *A, const BinaryBasicBlock *B, double Weight); template <> -void updateEdgeWeight( - EdgeWeightMap &EdgeWeights, const BinaryBasicBlock *A, - const BinaryBasicBlock *B, double Weight) { +void updateEdgeWeight(EdgeWeightMap &EdgeWeights, + const BinaryBasicBlock *A, + const BinaryBasicBlock *B, + double Weight) { EdgeWeights[std::make_pair(A, B)] = Weight; return; } template <> -void updateEdgeWeight>( - EdgeWeightMap &EdgeWeights, const BinaryBasicBlock *A, - const BinaryBasicBlock *B, double Weight) { +void updateEdgeWeight>(EdgeWeightMap &EdgeWeights, + const BinaryBasicBlock *A, + const BinaryBasicBlock *B, + double Weight) { EdgeWeights[std::make_pair(B, A)] = Weight; return; } @@ -111,14 +113,15 @@ void updateEdgeWeight>( template void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) { typedef GraphTraits GraphT; - typedef GraphTraits > InvTraits; + typedef GraphTraits> InvTraits; double TotalChildrenCount = 0.0; SmallVector ChildrenExecCount; // First pass computes total children execution count that directly // contribute to this BB. for (typename GraphT::ChildIteratorType CI = GraphT::child_begin(BB), - E = GraphT::child_end(BB); CI != E; ++CI) { + E = GraphT::child_end(BB); + CI != E; ++CI) { typename GraphT::NodeRef Child = *CI; double ChildExecCount = Child->getExecutionCount(); // Is self-reference? @@ -152,7 +155,8 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) { // Second pass fixes the weight of a possible self-reference edge uint32_t ChildIndex = 0; for (typename GraphT::ChildIteratorType CI = GraphT::child_begin(BB), - E = GraphT::child_end(BB); CI != E; ++CI) { + E = GraphT::child_end(BB); + CI != E; ++CI) { typename GraphT::NodeRef Child = *CI; if (Child != BB) { ++ChildIndex; @@ -168,7 +172,8 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) { // Third pass finally assigns weights to edges ChildIndex = 0; for (typename GraphT::ChildIteratorType CI = GraphT::child_begin(BB), - E = GraphT::child_end(BB); CI != E; ++CI) { + E = GraphT::child_end(BB); + CI != E; ++CI) { typename GraphT::NodeRef Child = *CI; double Weight = 1 / (GraphT::child_end(BB) - GraphT::child_begin(BB)); if (TotalChildrenCount != 0.0) @@ -178,7 +183,7 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) { } } -template +template void computeEdgeWeights(BinaryFunction &BF, EdgeWeightMap &EdgeWeights) { for (BinaryBasicBlock &BB : BF) { computeEdgeWeights(&BB, EdgeWeights); @@ -333,8 +338,10 @@ void guessEdgeByIterativeApproach(BinaryFunction &BF) { do { Changed = false; for (BinaryBasicBlock &BB : BF) { - if (guessPredEdgeCounts(&BB, KnownArcs)) Changed = true; - if (guessSuccEdgeCounts(&BB, KnownArcs)) Changed = true; + if (guessPredEdgeCounts(&BB, KnownArcs)) + Changed = true; + if (guessSuccEdgeCounts(&BB, KnownArcs)) + Changed = true; } } while (Changed); @@ -345,7 +352,7 @@ void guessEdgeByIterativeApproach(BinaryFunction &BF) { continue; BinaryBasicBlock::BinaryBranchInfo &BI = Pred->getBranchInfo(BB); BI.Count = - std::min(Pred->getExecutionCount(), BB.getExecutionCount()) / 2; + std::min(Pred->getExecutionCount(), BB.getExecutionCount()) / 2; KnownArcs.insert(std::make_pair(Pred, &BB)); } auto BI = BB.branch_info_begin(); @@ -364,9 +371,9 @@ void guessEdgeByIterativeApproach(BinaryFunction &BF) { /// Associate each basic block with the BinaryLoop object corresponding to the /// innermost loop containing this block. -DenseMap +DenseMap createLoopNestLevelMap(BinaryFunction &BF) { - DenseMap LoopNestLevel; + DenseMap LoopNestLevel; const BinaryLoopInfo &BLI = BF.getLoopInfo(); for (BinaryBasicBlock &BB : BF) { @@ -417,7 +424,7 @@ void equalizeBBCounts(BinaryFunction &BF) { return; if (LoopNestLevel[&BB] != LoopNestLevel[DomBB]) return; - if (BBsToEC[DomBB] == -1 && BBsToEC[&BB] == -1) { + if (BBsToEC[DomBB] == -1 && BBsToEC[&BB] == -1) { BBsToEC[DomBB] = Classes.size(); BBsToEC[&BB] = Classes.size(); Classes.emplace_back(); @@ -483,5 +490,5 @@ void solveMCF(BinaryFunction &BF, MCFCostFunction CostFunction) { llvm_unreachable("not implemented"); } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/PLTCall.cpp b/bolt/lib/Passes/PLTCall.cpp index 4fd44ba04014..0cda28041ffe 100644 --- a/bolt/lib/Passes/PLTCall.cpp +++ b/bolt/lib/Passes/PLTCall.cpp @@ -69,8 +69,7 @@ void PLTCall::runOnFunctions(BinaryContext &BC) { const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CallSymbol); if (!CalleeBF || !CalleeBF->isPLTFunction()) continue; - BC.MIB->convertCallToIndirectCall(Instr, - CalleeBF->getPLTSymbol(), + BC.MIB->convertCallToIndirectCall(Instr, CalleeBF->getPLTSymbol(), BC.Ctx.get()); BC.MIB->addAnnotation(Instr, "PLTCall", true); ++NumCallsOptimized; @@ -85,6 +84,5 @@ void PLTCall::runOnFunctions(BinaryContext &BC) { } } - } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Passes/PatchEntries.cpp b/bolt/lib/Passes/PatchEntries.cpp index 21e4c09bda06..974a9e2df9b5 100644 --- a/bolt/lib/Passes/PatchEntries.cpp +++ b/bolt/lib/Passes/PatchEntries.cpp @@ -50,9 +50,8 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { return; } - if (opts::Verbosity >= 1) { + if (opts::Verbosity >= 1) outs() << "BOLT-INFO: patching entries in original code\n"; - } // Calculate the size of the patch. static size_t PatchSize = 0; @@ -71,9 +70,8 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { // Check if we can skip patching the function. if (!opts::ForcePatch && !Function.hasEHRanges() && - Function.getSize() < PatchThreshold) { + Function.getSize() < PatchThreshold) continue; - } // List of patches for function entries. We either successfully patch // all entries or, if we cannot patch one or more, do no patch any and @@ -84,10 +82,9 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { bool Success = Function.forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Symbol) { if (Offset < NextValidByte) { - if (opts::Verbosity >= 1) { + if (opts::Verbosity >= 1) outs() << "BOLT-INFO: unable to patch entry point in " << Function << " at offset 0x" << Twine::utohexstr(Offset) << '\n'; - } return false; } @@ -96,10 +93,9 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { Function.getOriginSection()}); NextValidByte = Offset + PatchSize; if (NextValidByte > Function.getMaxSize()) { - if (opts::Verbosity >= 1) { - outs() << "BOLT-INFO: function " << Function << " too small to patch " - "its entry point\n"; - } + if (opts::Verbosity >= 1) + outs() << "BOLT-INFO: function " << Function + << " too small to patch its entry point\n"; return false; } @@ -116,9 +112,8 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { } for (Patch &Patch : PendingPatches) { - BinaryFunction *PatchFunction = - BC.createInjectedBinaryFunction( - NameResolver::append(Patch.Symbol->getName(), ".org.0")); + BinaryFunction *PatchFunction = BC.createInjectedBinaryFunction( + NameResolver::append(Patch.Symbol->getName(), ".org.0")); // Force the function to be emitted at the given address. PatchFunction->setOutputAddress(Patch.Address); PatchFunction->setFileOffset(Patch.FileOffset); diff --git a/bolt/lib/Passes/PettisAndHansen.cpp b/bolt/lib/Passes/PettisAndHansen.cpp index fe4bba6e8c6a..1d9afd451969 100644 --- a/bolt/lib/Passes/PettisAndHansen.cpp +++ b/bolt/lib/Passes/PettisAndHansen.cpp @@ -15,7 +15,6 @@ #include #include -#undef DEBUG_TYPE #define DEBUG_TYPE "hfsort" namespace llvm { @@ -29,10 +28,7 @@ namespace { class ClusterArc { public: ClusterArc(Cluster *Ca, Cluster *Cb, double W = 0) - : C1(std::min(Ca, Cb)) - , C2(std::max(Ca, Cb)) - , Weight(W) - {} + : C1(std::min(Ca, Cb)), C2(std::max(Ca, Cb)), Weight(W) {} friend bool operator==(const ClusterArc &Lhs, const ClusterArc &Rhs) { return Lhs.C1 == Rhs.C1 && Lhs.C2 == Rhs.C2; @@ -95,18 +91,19 @@ void orderFuncs(const CallGraph &Cg, Cluster *C1, Cluster *C2) { C2->reverseTargets(); } } -} +} // namespace std::vector pettisAndHansen(const CallGraph &Cg) { // indexed by NodeId, keeps its current cluster - std::vector FuncCluster(Cg.numNodes(), nullptr); + std::vector FuncCluster(Cg.numNodes(), nullptr); std::vector Clusters; std::vector Funcs; Clusters.reserve(Cg.numNodes()); for (NodeId F = 0; F < Cg.numNodes(); F++) { - if (Cg.samples(F) == 0) continue; + if (Cg.samples(F) == 0) + continue; Clusters.emplace_back(F, Cg.getNode(F)); FuncCluster[F] = &Clusters.back(); Funcs.push_back(F); @@ -124,18 +121,21 @@ std::vector pettisAndHansen(const CallGraph &Cg) { // Create a std::vector of cluster arcs for (const Arc &Arc : Cg.arcs()) { - if (Arc.weight() == 0) continue; + if (Arc.weight() == 0) + continue; Cluster *const S = FuncCluster[Arc.src()]; Cluster *const D = FuncCluster[Arc.dst()]; // ignore if s or d is nullptr - if (S == nullptr || D == nullptr) continue; + if (S == nullptr || D == nullptr) + continue; // ignore self-edges - if (S == D) continue; + if (S == D) + continue; insertOrInc(S, D, Arc.weight()); } @@ -143,13 +143,11 @@ std::vector pettisAndHansen(const CallGraph &Cg) { // Find an arc with max weight and merge its nodes while (!Carcs.empty()) { - auto Maxpos = std::max_element( - Carcs.begin(), - Carcs.end(), - [&] (const ClusterArc &Carc1, const ClusterArc &Carc2) { - return Carc1.Weight < Carc2.Weight; - } - ); + auto Maxpos = + std::max_element(Carcs.begin(), Carcs.end(), + [&](const ClusterArc &Carc1, const ClusterArc &Carc2) { + return Carc1.Weight < Carc2.Weight; + }); ClusterArc Max = *Maxpos; Carcs.erase(Maxpos); @@ -157,9 +155,11 @@ std::vector pettisAndHansen(const CallGraph &Cg) { Cluster *const C1 = Max.C1; Cluster *const C2 = Max.C2; - if (C1->size() + C2->size() > MaxClusterSize) continue; + if (C1->size() + C2->size() > MaxClusterSize) + continue; - if (C1->frozen() || C2->frozen()) continue; + if (C1->frozen() || C2->frozen()) + continue; // order functions and merge cluster @@ -173,8 +173,10 @@ std::vector pettisAndHansen(const CallGraph &Cg) { std::unordered_map C2arcs; for (const ClusterArc &Carc : Carcs) { - if (Carc.C1 == C2) C2arcs.emplace(Carc, Carc.C2); - if (Carc.C2 == C2) C2arcs.emplace(Carc, Carc.C1); + if (Carc.C1 == C2) + C2arcs.emplace(Carc, Carc.C2); + if (Carc.C2 == C2) + C2arcs.emplace(Carc, Carc.C1); } for (auto It : C2arcs) { @@ -197,7 +199,7 @@ std::vector pettisAndHansen(const CallGraph &Cg) { // Return the set of Clusters that are left, which are the ones that // didn't get merged. - std::set LiveClusters; + std::set LiveClusters; std::vector OutClusters; for (NodeId Fid : Funcs) { @@ -207,12 +209,10 @@ std::vector pettisAndHansen(const CallGraph &Cg) { OutClusters.push_back(std::move(*C)); } - std::sort(OutClusters.begin(), - OutClusters.end(), - compareClustersDensity); + std::sort(OutClusters.begin(), OutClusters.end(), compareClustersDensity); return OutClusters; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/RegAnalysis.cpp b/bolt/lib/Passes/RegAnalysis.cpp index 6f28d48c94e6..88e2ff030b18 100644 --- a/bolt/lib/Passes/RegAnalysis.cpp +++ b/bolt/lib/Passes/RegAnalysis.cpp @@ -203,7 +203,7 @@ BitVector RegAnalysis::getFunctionUsedRegsList(const BinaryFunction *Func) { for (const BinaryBasicBlock &BB : *Func) { for (const MCInst &Inst : BB) { - getInstUsedRegsList(Inst, UsedRegs, /*GetClobbers*/false); + getInstUsedRegsList(Inst, UsedRegs, /*GetClobbers*/ false); if (UsedRegs.all()) return UsedRegs; } @@ -239,5 +239,5 @@ void RegAnalysis::printStats() { (100.0 * CountFunctionsAllClobber / CountDenominator)); } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/RegReAssign.cpp b/bolt/lib/Passes/RegReAssign.cpp index 88a4aacdec55..69489c903dd3 100644 --- a/bolt/lib/Passes/RegReAssign.cpp +++ b/bolt/lib/Passes/RegReAssign.cpp @@ -359,7 +359,7 @@ bool RegReAssign::conservativePassOverFunction(BinaryFunction &Function) { } void RegReAssign::setupAggressivePass(BinaryContext &BC, - std::map &BFs) { + std::map &BFs) { setupConservativePass(BC, BFs); CG.reset(new BinaryFunctionCallGraph(buildCallGraph(BC))); RA.reset(new RegAnalysis(BC, &BFs, &*CG)); @@ -446,5 +446,5 @@ void RegReAssign::runOnFunctions(BinaryContext &BC) { outs() << "\t " << DynBytesSaved << " dynamic bytes saved.\n"; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/ReorderAlgorithm.cpp b/bolt/lib/Passes/ReorderAlgorithm.cpp index 1718caed0c08..2d441b3584dc 100644 --- a/bolt/lib/Passes/ReorderAlgorithm.cpp +++ b/bolt/lib/Passes/ReorderAlgorithm.cpp @@ -56,15 +56,13 @@ RandomSeed("bolt-seed", namespace { -template -inline void hashCombine(size_t &Seed, const T &Val) { +template inline void hashCombine(size_t &Seed, const T &Val) { std::hash Hasher; Seed ^= Hasher(Val) + 0x9e3779b9 + (Seed << 6) + (Seed >> 2); } -template -struct HashPair { - size_t operator()(const std::pair& Val) const { +template struct HashPair { + size_t operator()(const std::pair &Val) const { std::hash Hasher; size_t Seed = Hasher(Val.first); hashCombine(Seed, Val.second); @@ -72,7 +70,7 @@ struct HashPair { } }; -} +} // namespace void ClusterAlgorithm::computeClusterAverageFrequency(const BinaryContext &BC) { // Create a separate MCCodeEmitter to allow lock-free execution @@ -127,8 +125,8 @@ size_t GreedyClusterAlgorithm::EdgeHash::operator()(const EdgeTy &E) const { return Hasher(std::make_pair(E.Src, E.Dst)); } -bool GreedyClusterAlgorithm::EdgeEqual::operator()( - const EdgeTy &A, const EdgeTy &B) const { +bool GreedyClusterAlgorithm::EdgeEqual::operator()(const EdgeTy &A, + const EdgeTy &B) const { return A.Src == B.Src && A.Dst == B.Dst; } @@ -237,18 +235,18 @@ void GreedyClusterAlgorithm::reset() { BBToClusterMap.clear(); } -void PHGreedyClusterAlgorithm::initQueue( - std::vector &Queue, const BinaryFunction &BF) { +void PHGreedyClusterAlgorithm::initQueue(std::vector &Queue, + const BinaryFunction &BF) { // Define a comparison function to establish SWO between edges. - auto Comp = [&BF] (const EdgeTy &A, const EdgeTy &B) { + auto Comp = [&BF](const EdgeTy &A, const EdgeTy &B) { // With equal weights, prioritize branches with lower index // source/destination. This helps to keep original block order for blocks // when optimal order cannot be deducted from a profile. if (A.Count == B.Count) { const signed SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src); return (SrcOrder != 0) - ? SrcOrder > 0 - : BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0; + ? SrcOrder > 0 + : BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0; } return A.Count < B.Count; }; @@ -257,14 +255,15 @@ void PHGreedyClusterAlgorithm::initQueue( std::sort(Queue.begin(), Queue.end(), Comp); } -void PHGreedyClusterAlgorithm::adjustQueue( - std::vector &Queue, const BinaryFunction &BF) { +void PHGreedyClusterAlgorithm::adjustQueue(std::vector &Queue, + const BinaryFunction &BF) { // Nothing to do. return; } -bool PHGreedyClusterAlgorithm::areClustersCompatible( - const ClusterTy &Front, const ClusterTy &Back, const EdgeTy &E) const { +bool PHGreedyClusterAlgorithm::areClustersCompatible(const ClusterTy &Front, + const ClusterTy &Back, + const EdgeTy &E) const { return Front.back() == E.Src && Back.front() == E.Dst; } @@ -314,8 +313,8 @@ int64_t MinBranchGreedyClusterAlgorithm::calculateWeight( return W; } -void MinBranchGreedyClusterAlgorithm::initQueue( - std::vector &Queue, const BinaryFunction &BF) { +void MinBranchGreedyClusterAlgorithm::initQueue(std::vector &Queue, + const BinaryFunction &BF) { // Initialize edge weights. for (const EdgeTy &E : Queue) Weight.emplace(std::make_pair(E, calculateWeight(E, BF))); @@ -324,18 +323,18 @@ void MinBranchGreedyClusterAlgorithm::initQueue( adjustQueue(Queue, BF); } -void MinBranchGreedyClusterAlgorithm::adjustQueue( - std::vector &Queue, const BinaryFunction &BF) { +void MinBranchGreedyClusterAlgorithm::adjustQueue(std::vector &Queue, + const BinaryFunction &BF) { // Define a comparison function to establish SWO between edges. - auto Comp = [&] (const EdgeTy &A, const EdgeTy &B) { + auto Comp = [&](const EdgeTy &A, const EdgeTy &B) { // With equal weights, prioritize branches with lower index // source/destination. This helps to keep original block order for blocks // when optimal order cannot be deduced from a profile. if (Weight[A] == Weight[B]) { const signed SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src); return (SrcOrder != 0) - ? SrcOrder > 0 - : BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0; + ? SrcOrder > 0 + : BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0; } return Weight[A] < Weight[B]; }; @@ -407,8 +406,8 @@ void MinBranchGreedyClusterAlgorithm::reset() { Weight.clear(); } -void TSPReorderAlgorithm::reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const { +void TSPReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const { std::vector> Weight; std::vector IndexToBB; @@ -523,7 +522,7 @@ void OptimizeReorderAlgorithm::reorderBasicBlocks( // Arrange basic blocks according to clusters. for (ClusterAlgorithm::ClusterTy &Cluster : CAlgo->Clusters) - Order.insert(Order.end(), Cluster.begin(), Cluster.end()); + Order.insert(Order.end(), Cluster.begin(), Cluster.end()); } void OptimizeBranchReorderAlgorithm::reorderBasicBlocks( @@ -532,7 +531,7 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks( return; // Cluster basic blocks. - CAlgo->clusterBasicBlocks(BF, /* ComputeEdges = */true); + CAlgo->clusterBasicBlocks(BF, /* ComputeEdges = */ true); std::vector &Clusters = CAlgo->Clusters; std::vector> &ClusterEdges = CAlgo->ClusterEdges; @@ -567,7 +566,8 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks( return ClusterEdges[I][A] > ClusterEdges[I][B]; }; std::priority_queue, - decltype(ClusterComp)> SuccQueue(ClusterComp); + decltype(ClusterComp)> + SuccQueue(ClusterComp); for (std::pair &Target : ClusterEdges[I]) { if (Target.second > 0 && !(Status[Target.first] & STACKED) && !Clusters[Target.first].empty()) { @@ -626,12 +626,12 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks( // Arrange basic blocks according to cluster order. for (uint32_t ClusterIndex : ClusterOrder) { ClusterAlgorithm::ClusterTy &Cluster = Clusters[ClusterIndex]; - Order.insert(Order.end(), Cluster.begin(), Cluster.end()); + Order.insert(Order.end(), Cluster.begin(), Cluster.end()); } } void OptimizeCacheReorderAlgorithm::reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const { + const BinaryFunction &BF, BasicBlockOrder &Order) const { if (BF.layout_empty()) return; @@ -657,11 +657,9 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks( if (!Clusters[I].empty()) ClusterOrder.push_back(I); // Don't reorder the first cluster, which contains the function entry point - std::stable_sort(std::next(ClusterOrder.begin()), - ClusterOrder.end(), - [&AvgFreq](uint32_t A, uint32_t B) { - return AvgFreq[A] > AvgFreq[B]; - }); + std::stable_sort( + std::next(ClusterOrder.begin()), ClusterOrder.end(), + [&AvgFreq](uint32_t A, uint32_t B) { return AvgFreq[A] > AvgFreq[B]; }); if (opts::PrintClusters) { errs() << "New cluster order: "; @@ -676,7 +674,7 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks( // Arrange basic blocks according to cluster order. for (uint32_t ClusterIndex : ClusterOrder) { ClusterAlgorithm::ClusterTy &Cluster = Clusters[ClusterIndex]; - Order.insert(Order.end(), Cluster.begin(), Cluster.end()); + Order.insert(Order.end(), Cluster.begin(), Cluster.end()); // Force zero execution count on clusters that do not meet the cut off // specified by --cold-threshold. if (AvgFreq[ClusterIndex] < static_cast(ColdThreshold)) { @@ -687,8 +685,8 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks( } } -void ReverseReorderAlgorithm::reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const { +void ReverseReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF, + BasicBlockOrder &Order) const { if (BF.layout_empty()) return; @@ -699,7 +697,7 @@ void ReverseReorderAlgorithm::reorderBasicBlocks( } void RandomClusterReorderAlgorithm::reorderBasicBlocks( - const BinaryFunction &BF, BasicBlockOrder &Order) const { + const BinaryFunction &BF, BasicBlockOrder &Order) const { if (BF.layout_empty()) return; @@ -734,6 +732,6 @@ void RandomClusterReorderAlgorithm::reorderBasicBlocks( // Arrange basic blocks according to cluster order. for (uint32_t ClusterIndex : ClusterOrder) { ClusterAlgorithm::ClusterTy &Cluster = Clusters[ClusterIndex]; - Order.insert(Order.end(), Cluster.begin(), Cluster.end()); + Order.insert(Order.end(), Cluster.begin(), Cluster.end()); } } diff --git a/bolt/lib/Passes/ReorderData.cpp b/bolt/lib/Passes/ReorderData.cpp index eaff155aa3cf..6d180de850d3 100644 --- a/bolt/lib/Passes/ReorderData.cpp +++ b/bolt/lib/Passes/ReorderData.cpp @@ -9,7 +9,8 @@ //===----------------------------------------------------------------------===// // TODO: -// - make sure writeable data isn't put on same cache line unless temporally local +// - make sure writeable data isn't put on same cache line unless temporally +// local // - estimate temporal locality by looking at CFG? #include "bolt/Passes/ReorderData.h" @@ -105,9 +106,7 @@ namespace { static constexpr uint16_t MinAlignment = 16; -bool isSupported(const BinarySection &BS) { - return BS.isData() && !BS.isTLS(); -} +bool isSupported(const BinarySection &BS) { return BS.isData() && !BS.isTLS(); } bool filterSymbol(const BinaryData *BD) { if (!BD->isAtomic() || BD->isJumpTable() || !BD->isMoveable()) @@ -140,7 +139,7 @@ bool filterSymbol(const BinaryData *BD) { return IsValid; } -} +} // namespace using DataOrder = ReorderData::DataOrder; @@ -153,13 +152,13 @@ void ReorderData::printOrder(const BinarySection &Section, const BinaryData *BD = Begin->first; if (!PrintHeader) { - outs() << "BOLT-INFO: Hot global symbols for " - << Section.getName() << ":\n"; + outs() << "BOLT-INFO: Hot global symbols for " << Section.getName() + << ":\n"; PrintHeader = true; } outs() << "BOLT-INFO: " << *BD << ", moveable=" << BD->isMoveable() - << format(", weight=%.5f\n", double(Begin->second)/BD->getSize()); + << format(", weight=%.5f\n", double(Begin->second) / BD->getSize()); TotalSize += BD->getSize(); ++Begin; @@ -195,8 +194,8 @@ void ReorderData::assignMemData(BinaryContext &BC) { for (const BinaryBasicBlock &BB : BF) { for (const MCInst &Inst : BB) { auto ErrorOrMemAccesssProfile = - BC.MIB->tryGetAnnotationAs( - Inst, "MemoryAccessProfile"); + BC.MIB->tryGetAnnotationAs( + Inst, "MemoryAccessProfile"); if (!ErrorOrMemAccesssProfile) continue; @@ -225,11 +224,11 @@ void ReorderData::assignMemData(BinaryContext &BC) { StringRef Section = Entry.first(); const uint64_t Count = Entry.second; outs() << "BOLT-INFO: " << Section << " = " << Count - << format(" (%.1f%%)\n", 100.0*Count/TotalCount); + << format(" (%.1f%%)\n", 100.0 * Count / TotalCount); if (JumpTableCounts.count(Section) != 0) { const uint64_t JTCount = JumpTableCounts[Section]; outs() << "BOLT-INFO: jump tables = " << JTCount - << format(" (%.1f%%)\n", 100.0*JTCount/Count); + << format(" (%.1f%%)\n", 100.0 * JTCount / Count); } } outs() << "BOLT-INFO: Total memory events: " << TotalCount << "\n"; @@ -238,11 +237,9 @@ void ReorderData::assignMemData(BinaryContext &BC) { /// Only consider moving data that is used by the hottest functions with /// valid profiles. -std::pair ReorderData::sortedByFunc( - BinaryContext &BC, - const BinarySection &Section, - std::map &BFs -) const { +std::pair +ReorderData::sortedByFunc(BinaryContext &BC, const BinarySection &Section, + std::map &BFs) const { std::map> BDtoFunc; std::map BDtoFuncCount; @@ -254,8 +251,8 @@ std::pair ReorderData::sortedByFunc( for (const MCInst &Inst : BB) { auto ErrorOrMemAccesssProfile = - BC.MIB->tryGetAnnotationAs( - Inst, "MemoryAccessProfile"); + BC.MIB->tryGetAnnotationAs( + Inst, "MemoryAccessProfile"); if (!ErrorOrMemAccesssProfile) continue; @@ -286,21 +283,21 @@ std::pair ReorderData::sortedByFunc( DataOrder Order = baseOrder(BC, Section); unsigned SplitPoint = Order.size(); - std::sort(Order.begin(), Order.end(), - [&](const DataOrder::value_type &A, - const DataOrder::value_type &B) { - // Total execution counts of functions referencing BD. - const uint64_t ACount = BDtoFuncCount[A.first]; - const uint64_t BCount = BDtoFuncCount[B.first]; - // Weight by number of loads/data size. - const double AWeight = double(A.second) / A.first->getSize(); - const double BWeight = double(B.second) / B.first->getSize(); - return (ACount > BCount || - (ACount == BCount && - (AWeight > BWeight || - (AWeight == BWeight && - A.first->getAddress() < B.first->getAddress())))); - }); + std::sort( + Order.begin(), Order.end(), + [&](const DataOrder::value_type &A, const DataOrder::value_type &B) { + // Total execution counts of functions referencing BD. + const uint64_t ACount = BDtoFuncCount[A.first]; + const uint64_t BCount = BDtoFuncCount[B.first]; + // Weight by number of loads/data size. + const double AWeight = double(A.second) / A.first->getSize(); + const double BWeight = double(B.second) / B.first->getSize(); + return (ACount > BCount || + (ACount == BCount && + (AWeight > BWeight || + (AWeight == BWeight && + A.first->getAddress() < B.first->getAddress())))); + }); for (unsigned Idx = 0; Idx < Order.size(); ++Idx) { if (!BDtoFuncCount[Order[Idx].first]) { @@ -312,16 +309,14 @@ std::pair ReorderData::sortedByFunc( return std::make_pair(Order, SplitPoint); } -std::pair ReorderData::sortedByCount( - BinaryContext &BC, - const BinarySection &Section -) const { +std::pair +ReorderData::sortedByCount(BinaryContext &BC, + const BinarySection &Section) const { DataOrder Order = baseOrder(BC, Section); unsigned SplitPoint = Order.size(); std::sort(Order.begin(), Order.end(), - [](const DataOrder::value_type &A, - const DataOrder::value_type &B) { + [](const DataOrder::value_type &A, const DataOrder::value_type &B) { // Weight by number of loads/data size. const double AWeight = double(A.second) / A.first->getSize(); const double BWeight = double(B.second) / B.first->getSize(); @@ -415,7 +410,7 @@ void ReorderData::setSectionOrder(BinaryContext &BC, OutputSection.reorderContents(NewOrder, opts::ReorderInplace); outs() << "BOLT-INFO: reorder-data: " << Count << "/" << TotalCount - << format(" (%.1f%%)", 100.0*Count/TotalCount) << " events, " + << format(" (%.1f%%)", 100.0 * Count / TotalCount) << " events, " << Offset << " hot bytes\n"; } @@ -443,8 +438,7 @@ bool ReorderData::markUnmoveableSymbols(BinaryContext &BC, } else { // check for overlapping symbols. BinaryData *Next = Itr != Range.end() ? std::next(Itr)->second : nullptr; - if (Next && - Itr->second->getEndAddress() != Next->getAddress() && + if (Next && Itr->second->getEndAddress() != Next->getAddress() && Next->containsAddress(Itr->second->getEndAddress())) { Itr->second->setIsMoveable(false); Next->setIsMoveable(false); @@ -456,12 +450,7 @@ bool ReorderData::markUnmoveableSymbols(BinaryContext &BC, } void ReorderData::runOnFunctions(BinaryContext &BC) { - static const char* DefaultSections[] = { - ".rodata", - ".data", - ".bss", - nullptr - }; + static const char *DefaultSections[] = {".rodata", ".data", ".bss", nullptr}; if (!BC.HasRelocations || opts::ReorderData.empty()) return; @@ -514,7 +503,7 @@ void ReorderData::runOnFunctions(BinaryContext &BC) { } else { outs() << "BOLT-INFO: reorder-sections: ordering data by funcs\n"; std::tie(Order, SplitPointIdx) = - sortedByFunc(BC, *Section, BC.getBinaryFunctions()); + sortedByFunc(BC, *Section, BC.getBinaryFunctions()); } auto SplitPoint = Order.begin() + SplitPointIdx; @@ -551,5 +540,5 @@ void ReorderData::runOnFunctions(BinaryContext &BC) { } } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Passes/ReorderFunctions.cpp b/bolt/lib/Passes/ReorderFunctions.cpp index 95bdc9e1e93c..ec734a264038 100644 --- a/bolt/lib/Passes/ReorderFunctions.cpp +++ b/bolt/lib/Passes/ReorderFunctions.cpp @@ -119,7 +119,7 @@ using Node = CallGraph::Node; void ReorderFunctions::reorder(std::vector &&Clusters, std::map &BFs) { - std::vector FuncAddr(Cg.numNodes()); // Just for computing stats + std::vector FuncAddr(Cg.numNodes()); // Just for computing stats uint64_t TotalSize = 0; uint32_t Index = 0; @@ -163,9 +163,9 @@ void ReorderFunctions::reorder(std::vector &&Clusters, } for (Cluster &Cluster : Clusters) { if (PrintDetailed) { - outs() << - format("BOLT-INFO: -------- density = %.3lf (%u / %u) --------\n", - Cluster.density(), Cluster.samples(), Cluster.size()); + outs() << format( + "BOLT-INFO: -------- density = %.3lf (%u / %u) --------\n", + Cluster.density(), Cluster.samples(), Cluster.size()); } for (NodeId FuncId : Cluster.targets()) { @@ -173,8 +173,8 @@ void ReorderFunctions::reorder(std::vector &&Clusters, Hotfuncs++; if (PrintDetailed) { - outs() << "BOLT-INFO: hot func " << *Cg.nodeIdToFunc(FuncId) - << " (" << Cg.size(FuncId) << ")\n"; + outs() << "BOLT-INFO: hot func " << *Cg.nodeIdToFunc(FuncId) << " (" + << Cg.size(FuncId) << ")\n"; } uint64_t Dist = 0; @@ -184,7 +184,7 @@ void ReorderFunctions::reorder(std::vector &&Clusters, continue; const Arc &Arc = *Cg.findArc(FuncId, Dst); const auto D = std::abs(FuncAddr[Arc.dst()] - - (FuncAddr[FuncId] + Arc.avgCallOffset())); + (FuncAddr[FuncId] + Arc.avgCallOffset())); const double W = Arc.weight(); if (D < 64 && PrintDetailed && opts::Verbosity > 2) { outs() << "BOLT-INFO: short (" << D << "B) call:\n" @@ -215,15 +215,13 @@ void ReorderFunctions::reorder(std::vector &&Clusters, if (PrintDetailed) { outs() << format("BOLT-INFO: start = %6u : avgCallDist = %lu : ", - TotalSize, - Calls ? Dist / Calls : 0) + TotalSize, Calls ? Dist / Calls : 0) << Cg.nodeIdToFunc(FuncId)->getPrintName() << '\n'; const uint64_t NewPage = TotalSize / HugePageSize; if (NewPage != CurPage) { CurPage = NewPage; - outs() << - format("BOLT-INFO: ============== page %u ==============\n", - CurPage); + outs() << format( + "BOLT-INFO: ============== page %u ==============\n", CurPage); } } } @@ -235,8 +233,8 @@ void ReorderFunctions::reorder(std::vector &&Clusters, Hotfuncs, Clusters.size()) << format("BOLT-INFO: Final average call distance = %.1lf " "(%.0lf / %.0lf)\n", - TotalCalls ? TotalDistance / TotalCalls : 0, - TotalDistance, TotalCalls) + TotalCalls ? TotalDistance / TotalCalls : 0, TotalDistance, + TotalCalls) << format("BOLT-INFO: Total Calls = %.0lf\n", TotalCalls); if (TotalCalls) { outs() << format("BOLT-INFO: Total Calls within 64B = %.0lf (%.2lf%%)\n", @@ -291,7 +289,7 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) { std::vector Clusters; - switch(opts::ReorderFunctions) { + switch (opts::ReorderFunctions) { case RT_NONE: break; case RT_EXEC_COUNT: @@ -395,9 +393,8 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) { std::unique_ptr FuncsFile; if (!opts::GenerateFunctionOrderFile.empty()) { - FuncsFile = - std::make_unique(opts::GenerateFunctionOrderFile, - std::ios::out); + FuncsFile = std::make_unique(opts::GenerateFunctionOrderFile, + std::ios::out); if (!FuncsFile) { errs() << "BOLT-ERROR: ordered functions file " << opts::GenerateFunctionOrderFile << " cannot be opened\n"; @@ -408,20 +405,17 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) { std::unique_ptr LinkSectionsFile; if (!opts::LinkSectionsFile.empty()) { LinkSectionsFile = - std::make_unique(opts::LinkSectionsFile, - std::ios::out); + std::make_unique(opts::LinkSectionsFile, std::ios::out); if (!LinkSectionsFile) { - errs() << "BOLT-ERROR: link sections file " - << opts::LinkSectionsFile << " cannot be opened\n"; + errs() << "BOLT-ERROR: link sections file " << opts::LinkSectionsFile + << " cannot be opened\n"; exit(1); } } if (FuncsFile || LinkSectionsFile) { std::vector SortedFunctions(BFs.size()); - std::transform(BFs.begin(), - BFs.end(), - SortedFunctions.begin(), + std::transform(BFs.begin(), BFs.end(), SortedFunctions.begin(), [](std::pair &BFI) { return &BFI.second; }); diff --git a/bolt/lib/Passes/RetpolineInsertion.cpp b/bolt/lib/Passes/RetpolineInsertion.cpp index d24a128167ff..e2082b5bc946 100644 --- a/bolt/lib/Passes/RetpolineInsertion.cpp +++ b/bolt/lib/Passes/RetpolineInsertion.cpp @@ -297,8 +297,8 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) { // Determine if r11 is available before this instruction if (BrInfo.isMem()) { - if(MIB.hasAnnotation(Inst, "PLTCall")) - R11Available= true; + if (MIB.hasAnnotation(Inst, "PLTCall")) + R11Available = true; else if (opts::R11Availability == AvailabilityOptions::ALWAYS) R11Available = true; else if (opts::R11Availability == AvailabilityOptions::ABI) @@ -320,7 +320,7 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) { TargetRetpoline = getOrCreateRetpoline(BC, BrInfo, R11Available); createBranchReplacement(BC, BrInfo, R11Available, Replacement, - TargetRetpoline->getSymbol()); + TargetRetpoline->getSymbol()); It = BB.replaceInstruction(It, Replacement.begin(), Replacement.end()); RetpolinedBranches++; @@ -329,8 +329,8 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) { } outs() << "BOLT-INFO: The number of created retpoline functions is : " << CreatedRetpolines.size() - << "\nBOLT-INFO: The number of retpolined branches is : " << RetpolinedBranches - << "\n"; + << "\nBOLT-INFO: The number of retpolined branches is : " + << RetpolinedBranches << "\n"; } } // namespace bolt diff --git a/bolt/lib/Passes/ShrinkWrapping.cpp b/bolt/lib/Passes/ShrinkWrapping.cpp index 8574a1e82f20..8ee6de1f55d0 100644 --- a/bolt/lib/Passes/ShrinkWrapping.cpp +++ b/bolt/lib/Passes/ShrinkWrapping.cpp @@ -29,7 +29,7 @@ static cl::opt ShrinkWrappingThreshold( " evaluating whether a block is cold enough to be profitable to" " move eligible spills there"), cl::init(30), cl::ZeroOrMore, cl::cat(BoltOptCategory)); -} +} // namespace opts namespace llvm { namespace bolt { @@ -261,15 +261,15 @@ void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) { void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) { StackPointerTracking &SPT = Info.getStackPointerTracking(); if (!BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, BC.MIB->getStackPointer(), *BC.MRI)) + .hasDefOfPhysReg(Point, BC.MIB->getStackPointer(), *BC.MRI)) return; // Check if the definition of SP comes from FP -- in this case, this // value may need to be updated depending on our stack layout changes const MCInstrDesc &InstInfo = BC.MII->get(Point.getOpcode()); unsigned NumDefs = InstInfo.getNumDefs(); bool UsesFP = false; - for (unsigned I = NumDefs, E = MCPlus::getNumPrimeOperands(Point); - I < E; ++I) { + for (unsigned I = NumDefs, E = MCPlus::getNumPrimeOperands(Point); I < E; + ++I) { MCOperand &Operand = Point.getOperand(I); if (!Operand.isReg()) continue; @@ -509,7 +509,7 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, continue; // SP update based on frame pointer scheduleChange( - Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, RegionSz)); + Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, RegionSz)); continue; } @@ -521,7 +521,6 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, continue; } - if (FIE->StackPtrReg == BC.MIB->getStackPointer() && Slot < RegionAddr) continue; @@ -609,7 +608,7 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) { if (Slot >= RegionAddr) continue; scheduleChange( - Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, -RegionSz)); + Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, -RegionSz)); continue; } @@ -728,8 +727,7 @@ void ShrinkWrapping::classifyCSRUses() { UsesByReg = std::vector(BC.MRI->getNumRegs(), BitVector(DA.NumInstrs, false)); - const BitVector &FPAliases = - BC.MIB->getAliases(BC.MIB->getFramePointer()); + const BitVector &FPAliases = BC.MIB->getAliases(BC.MIB->getFramePointer()); for (BinaryBasicBlock &BB : BF) { for (MCInst &Inst : BB) { if (BC.MIB->isCFI(Inst)) @@ -1083,8 +1081,9 @@ bool ShrinkWrapping::validatePushPopsMode(unsigned CSR, MCInst *BestPosSave, int64_t SaveOffset) { if (FA.requiresAlignment(BF)) { LLVM_DEBUG({ - dbgs() << "Reg " << CSR << " is not using push/pops due to function " - "alignment requirements.\n"; + dbgs() << "Reg " << CSR + << " is not using push/pops due to function " + "alignment requirements.\n"; }); return false; } @@ -1105,12 +1104,12 @@ bool ShrinkWrapping::validatePushPopsMode(unsigned CSR, MCInst *BestPosSave, StackPointerTracking &SPT = Info.getStackPointerTracking(); // Abort if we are inserting a push into an entry BB (offset -8) and this // func sets up a frame pointer. - if (!SLM.canInsertRegion(BestPosSave) || - SaveOffset == SPT.SUPERPOSITION || SaveOffset == SPT.EMPTY || - (SaveOffset == -8 && SPT.HasFramePointer)) { + if (!SLM.canInsertRegion(BestPosSave) || SaveOffset == SPT.SUPERPOSITION || + SaveOffset == SPT.EMPTY || (SaveOffset == -8 && SPT.HasFramePointer)) { LLVM_DEBUG({ - dbgs() << "Reg " << CSR << " cannot insert region or we are " - "trying to insert a push into entry bb.\n"; + dbgs() << "Reg " << CSR + << " cannot insert region or we are " + "trying to insert a push into entry bb.\n"; }); return false; } @@ -1242,8 +1241,9 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions( BestPosSave->dump(); }); - scheduleChange(BestPosSave, UsePushPops ? WorklistItem::InsertPushOrPop - : WorklistItem::InsertLoadOrStore, + scheduleChange(BestPosSave, + UsePushPops ? WorklistItem::InsertPushOrPop + : WorklistItem::InsertLoadOrStore, *FIESave, CSR); for (ProgramPoint &PP : RestorePoints) { @@ -1268,8 +1268,8 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions( PrecededByPrefix = BC.MIB->isPrefix(*Iter); } } - if (PP.isInst() && (doesInstUsesCSR(*PP.getInst(), CSR) || - PrecededByPrefix)) { + if (PP.isInst() && + (doesInstUsesCSR(*PP.getInst(), CSR) || PrecededByPrefix)) { assert(!InsnToBB[PP.getInst()]->hasTerminatorAfter(PP.getInst()) && "cannot move to end of bb"); scheduleChange(InsnToBB[PP.getInst()], @@ -1278,8 +1278,9 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions( *FIELoad, CSR); continue; } - scheduleChange(PP, UsePushPops ? WorklistItem::InsertPushOrPop - : WorklistItem::InsertLoadOrStore, + scheduleChange(PP, + UsePushPops ? WorklistItem::InsertPushOrPop + : WorklistItem::InsertLoadOrStore, *FIELoad, CSR); } } @@ -1391,8 +1392,7 @@ void ShrinkWrapping::moveSaveRestores() { namespace { /// Helper function to identify whether two basic blocks created by splitting /// a critical edge have the same contents. -bool isIdenticalSplitEdgeBB(const BinaryContext &BC, - const BinaryBasicBlock &A, +bool isIdenticalSplitEdgeBB(const BinaryContext &BC, const BinaryBasicBlock &A, const BinaryBasicBlock &B) { if (A.succ_size() != B.succ_size()) return false; @@ -1411,15 +1411,16 @@ bool isIdenticalSplitEdgeBB(const BinaryContext &BC, while (I != E && OtherI != OtherE) { if (I->getOpcode() != OtherI->getOpcode()) return false; - if (!BC.MIB->equals(*I, *OtherI, - [](const MCSymbol *A, const MCSymbol *B) { return true; })) + if (!BC.MIB->equals(*I, *OtherI, [](const MCSymbol *A, const MCSymbol *B) { + return true; + })) return false; ++I; ++OtherI; } return true; } -} +} // namespace bool ShrinkWrapping::foldIdenticalSplitEdges() { bool Changed = false; @@ -1431,8 +1432,7 @@ bool ShrinkWrapping::foldIdenticalSplitEdges() { BinaryBasicBlock &RBB = *RIter; if (&RBB == &BB) break; - if (!RBB.getName().startswith(".LSplitEdge") || - !RBB.isValid() || + if (!RBB.getName().startswith(".LSplitEdge") || !RBB.isValid() || !isIdenticalSplitEdgeBB(BC, *Iter, RBB)) continue; assert(RBB.pred_size() == 1 && "Invalid split edge BB"); @@ -1551,8 +1551,8 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush, bool IsStoreFromReg = false; uint8_t Size = 0; if (!BC.MIB->isStackAccess(*InstIter, IsLoad, IsStore, IsStoreFromReg, - Reg, SrcImm, StackPtrReg, StackOffset, - Size, IsSimple, IsIndexed)) + Reg, SrcImm, StackPtrReg, StackOffset, Size, + IsSimple, IsIndexed)) continue; if (Reg != CSR || !IsStore || !IsSimple) continue; @@ -1660,7 +1660,7 @@ void ShrinkWrapping::rebuildCFIForSP() { } for (BinaryBasicBlock &BB : BF) { - for (auto I = BB.begin(); I != BB.end(); ) { + for (auto I = BB.begin(); I != BB.end();) { if (BC.MIB->hasAnnotation(*I, "DeleteMe")) I = BB.eraseInstruction(I); else @@ -1849,18 +1849,21 @@ BBIterTy ShrinkWrapping::processInsertionsList( } // Reorder POPs to obey the correct dominance relation between them - std::stable_sort(TodoList.begin(), TodoList.end(), [&](const WorklistItem &A, - const WorklistItem - &B) { - if ((A.Action != WorklistItem::InsertPushOrPop || !A.FIEToInsert.IsLoad) && - (B.Action != WorklistItem::InsertPushOrPop || !B.FIEToInsert.IsLoad)) - return false; - if ((A.Action != WorklistItem::InsertPushOrPop || !A.FIEToInsert.IsLoad)) - return true; - if ((B.Action != WorklistItem::InsertPushOrPop || !B.FIEToInsert.IsLoad)) - return false; - return DomOrder[B.AffectedReg] < DomOrder[A.AffectedReg]; - }); + std::stable_sort(TodoList.begin(), TodoList.end(), + [&](const WorklistItem &A, const WorklistItem &B) { + if ((A.Action != WorklistItem::InsertPushOrPop || + !A.FIEToInsert.IsLoad) && + (B.Action != WorklistItem::InsertPushOrPop || + !B.FIEToInsert.IsLoad)) + return false; + if ((A.Action != WorklistItem::InsertPushOrPop || + !A.FIEToInsert.IsLoad)) + return true; + if ((B.Action != WorklistItem::InsertPushOrPop || + !B.FIEToInsert.IsLoad)) + return false; + return DomOrder[B.AffectedReg] < DomOrder[A.AffectedReg]; + }); // Process insertions for (WorklistItem &Item : TodoList) { diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp index 263af6b1478f..92ef63da8e34 100644 --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -194,9 +194,9 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) { // Even if they were natural to cluster formation and were seen in-between // hot basic blocks. std::stable_sort(BF.layout_begin(), BF.layout_end(), - [&] (BinaryBasicBlock *A, BinaryBasicBlock *B) { - return A->canOutline() < B->canOutline(); - }); + [&](BinaryBasicBlock *A, BinaryBasicBlock *B) { + return A->canOutline() < B->canOutline(); + }); } else if (BF.hasEHRanges() && !opts::SplitEH) { // Typically functions with exception handling have landing pads at the end. // We cannot move beginning of landing pads, but we can move 0-count blocks @@ -206,14 +206,13 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) { ++FirstLP; std::stable_sort(FirstLP, BF.layout_end(), - [&] (BinaryBasicBlock *A, BinaryBasicBlock *B) { - return A->canOutline() < B->canOutline(); - }); + [&](BinaryBasicBlock *A, BinaryBasicBlock *B) { + return A->canOutline() < B->canOutline(); + }); } // Separate hot from cold starting from the bottom. - for (auto I = BF.layout_rbegin(), E = BF.layout_rend(); - I != E; ++I) { + for (auto I = BF.layout_rbegin(), E = BF.layout_rend(); I != E; ++I) { BinaryBasicBlock *BB = *I; if (!BB->canOutline()) break; diff --git a/bolt/lib/Passes/StackAllocationAnalysis.cpp b/bolt/lib/Passes/StackAllocationAnalysis.cpp index 17e6d434f120..8718256c6eee 100644 --- a/bolt/lib/Passes/StackAllocationAnalysis.cpp +++ b/bolt/lib/Passes/StackAllocationAnalysis.cpp @@ -24,9 +24,10 @@ void StackAllocationAnalysis::preflight() { for (BinaryBasicBlock &BB : this->Func) { for (MCInst &Inst : BB) { MCPhysReg From, To; - if (!BC.MIB->isPush(Inst) && (!BC.MIB->isRegToRegMove(Inst, From, To) || - To != BC.MIB->getStackPointer() || - From != BC.MIB->getFramePointer()) && + if (!BC.MIB->isPush(Inst) && + (!BC.MIB->isRegToRegMove(Inst, From, To) || + To != BC.MIB->getStackPointer() || + From != BC.MIB->getFramePointer()) && !BC.MII->get(Inst.getOpcode()) .hasDefOfPhysReg(Inst, BC.MIB->getStackPointer(), *BC.MRI)) continue; diff --git a/bolt/lib/Passes/StokeInfo.cpp b/bolt/lib/Passes/StokeInfo.cpp index 1d97f332d395..e7e9ff256093 100644 --- a/bolt/lib/Passes/StokeInfo.cpp +++ b/bolt/lib/Passes/StokeInfo.cpp @@ -13,7 +13,6 @@ #include "bolt/Passes/DataflowInfoManager.h" #include "llvm/Support/CommandLine.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "stoke" using namespace llvm; @@ -32,9 +31,8 @@ StokeOutputDataFilename("stoke-out", namespace llvm { namespace bolt { - void getRegNameFromBitVec(const BinaryContext &BC, const BitVector &RegV, - std::set *NameVec = nullptr) { + std::set *NameVec = nullptr) { int RegIdx = RegV.find_first(); while (RegIdx != -1) { LLVM_DEBUG(dbgs() << BC.MRI->getName(RegIdx) << " "); diff --git a/bolt/lib/Passes/ValidateInternalCalls.cpp b/bolt/lib/Passes/ValidateInternalCalls.cpp index f0ebc06e9765..76c34c0ac849 100644 --- a/bolt/lib/Passes/ValidateInternalCalls.cpp +++ b/bolt/lib/Passes/ValidateInternalCalls.cpp @@ -32,7 +32,6 @@ BinaryBasicBlock *getInternalCallTarget(BinaryFunction &Function, return Function.getBasicBlockForLabel(BC.MIB->getTargetSymbol(Inst)); } - // A special StackPointerTracking that considers internal calls class StackPointerTrackingForInternalCalls : public StackPointerTrackingBase { @@ -134,7 +133,7 @@ bool ValidateInternalCalls::fixCFGForIC(BinaryFunction &Function) const { bool Updated = false; - auto processReturns = [&] (BinaryBasicBlock &BB, MCInst &Return) { + auto processReturns = [&](BinaryBasicBlock &BB, MCInst &Return) { // Check all reaching internal calls for (auto I = RI.expr_begin(Return), E = RI.expr_end(); I != E; ++I) { MCInst &ReachingInst = **I; @@ -181,7 +180,8 @@ bool ValidateInternalCalls::fixCFGForIC(BinaryFunction &Function) const { return Updated; } -bool ValidateInternalCalls::hasTailCallsInRange(BinaryFunction &Function) const { +bool ValidateInternalCalls::hasTailCallsInRange( + BinaryFunction &Function) const { const BinaryContext &BC = Function.getBinaryContext(); for (BinaryBasicBlock &BB : Function) { for (MCInst &Inst : BB) { @@ -193,9 +193,11 @@ bool ValidateInternalCalls::hasTailCallsInRange(BinaryFunction &Function) const } bool ValidateInternalCalls::analyzeFunction(BinaryFunction &Function) const { - while (fixCFGForPIC(Function)) {} + while (fixCFGForPIC(Function)) { + } clearAnnotations(Function); - while (fixCFGForIC(Function)) {} + while (fixCFGForIC(Function)) { + } BinaryContext &BC = Function.getBinaryContext(); RegAnalysis RA = RegAnalysis(BC, nullptr, nullptr); @@ -338,5 +340,5 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) { } } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 0f6b699a95cc..0f1ca99492c9 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -17,7 +17,7 @@ namespace llvm { namespace bolt { -const char* BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; +const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, @@ -218,8 +218,9 @@ uint64_t BoltAddressTranslation::translate(const BinaryFunction &Func, } Optional -BoltAddressTranslation::getFallthroughsInTrace( - const BinaryFunction &Func, uint64_t From, uint64_t To) const { +BoltAddressTranslation::getFallthroughsInTrace(const BinaryFunction &Func, + uint64_t From, + uint64_t To) const { SmallVector, 16> Res; // Filter out trivial case @@ -253,7 +254,7 @@ BoltAddressTranslation::getFallthroughsInTrace( if (FromIter->first >= ToIter->first) return Res; - for (auto Iter = FromIter; Iter != ToIter; ) { + for (auto Iter = FromIter; Iter != ToIter;) { const uint32_t Src = Iter->first; if (Iter->second & BRANCHENTRY) { ++Iter; @@ -291,5 +292,5 @@ bool BoltAddressTranslation::enabledFor( } return false; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 35d659f575ea..48a813fb15f4 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -107,7 +107,7 @@ WriteAutoFDOData("autofdo", cl::ZeroOrMore, cl::cat(AggregatorCategory)); -} +} // namespace opts namespace { @@ -118,15 +118,13 @@ const char TimerGroupDesc[] = "Aggregator"; constexpr uint64_t DataAggregator::KernelBaseAddr; -DataAggregator::~DataAggregator() { - deleteTempFiles(); -} +DataAggregator::~DataAggregator() { deleteTempFiles(); } namespace { void deleteTempFile(const std::string &FileName) { if (std::error_code Errc = sys::fs::remove(FileName.c_str())) { - errs() << "PERF2BOLT: failed to delete temporary file " - << FileName << " with error " << Errc.message() << "\n"; + errs() << "PERF2BOLT: failed to delete temporary file " << FileName + << " with error " << Errc.message() << "\n"; } } } @@ -149,8 +147,7 @@ void DataAggregator::findPerfExecutable() { } void DataAggregator::start() { - outs() << "PERF2BOLT: Starting data aggregation job for " << Filename - << "\n"; + outs() << "PERF2BOLT: Starting data aggregation job for " << Filename << "\n"; // Don't launch perf for pre-aggregated files if (opts::ReadPreAggregated) @@ -229,17 +226,16 @@ void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI, if (std::error_code Errc = sys::fs::createTemporaryFile("perf.script", "out", PPI.StdoutPath)) { - errs() << "PERF2BOLT: failed to create temporary file " - << PPI.StdoutPath << " with error " << Errc.message() - << "\n"; + errs() << "PERF2BOLT: failed to create temporary file " << PPI.StdoutPath + << " with error " << Errc.message() << "\n"; exit(1); } TempFiles.push_back(PPI.StdoutPath.data()); if (std::error_code Errc = sys::fs::createTemporaryFile("perf.script", "err", PPI.StderrPath)) { - errs() << "PERF2BOLT: failed to create temporary file " - << PPI.StderrPath << " with error " << Errc.message() << "\n"; + errs() << "PERF2BOLT: failed to create temporary file " << PPI.StderrPath + << " with error " << Errc.message() << "\n"; exit(1); } TempFiles.push_back(PPI.StderrPath.data()); @@ -277,7 +273,7 @@ void DataAggregator::processFileBuildID(StringRef FileBuildID) { if (BuildIDProcessInfo.PI.ReturnCode != 0) { ErrorOr> MB = - MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StderrPath.data()); + MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StderrPath.data()); StringRef ErrBuf = (*MB)->getBuffer(); errs() << "PERF-ERROR: return code " << BuildIDProcessInfo.PI.ReturnCode @@ -287,7 +283,7 @@ void DataAggregator::processFileBuildID(StringRef FileBuildID) { } ErrorOr> MB = - MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StdoutPath.data()); + MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StdoutPath.data()); if (std::error_code EC = MB.getError()) { errs() << "Cannot open " << BuildIDProcessInfo.StdoutPath.data() << ": " << EC.message() << "\n"; @@ -368,8 +364,8 @@ void DataAggregator::parsePreAggregated() { std::error_code DataAggregator::writeAutoFDOData(StringRef OutputFilename) { outs() << "PERF2BOLT: writing data for autofdo tools...\n"; - NamedRegionTimer T("writeAutoFDO", "Processing branch events", - TimerGroupName, TimerGroupDesc, opts::TimeAggregator); + NamedRegionTimer T("writeAutoFDO", "Processing branch events", TimerGroupName, + TimerGroupDesc, opts::TimeAggregator); std::error_code EC; raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None); @@ -446,8 +442,8 @@ void DataAggregator::filterBinaryMMapInfo() { if (errs().has_colors()) errs().changeColor(raw_ostream::RED); errs() << "PERF2BOLT-ERROR: could not find a profile matching PID \"" - << opts::FilterPID << "\"" << " for binary \"" - << BC->getFilename() <<"\"."; + << opts::FilterPID << "\"" + << " for binary \"" << BC->getFilename() << "\"."; assert(!BinaryMMapInfo.empty() && "No memory map for matching binary"); errs() << " Profile for the following process is available:\n"; for (std::pair &MMI : BinaryMMapInfo) { @@ -492,7 +488,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { if (PI.ReturnCode != 0) { ErrorOr> ErrorMB = - MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data()); + MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data()); StringRef ErrBuf = (*ErrorMB)->getBuffer(); errs() << "PERF-ERROR: return code " << PI.ReturnCode << "\n"; @@ -502,7 +498,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { } ErrorOr> MB = - MemoryBuffer::getFileOrSTDIN(Process.StdoutPath.data()); + MemoryBuffer::getFileOrSTDIN(Process.StdoutPath.data()); if (std::error_code EC = MB.getError()) { errs() << "Cannot open " << Process.StdoutPath.data() << ": " << EC.message() << "\n"; @@ -573,7 +569,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { sys::ProcessInfo PI = sys::Wait(MemEventsPPI.PI, 0, true, &Error); if (PI.ReturnCode != 0) { ErrorOr> MB = - MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StderrPath.data()); + MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StderrPath.data()); StringRef ErrBuf = (*MB)->getBuffer(); deleteTempFiles(); @@ -589,7 +585,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { } ErrorOr> MB = - MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StdoutPath.data()); + MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StdoutPath.data()); if (std::error_code EC = MB.getError()) { errs() << "Cannot open " << MemEventsPPI.StdoutPath.data() << ": " << EC.message() << "\n"; @@ -602,8 +598,8 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) { Col = 0; Line = 1; if (const std::error_code EC = parseMemEvents()) { - errs() << "PERF2BOLT: failed to parse memory events: " - << EC.message() << '\n'; + errs() << "PERF2BOLT: failed to parse memory events: " << EC.message() + << '\n'; } deleteTempFiles(); @@ -701,9 +697,9 @@ bool DataAggregator::doSample(BinaryFunction &Func, uint64_t Address, if (I == NamesToSamples.end()) { bool Success; StringRef LocName = getLocationName(Func, Count); - std::tie(I, Success) = NamesToSamples.insert(std::make_pair( - Func.getOneName(), - FuncSampleData(LocName, FuncSampleData::ContainerTy()))); + std::tie(I, Success) = NamesToSamples.insert( + std::make_pair(Func.getOneName(), + FuncSampleData(LocName, FuncSampleData::ContainerTy()))); } Address -= Func.getAddress(); @@ -922,7 +918,7 @@ bool DataAggregator::recordTrace( return false; } - // Record fall-through jumps + // Record fall-through jumps BinaryBasicBlock::BinaryBranchInfo &BI = BB->getBranchInfo(*NextBB); BI.Count += Count; @@ -980,8 +976,8 @@ bool DataAggregator::recordEntry(BinaryFunction &BF, uint64_t To, bool Mispred, return true; } -bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, - bool Mispred, uint64_t Count) const { +bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred, + uint64_t Count) const { if (!BF.isSimple() || From > BF.getSize()) return false; @@ -1025,7 +1021,7 @@ ErrorOr DataAggregator::parseLBREntry() { } Res.Mispred = MispredStr[0] == 'M'; - static bool MispredWarning = true;; + static bool MispredWarning = true; if (MispredStr[0] == '-' && MispredWarning) { errs() << "PERF2BOLT-WARNING: misprediction bit is missing in profile\n"; MispredWarning = false; @@ -1067,7 +1063,8 @@ void DataAggregator::consumeRestOfLine() { ErrorOr DataAggregator::parseBranchSample() { PerfBranchSample Res; - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr PIDRes = parseNumberField(FieldSeparator, true); if (std::error_code EC = PIDRes.getError()) @@ -1078,7 +1075,8 @@ ErrorOr DataAggregator::parseBranchSample() { return make_error_code(errc::no_such_process); } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr PCRes = parseHexField(FieldSeparator, true); if (std::error_code EC = PCRes.getError()) @@ -1106,7 +1104,8 @@ ErrorOr DataAggregator::parseBranchSample() { } ErrorOr DataAggregator::parseBasicSample() { - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr PIDRes = parseNumberField(FieldSeparator, true); if (std::error_code EC = PIDRes.getError()) @@ -1118,13 +1117,15 @@ ErrorOr DataAggregator::parseBasicSample() { return PerfBasicSample{StringRef(), 0}; } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr Event = parseString(FieldSeparator); if (std::error_code EC = Event.getError()) return EC; - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr AddrRes = parseHexField(FieldSeparator, true); if (std::error_code EC = AddrRes.getError()) { @@ -1144,9 +1145,10 @@ ErrorOr DataAggregator::parseBasicSample() { } ErrorOr DataAggregator::parseMemSample() { - PerfMemSample Res{0,0}; + PerfMemSample Res{0, 0}; - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr PIDRes = parseNumberField(FieldSeparator, true); if (std::error_code EC = PIDRes.getError()) @@ -1158,7 +1160,8 @@ ErrorOr DataAggregator::parseMemSample() { return Res; } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr Event = parseString(FieldSeparator); if (std::error_code EC = Event.getError()) @@ -1168,14 +1171,16 @@ ErrorOr DataAggregator::parseMemSample() { return Res; } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr AddrRes = parseHexField(FieldSeparator); if (std::error_code EC = AddrRes.getError()) { return EC; } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr PCRes = parseHexField(FieldSeparator, true); if (std::error_code EC = PCRes.getError()) { @@ -1221,7 +1226,8 @@ ErrorOr DataAggregator::parseLocationOrOffset() { ErrorOr DataAggregator::parseAggregatedLBREntry() { - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr TypeOrErr = parseString(FieldSeparator); if (std::error_code EC = TypeOrErr.getError()) @@ -1238,17 +1244,20 @@ DataAggregator::parseAggregatedLBREntry() { return make_error_code(llvm::errc::io_error); } - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr From = parseLocationOrOffset(); if (std::error_code EC = From.getError()) return EC; - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr To = parseLocationOrOffset(); if (std::error_code EC = To.getError()) return EC; - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr Frequency = parseNumberField(FieldSeparator, Type != AggregatedLBREntry::BRANCH); if (std::error_code EC = Frequency.getError()) @@ -1256,7 +1265,8 @@ DataAggregator::parseAggregatedLBREntry() { uint64_t Mispreds = 0; if (Type == AggregatedLBREntry::BRANCH) { - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr MispredsOrErr = parseNumberField(FieldSeparator, true); if (std::error_code EC = MispredsOrErr.getError()) return EC; @@ -1501,8 +1511,8 @@ std::error_code DataAggregator::parseBranchEvents() { OS << ")"; }; - outs() << "PERF2BOLT: read " << NumSamples << " samples and " - << NumEntries << " LBR entries\n"; + outs() << "PERF2BOLT: read " << NumSamples << " samples and " << NumEntries + << " LBR entries\n"; if (NumTotalSamples) { if (NumSamples && NumSamplesNoLBR == NumSamples) { // Note: we don't know if perf2bolt is being used to parse memory samples @@ -1611,8 +1621,8 @@ std::error_code DataAggregator::parseBasicEvents() { void DataAggregator::processBasicEvents() { outs() << "PERF2BOLT: processing basic events (without LBR)...\n"; - NamedRegionTimer T("processBasic", "Processing basic events", - TimerGroupName, TimerGroupDesc, opts::TimeAggregator); + NamedRegionTimer T("processBasic", "Processing basic events", TimerGroupName, + TimerGroupDesc, opts::TimeAggregator); uint64_t OutOfRangeSamples = 0; uint64_t NumSamples = 0; for (auto &Sample : BasicSamples) { @@ -1845,9 +1855,9 @@ Optional parsePerfTime(const StringRef TimeStr) { } } -Optional -DataAggregator::parseForkEvent() { - while (checkAndConsumeFS()) {} +Optional DataAggregator::parseForkEvent() { + while (checkAndConsumeFS()) { + } size_t LineEnd = ParsingBuf.find_first_of("\n"); if (LineEnd == StringRef::npos) { @@ -1896,7 +1906,8 @@ DataAggregator::parseForkEvent() { ErrorOr> DataAggregator::parseMMapEvent() { - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } MMapInfo ParsedInfo; @@ -2077,8 +2088,7 @@ std::error_code DataAggregator::parseTaskEvents() { if (Optional CommInfo = parseCommExecEvent()) { // Remove forked child that ran execve auto MMapInfoIter = BinaryMMapInfo.find(*CommInfo); - if (MMapInfoIter != BinaryMMapInfo.end() && - MMapInfoIter->second.Forked) { + if (MMapInfoIter != BinaryMMapInfo.end() && MMapInfoIter->second.Forked) { BinaryMMapInfo.erase(MMapInfoIter); } consumeRestOfLine(); @@ -2114,8 +2124,8 @@ std::error_code DataAggregator::parseTaskEvents() { LLVM_DEBUG({ for (std::pair &MMI : BinaryMMapInfo) { outs() << " " << MMI.second.PID << (MMI.second.Forked ? " (forked)" : "") - << ": (0x" << Twine::utohexstr(MMI.second.BaseAddress) - << ": 0x" << Twine::utohexstr(MMI.second.Size) << ")\n"; + << ": (0x" << Twine::utohexstr(MMI.second.BaseAddress) << ": 0x" + << Twine::utohexstr(MMI.second.Size) << ")\n"; } }); @@ -2124,7 +2134,8 @@ std::error_code DataAggregator::parseTaskEvents() { Optional> DataAggregator::parseNameBuildIDPair() { - while (checkAndConsumeFS()) {} + while (checkAndConsumeFS()) { + } ErrorOr BuildIDStr = parseString(FieldSeparator, true); if (std::error_code EC = BuildIDStr.getError()) @@ -2160,7 +2171,7 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const { bool WriteMemLocs = false; - auto writeLocation = [&OutFile,&WriteMemLocs](const Location &Loc) { + auto writeLocation = [&OutFile, &WriteMemLocs](const Location &Loc) { if (WriteMemLocs) OutFile << (Loc.IsSymbol ? "4 " : "3 "); else @@ -2219,15 +2230,13 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const { } } - outs() << "PERF2BOLT: wrote " << BranchValues << " objects and " - << MemValues << " memory objects to " << OutputFilename << "\n"; + outs() << "PERF2BOLT: wrote " << BranchValues << " objects and " << MemValues + << " memory objects to " << OutputFilename << "\n"; return std::error_code(); } -void DataAggregator::dump() const { - DataReader::dump(); -} +void DataAggregator::dump() const { DataReader::dump(); } void DataAggregator::dump(const LBREntry &LBR) const { Diag << "From: " << Twine::utohexstr(LBR.From) diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp index a3d213e2d027..ecdc00a8b11e 100644 --- a/bolt/lib/Profile/DataReader.cpp +++ b/bolt/lib/Profile/DataReader.cpp @@ -129,17 +129,14 @@ uint64_t FuncBranchData::getNumExecutedBranches() const { return ExecutedBranches; } -void SampleInfo::mergeWith(const SampleInfo &SI) { - Hits += SI.Hits; -} +void SampleInfo::mergeWith(const SampleInfo &SI) { Hits += SI.Hits; } void SampleInfo::print(raw_ostream &OS) const { OS << Loc.IsSymbol << " " << Loc.Name << " " << Twine::utohexstr(Loc.Offset) << " " << Hits << "\n"; } -uint64_t -FuncSampleData::getSamples(uint64_t Start, uint64_t End) const { +uint64_t FuncSampleData::getSamples(uint64_t Start, uint64_t End) const { assert(std::is_sorted(Data.begin(), Data.end())); struct Compare { bool operator()(const SampleInfo &SI, const uint64_t Val) const { @@ -217,17 +214,15 @@ void BranchInfo::mergeWith(const BranchInfo &BI) { void BranchInfo::print(raw_ostream &OS) const { OS << From.IsSymbol << " " << From.Name << " " - << Twine::utohexstr(From.Offset) << " " - << To.IsSymbol << " " << To.Name << " " - << Twine::utohexstr(To.Offset) << " " - << Mispreds << " " << Branches << '\n'; + << Twine::utohexstr(From.Offset) << " " << To.IsSymbol << " " << To.Name + << " " << Twine::utohexstr(To.Offset) << " " << Mispreds << " " << Branches + << '\n'; } ErrorOr FuncBranchData::getBranch(uint64_t From, uint64_t To) const { for (const BranchInfo &I : Data) { - if (I.From.Offset == From && I.To.Offset == To && - I.From.Name == I.To.Name) + if (I.From.Offset == From && I.To.Offset == To && I.From.Name == I.To.Name) return I; } return make_error_code(llvm::errc::invalid_argument); @@ -255,10 +250,9 @@ FuncBranchData::getDirectCallBranch(uint64_t From) const { void MemInfo::print(raw_ostream &OS) const { OS << (Offset.IsSymbol + 3) << " " << Offset.Name << " " - << Twine::utohexstr(Offset.Offset) << " " - << (Addr.IsSymbol + 3) << " " << Addr.Name << " " - << Twine::utohexstr(Addr.Offset) << " " - << Count << "\n"; + << Twine::utohexstr(Offset.Offset) << " " << (Addr.IsSymbol + 3) << " " + << Addr.Name << " " << Twine::utohexstr(Addr.Offset) << " " << Count + << "\n"; } void MemInfo::prettyPrint(raw_ostream &OS) const { @@ -326,14 +320,14 @@ Error DataReader::readProfilePreCFG(BinaryContext &BC) { } auto &MemAccessProfile = - BC.MIB->getOrCreateAnnotationAs( - II->second, "MemoryAccessProfile"); + BC.MIB->getOrCreateAnnotationAs( + II->second, "MemoryAccessProfile"); BinaryData *BD = nullptr; if (MI.Addr.IsSymbol) { BD = BC.getBinaryDataByName(MI.Addr.Name); } - MemAccessProfile.AddressAccessInfo. - push_back({BD, MI.Addr.Offset, MI.Count}); + MemAccessProfile.AddressAccessInfo.push_back( + {BD, MI.Addr.Offset, MI.Count}); auto NextII = std::next(II); if (NextII == Function.Instructions.end()) { MemAccessProfile.NextInstrOffset = Function.getSize(); @@ -426,8 +420,8 @@ void DataReader::readProfile(BinaryFunction &BF) { continue; } - if (!recordBranch(BF, BI.From.Offset, BI.To.Offset, - BI.Branches, BI.Mispreds)) { + if (!recordBranch(BF, BI.From.Offset, BI.To.Offset, BI.Branches, + BI.Mispreds)) { LLVM_DEBUG(dbgs() << "bad branch : " << BI.From.Offset << " -> " << BI.To.Offset << '\n'); ++MismatchedBranches; @@ -569,10 +563,8 @@ float DataReader::evaluateProfileData(BinaryFunction &BF, // If it's a prefix - skip it. if (Instr && BC.MIB->isPrefix(*Instr)) Instr = BF.getInstructionAtOffset(BI.From.Offset + 1); - if (Instr && - (BC.MIB->isCall(*Instr) || - BC.MIB->isBranch(*Instr) || - BC.MIB->isReturn(*Instr))) { + if (Instr && (BC.MIB->isCall(*Instr) || BC.MIB->isBranch(*Instr) || + BC.MIB->isReturn(*Instr))) { IsValid = true; } } @@ -680,8 +672,8 @@ void DataReader::convertBranchData(BinaryFunction &BF) const { auto setOrUpdateAnnotation = [&](StringRef Name, uint64_t Count) { if (opts::Verbosity >= 1 && BC.MIB->hasAnnotation(*Instr, Name)) { errs() << "BOLT-WARNING: duplicate " << Name << " info for offset 0x" - << Twine::utohexstr(BI.From.Offset) - << " in function " << BF << '\n'; + << Twine::utohexstr(BI.From.Offset) << " in function " << BF + << '\n'; } auto &Value = BC.MIB->getOrCreateAnnotationAs(*Instr, Name); Value += Count; @@ -689,8 +681,8 @@ void DataReader::convertBranchData(BinaryFunction &BF) const { if (BC.MIB->isIndirectCall(*Instr) || BC.MIB->isIndirectBranch(*Instr)) { IndirectCallSiteProfile &CSP = - BC.MIB->getOrCreateAnnotationAs( - *Instr, "CallProfile"); + BC.MIB->getOrCreateAnnotationAs( + *Instr, "CallProfile"); MCSymbol *CalleeSymbol = nullptr; if (BI.To.IsSymbol) { if (BinaryData *BD = BC.getBinaryDataByName(BI.To.Name)) { @@ -707,8 +699,7 @@ void DataReader::convertBranchData(BinaryFunction &BF) const { } } -bool DataReader::recordBranch(BinaryFunction &BF, - uint64_t From, uint64_t To, +bool DataReader::recordBranch(BinaryFunction &BF, uint64_t From, uint64_t To, uint64_t Count, uint64_t Mispreds) const { BinaryContext &BC = BF.getBinaryContext(); @@ -736,8 +727,8 @@ bool DataReader::recordBranch(BinaryFunction &BF, // Very rarely we will see ignored branches. Do a linear check. for (std::pair &Branch : BF.IgnoredBranches) { - if (Branch == std::make_pair(static_cast(From), - static_cast(To))) + if (Branch == + std::make_pair(static_cast(From), static_cast(To))) return true; } @@ -806,8 +797,8 @@ bool DataReader::recordBranch(BinaryFunction &BF, } // For data collected in a bolted binary, we may have created two output BBs // that map to one original block. Branches between these two blocks will - // appear here as one BB jumping to itself, even though it has no loop edges. - // Ignore these. + // appear here as one BB jumping to itself, even though it has no loop + // edges. Ignore these. if (collectedInBoltedBinary() && FromBB == ToBB) return true; @@ -889,8 +880,7 @@ ErrorOr DataReader::parseString(char EndChar, bool EndNl) { // If EndNl was set and nl was found instead of EndChar, do not consume the // new line. - bool EndNlInsteadOfEndChar = - ParsingBuf[StringEnd] == '\n' && EndChar != '\n'; + bool EndNlInsteadOfEndChar = ParsingBuf[StringEnd] == '\n' && EndChar != '\n'; unsigned End = EndNlInsteadOfEndChar ? StringEnd : StringEnd + 1; ParsingBuf = ParsingBuf.drop_front(End); @@ -931,29 +921,28 @@ ErrorOr DataReader::parseHexField(char EndChar, bool EndNl) { return Num; } -ErrorOr DataReader::parseLocation(char EndChar, - bool EndNl, +ErrorOr DataReader::parseLocation(char EndChar, bool EndNl, bool ExpectMemLoc) { // Read whether the location of the branch should be DSO or a symbol // 0 means it is a DSO. 1 means it is a global symbol. 2 means it is a local // symbol. // The symbol flag is also used to tag memory load events by adding 3 to the // base values, i.e. 3 not a symbol, 4 global symbol and 5 local symbol. - if (!ExpectMemLoc && - ParsingBuf[0] != '0' && ParsingBuf[0] != '1' && ParsingBuf[0] != '2') { + if (!ExpectMemLoc && ParsingBuf[0] != '0' && ParsingBuf[0] != '1' && + ParsingBuf[0] != '2') { reportError("expected 0, 1 or 2"); return make_error_code(llvm::errc::io_error); } - if (ExpectMemLoc && - ParsingBuf[0] != '3' && ParsingBuf[0] != '4' && ParsingBuf[0] != '5') { + if (ExpectMemLoc && ParsingBuf[0] != '3' && ParsingBuf[0] != '4' && + ParsingBuf[0] != '5') { reportError("expected 3, 4 or 5"); return make_error_code(llvm::errc::io_error); } bool IsSymbol = - (!ExpectMemLoc && (ParsingBuf[0] == '1' || ParsingBuf[0] == '2')) || - (ExpectMemLoc && (ParsingBuf[0] == '4' || ParsingBuf[0] == '5')); + (!ExpectMemLoc && (ParsingBuf[0] == '1' || ParsingBuf[0] == '2')) || + (ExpectMemLoc && (ParsingBuf[0] == '4' || ParsingBuf[0] == '5')); ParsingBuf = ParsingBuf.drop_front(1); Col += 1; @@ -1092,7 +1081,6 @@ ErrorOr DataReader::maybeParseBATFlag() { return true; } - bool DataReader::hasBranchData() { if (ParsingBuf.size() == 0) return false; @@ -1325,11 +1313,11 @@ fetchMapEntry(MapTy &Map, const std::vector &FuncNames) { } template -std::vector -fetchMapEntriesRegex( - MapTy &Map, - const StringMap> <OCommonNameMap, - const std::vector &FuncNames) { +std::vector fetchMapEntriesRegex( + MapTy &Map, + const StringMap> + <OCommonNameMap, + const std::vector &FuncNames) { std::vector AllData; // Do a reverse order iteration since the name in profile has a higher chance // of matching a name at the end of the list. @@ -1399,8 +1387,7 @@ DataReader::getFuncSampleData(const std::vector &FuncNames) { return fetchMapEntry(NamesToSamples, FuncNames); } -std::vector -DataReader::getBranchDataForNamesRegex( +std::vector DataReader::getBranchDataForNamesRegex( const std::vector &FuncNames) { return fetchMapEntriesRegex(NamesToBranches, LTOCommonNameMap, FuncNames); } @@ -1440,8 +1427,7 @@ void DataReader::dump() const { for (const StringMapEntry &Func : NamesToSamples) { Diag << Func.getKey() << " samples:\n"; for (const SampleInfo &SI : Func.getValue().Data) { - Diag << SI.Loc.Name << " " << SI.Loc.Offset << " " - << SI.Hits << "\n"; + Diag << SI.Loc.Name << " " << SI.Loc.Offset << " " << SI.Hits << "\n"; } } diff --git a/bolt/lib/Profile/Heatmap.cpp b/bolt/lib/Profile/Heatmap.cpp index 14a5ab6bf08d..8b42e6f1e7f7 100644 --- a/bolt/lib/Profile/Heatmap.cpp +++ b/bolt/lib/Profile/Heatmap.cpp @@ -35,8 +35,7 @@ void Heatmap::registerAddressRange(uint64_t StartAddress, uint64_t EndAddress, return; } - if (StartAddress > EndAddress || - EndAddress - StartAddress > 64 * 1024) { + if (StartAddress > EndAddress || EndAddress - StartAddress > 64 * 1024) { LLVM_DEBUG(dbgs() << "invalid range : 0x" << Twine::utohexstr(StartAddress) << " -> 0x" << Twine::utohexstr(EndAddress) << '\n'); ++NumSkippedRanges; @@ -135,21 +134,15 @@ void Heatmap::print(raw_ostream &OS) const { }; static raw_ostream::Colors Colors[] = { - raw_ostream::WHITE, - raw_ostream::WHITE, - raw_ostream::CYAN, - raw_ostream::GREEN, - raw_ostream::YELLOW, - raw_ostream::RED - }; + raw_ostream::WHITE, raw_ostream::WHITE, raw_ostream::CYAN, + raw_ostream::GREEN, raw_ostream::YELLOW, raw_ostream::RED}; constexpr size_t NumRanges = sizeof(Colors) / sizeof(Colors[0]); uint64_t Range[NumRanges]; for (uint64_t I = 0; I < NumRanges; ++I) - Range[I] = std::max(I + 1, - (uint64_t) std::pow((double) MaxValue, - (double) (I + 1) / NumRanges)); - Range[NumRanges - 1] = std::max((uint64_t) NumRanges, MaxValue); + Range[I] = std::max(I + 1, (uint64_t)std::pow((double)MaxValue, + (double)(I + 1) / NumRanges)); + Range[NumRanges - 1] = std::max((uint64_t)NumRanges, MaxValue); // Print scaled value auto printValue = [&](uint64_t Value, bool ResetColor = false) { diff --git a/bolt/lib/Profile/ProfileReaderBase.cpp b/bolt/lib/Profile/ProfileReaderBase.cpp index 001ccc6fdfe7..5697a1ed0591 100644 --- a/bolt/lib/Profile/ProfileReaderBase.cpp +++ b/bolt/lib/Profile/ProfileReaderBase.cpp @@ -19,5 +19,5 @@ bool ProfileReaderBase::mayHaveProfileData(const BinaryFunction &BF) { return true; } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp index 36de1a23274a..cb135998aada 100644 --- a/bolt/lib/Profile/YAMLProfileReader.cpp +++ b/bolt/lib/Profile/YAMLProfileReader.cpp @@ -80,8 +80,7 @@ bool YAMLProfileReader::hasLocalsWithFileName() const { } bool YAMLProfileReader::parseFunctionProfile( - BinaryFunction &BF, - const yaml::bolt::BinaryFunctionProfile &YamlBF) { + BinaryFunction &BF, const yaml::bolt::BinaryFunctionProfile &YamlBF) { BinaryContext &BC = BF.getBinaryContext(); bool ProfileMatched = true; @@ -148,8 +147,8 @@ bool YAMLProfileReader::parseFunctionProfile( if (IsFunction) { CalleeSymbol = Callee->getSymbolForEntryID(YamlCSI.EntryDiscriminator); } - BF.getAllCallSites().emplace_back( - CalleeSymbol, YamlCSI.Count, YamlCSI.Mispreds, YamlCSI.Offset); + BF.getAllCallSites().emplace_back(CalleeSymbol, YamlCSI.Count, + YamlCSI.Mispreds, YamlCSI.Offset); if (YamlCSI.Offset >= BB.getOriginalSize()) { if (opts::Verbosity >= 2) diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index 1f5ab511dad4..cc6a20818b42 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -25,8 +25,8 @@ namespace llvm { namespace bolt { namespace { -void -convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { +void convert(const BinaryFunction &BF, + yaml::bolt::BinaryFunctionProfile &YamlBF) { const BinaryContext &BC = BF.getBinaryContext(); const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR; @@ -62,9 +62,8 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { CSI.Offset = Offset.get() - BB->getInputOffset(); if (BC.MIB->isIndirectCall(Instr) || BC.MIB->isIndirectBranch(Instr)) { - auto ICSP = - BC.MIB->tryGetAnnotationAs(Instr, - "CallProfile"); + auto ICSP = BC.MIB->tryGetAnnotationAs( + Instr, "CallProfile"); if (!ICSP) continue; for (const IndirectCallProfile &CSP : ICSP.get()) { @@ -86,17 +85,17 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { const BinaryFunction *const Callee = BC.getFunctionForSymbol(CalleeSymbol, &EntryID); if (Callee) { - CSI.DestId = Callee->getFunctionNumber();; + CSI.DestId = Callee->getFunctionNumber(); CSI.EntryDiscriminator = EntryID; } if (BC.MIB->getConditionalTailCall(Instr)) { auto CTCCount = - BC.MIB->tryGetAnnotationAs(Instr, "CTCTakenCount"); + BC.MIB->tryGetAnnotationAs(Instr, "CTCTakenCount"); if (CTCCount) { CSI.Count = *CTCCount; auto CTCMispreds = - BC.MIB->tryGetAnnotationAs(Instr, "CTCMispredCount"); + BC.MIB->tryGetAnnotationAs(Instr, "CTCMispredCount"); if (CTCMispreds) CSI.Mispreds = *CTCMispreds; } @@ -115,8 +114,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { // Skip printing if there's no profile data for non-entry basic block. // Include landing pads with non-zero execution count. - if (YamlBB.CallSites.empty() && - !BB->isEntryPoint() && + if (YamlBB.CallSites.empty() && !BB->isEntryPoint() && !(BB->isLandingPad() && BB->getKnownExecutionCount() != 0)) { uint64_t SuccessorExecCount = 0; for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo : @@ -144,8 +142,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { } } // end anonymous namespace -std::error_code -YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { +std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { const BinaryContext &BC = RI.getBinaryContext(); const auto &Functions = BC.getBinaryFunctions(); diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp index 8a5d70dca918..ec29abb09663 100644 --- a/bolt/lib/Rewrite/BinaryPassManager.cpp +++ b/bolt/lib/Rewrite/BinaryPassManager.cpp @@ -324,8 +324,7 @@ namespace bolt { using namespace opts; -const char BinaryFunctionPassManager::TimerGroupName[] = - "passman"; +const char BinaryFunctionPassManager::TimerGroupName[] = "passman"; const char BinaryFunctionPassManager::TimerGroupDesc[] = "Binary Function Pass Manager"; @@ -348,23 +347,16 @@ void BinaryFunctionPassManager::runPasses() { NamedRegionTimer T(Pass->getName(), Pass->getName(), TimerGroupName, TimerGroupDesc, TimeOpts); - callWithDynoStats( - [this,&Pass] { - Pass->runOnFunctions(BC); - }, - BFs, - Pass->getName(), - opts::DynoStatsAll - ); + callWithDynoStats([this, &Pass] { Pass->runOnFunctions(BC); }, BFs, + Pass->getName(), opts::DynoStatsAll); if (opts::VerifyCFG && !std::accumulate( - BFs.begin(), BFs.end(), - true, - [](const bool Valid, - const std::pair &It) { - return Valid && It.second.validateCFG(); - })) { + BFs.begin(), BFs.end(), true, + [](const bool Valid, + const std::pair &It) { + return Valid && It.second.validateCFG(); + })) { errs() << "BOLT-ERROR: Invalid CFG detected after pass " << Pass->getName() << "\n"; exit(1); @@ -442,8 +434,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { opts::JTFootprintReductionFlag); Manager.registerPass( - std::make_unique(PrintSimplifyROLoads), - opts::SimplifyRODataLoads); + std::make_unique(PrintSimplifyROLoads), + opts::SimplifyRODataLoads); Manager.registerPass(std::make_unique(PrintRegReAssign), opts::RegReAssign); @@ -460,9 +452,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { Manager.registerPass(std::make_unique(PrintReordered)); - Manager.registerPass( - std::make_unique(PrintUCE), - opts::EliminateUnreachable); + Manager.registerPass(std::make_unique(PrintUCE), + opts::EliminateUnreachable); Manager.registerPass(std::make_unique(PrintSplit)); @@ -480,13 +471,13 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { // size of a function to determine the order. It should definitely // also happen after any changes to the call graph are made, e.g. inlining. Manager.registerPass( - std::make_unique(PrintReorderedFunctions)); + std::make_unique(PrintReorderedFunctions)); // Print final dyno stats right while CFG and instruction analysis are intact. Manager.registerPass( - std::make_unique( - InitialDynoStats, "after all optimizations before SCTC and FOP"), - opts::PrintDynoStats | opts::DynoStatsAll); + std::make_unique( + InitialDynoStats, "after all optimizations before SCTC and FOP"), + opts::PrintDynoStats | opts::DynoStatsAll); // Add the StokeInfo pass, which extract functions for stoke optimization and // get the liveness information for them @@ -547,7 +538,7 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { // function reordering. It's unsafe to use any CFG or instruction analysis // after this point. Manager.registerPass( - std::make_unique(PrintAfterLowering)); + std::make_unique(PrintAfterLowering)); // In non-relocation mode, mark functions that do not fit into their original // space as non-simple if we have to (e.g. for correct debug info update). diff --git a/bolt/lib/Rewrite/BoltDiff.cpp b/bolt/lib/Rewrite/BoltDiff.cpp index 85797b2727c4..cff3f450f508 100644 --- a/bolt/lib/Rewrite/BoltDiff.cpp +++ b/bolt/lib/Rewrite/BoltDiff.cpp @@ -186,23 +186,25 @@ class RewriteInstanceDiff { double getNormalizedScore(const BinaryFunction &Function, const RewriteInstance &Ctx) { if (!opts::NormalizeByBin1) - return static_cast(Function.getFunctionScore()) / Ctx.getTotalScore(); - return static_cast(Function.getFunctionScore()) / RI1.getTotalScore(); + return static_cast(Function.getFunctionScore()) / + Ctx.getTotalScore(); + return static_cast(Function.getFunctionScore()) / + RI1.getTotalScore(); } double getNormalizedScore(const BinaryBasicBlock &BB, const RewriteInstance &Ctx) { if (!opts::NormalizeByBin1) - return static_cast(BB.getKnownExecutionCount()) / Ctx.getTotalScore(); - return static_cast(BB.getKnownExecutionCount()) / RI1.getTotalScore(); + return static_cast(BB.getKnownExecutionCount()) / + Ctx.getTotalScore(); + return static_cast(BB.getKnownExecutionCount()) / + RI1.getTotalScore(); } double getNormalizedScore(BinaryBasicBlock::branch_info_iterator BIIter, const RewriteInstance &Ctx) { double Score = - BIIter->Count == BinaryBasicBlock::COUNT_NO_PROFILE - ? 0 - : BIIter->Count; + BIIter->Count == BinaryBasicBlock::COUNT_NO_PROFILE ? 0 : BIIter->Count; if (!opts::NormalizeByBin1) return Score / Ctx.getTotalScore(); return Score / RI1.getTotalScore(); @@ -277,7 +279,7 @@ class RewriteInstanceDiff { } if (Match || !Function2.hasCFG()) continue; - auto Iter = HashLookup.find(Function2.computeHash(/*UseDFS*/true)); + auto Iter = HashLookup.find(Function2.computeHash(/*UseDFS*/ true)); if (Iter != HashLookup.end()) { FuncMap.insert(std::make_pair<>(&Function2, Iter->second)); Bin1MappedFuncs.insert(Iter->second); @@ -340,8 +342,7 @@ class RewriteInstanceDiff { return false; while (Iter1 != BB1.end()) { - if (Iter2 == BB2.end() || - Iter1->getOpcode() != Iter2->getOpcode()) + if (Iter2 == BB2.end() || Iter1->getOpcode() != Iter2->getOpcode()) return false; ++Iter1; @@ -464,9 +465,8 @@ class RewriteInstanceDiff { void reportHottestEdgeDiffs() { unsigned Printed = 0; setTitleColor(); - outs() - << "\nTop " << opts::DisplayCount - << " largest differences in edge hotness bin 2 -> bin 1:\n"; + outs() << "\nTop " << opts::DisplayCount + << " largest differences in edge hotness bin 2 -> bin 1:\n"; outs() << "=========================================================\n"; setRegularColor(); outs() << " * Functions with different contents do not appear here\n"; @@ -494,7 +494,7 @@ class RewriteInstanceDiff { if (Printed++ == opts::DisplayCount) break; } - } + } /// For LTO functions sharing the same prefix (for example, func1.lto_priv.1 /// and func1.lto_priv.2 share the func1.lto_priv prefix), compute aggregated @@ -527,7 +527,7 @@ class RewriteInstanceDiff { UnmappedScore += Score; } int64_t Unmapped = - RI2.BC->getBinaryFunctions().size() - Bin2MappedFuncs.size(); + RI2.BC->getBinaryFunctions().size() - Bin2MappedFuncs.size(); outs() << "BOLT-DIFF: " << Unmapped << " functions in Binary2 have no correspondence to any other " "function in Binary1.\n"; @@ -572,7 +572,7 @@ class RewriteInstanceDiff { &MapEntry = I->second; if (opts::IgnoreUnchanged && MapEntry.second->computeHash(/*UseDFS=*/true) == - MapEntry.first->computeHash(/*UseDFS=*/true)) + MapEntry.first->computeHash(/*UseDFS=*/true)) continue; const std::pair &Scores = ScoreMap[MapEntry.first]; outs() << "Function " << MapEntry.first->getDemangledName(); @@ -691,8 +691,9 @@ void RewriteInstance::compare(RewriteInstance &RI2) { outs() << "Trace for binary 2 has " << RI2.getTotalScore() << " instructions executed.\n"; if (opts::NormalizeByBin1) { - double Diff2to1 = static_cast(RI2.getTotalScore() - this->getTotalScore()) / - this->getTotalScore(); + double Diff2to1 = + static_cast(RI2.getTotalScore() - this->getTotalScore()) / + this->getTotalScore(); outs() << "Binary2 change in score with respect to Binary1: "; printColoredPercentage(Diff2to1 * 100.0); outs() << "\n"; diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index f20bed835d46..69fe758d9837 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -400,9 +400,11 @@ void DWARFRewriter::updateUnitDebugInfo(uint64_t CUIndex, DWARFUnit &Unit, uint64_t RangesSectionOffset = RangesSectionWriter->getEmptyRangesOffset(); Expected RangesOrError = DIE.getAddressRanges(); - const BinaryFunction *Function = RangesOrError && !RangesOrError->empty() - ? BC.getBinaryFunctionContainingAddress(RangesOrError->front().LowPC) - : nullptr; + const BinaryFunction *Function = + RangesOrError && !RangesOrError->empty() + ? BC.getBinaryFunctionContainingAddress( + RangesOrError->front().LowPC) + : nullptr; if (Function) { DebugAddressRangesVector OutputRanges = Function->translateInputToOutputRanges(*RangesOrError); @@ -431,8 +433,8 @@ void DWARFRewriter::updateUnitDebugInfo(uint64_t CUIndex, DWARFUnit &Unit, if (Value.isFormClass(DWARFFormValue::FC_Constant) || Value.isFormClass(DWARFFormValue::FC_SectionOffset)) { uint64_t Offset = Value.isFormClass(DWARFFormValue::FC_Constant) - ? Value.getAsUnsignedConstant().getValue() - : Value.getAsSectionOffset().getValue(); + ? Value.getAsUnsignedConstant().getValue() + : Value.getAsSectionOffset().getValue(); DebugLocationsVector InputLL; Optional SectionAddress = @@ -712,7 +714,7 @@ void DWARFRewriter::updateLineTableOffsets(const MCAsmLayout &Layout) { // Set .debug_info as finalized so it won't be skipped over when // we process sections while writing out the new binary. This ensures // that the pending relocations will be processed and not ignored. - if(DbgInfoSection) + if (DbgInfoSection) DbgInfoSection->setIsFinalized(); if (TypeInfoSection) @@ -726,15 +728,15 @@ void DWARFRewriter::finalizeDebugSections( SmallVector ARangesBuffer; raw_svector_ostream OS(ARangesBuffer); - auto MAB = std::unique_ptr(BC.TheTarget->createMCAsmBackend( - *BC.STI, *BC.MRI, MCTargetOptions())); + auto MAB = std::unique_ptr( + BC.TheTarget->createMCAsmBackend(*BC.STI, *BC.MRI, MCTargetOptions())); ARangesSectionWriter->writeARangesSection(OS); const StringRef &ARangesContents = OS.str(); BC.registerOrUpdateNoteSection(".debug_aranges", - copyByteArray(ARangesContents), - ARangesContents.size()); + copyByteArray(ARangesContents), + ARangesContents.size()); } if (StrWriter->isInitialized()) { diff --git a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp index 0343a8766569..e77a88ac49db 100644 --- a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp +++ b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp @@ -38,19 +38,16 @@ uint8_t *ExecutableFileMemoryManager::allocateSection(intptr_t Size, return DataCopy; } - if (!IsCode && - (SectionName == ".strtab" || - SectionName == ".symtab" || - SectionName == "" || - SectionName.startswith(".rela."))) { + if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" || + SectionName == "" || SectionName.startswith(".rela."))) { return SectionMemoryManager::allocateDataSection(Size, Alignment, SectionID, SectionName, IsReadOnly); } uint8_t *Ret; if (IsCode) { - Ret = SectionMemoryManager::allocateCodeSection(Size, Alignment, - SectionID, SectionName); + Ret = SectionMemoryManager::allocateCodeSection(Size, Alignment, SectionID, + SectionName); } else { Ret = SectionMemoryManager::allocateDataSection(Size, Alignment, SectionID, SectionName, IsReadOnly); @@ -93,8 +90,8 @@ bool ExecutableFileMemoryManager::finalizeMemory(std::string *ErrMsg) { return SectionMemoryManager::finalizeMemory(ErrMsg); } -ExecutableFileMemoryManager::~ExecutableFileMemoryManager() { } +ExecutableFileMemoryManager::~ExecutableFileMemoryManager() {} -} +} // namespace bolt -} +} // namespace llvm diff --git a/bolt/lib/Rewrite/MachORewriteInstance.cpp b/bolt/lib/Rewrite/MachORewriteInstance.cpp index 4c9357741833..ce6ec2ebdc78 100644 --- a/bolt/lib/Rewrite/MachORewriteInstance.cpp +++ b/bolt/lib/Rewrite/MachORewriteInstance.cpp @@ -82,7 +82,6 @@ MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch, } // anonymous namespace -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile, diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 7f6c3d14d952..5c226d4fb1ca 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -336,12 +336,11 @@ extern MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *, namespace { bool refersToReorderedSection(ErrorOr Section) { - auto Itr = std::find_if(opts::ReorderData.begin(), - opts::ReorderData.end(), - [&](const std::string &SectionName) { - return (Section && - Section->getName() == SectionName); - }); + auto Itr = + std::find_if(opts::ReorderData.begin(), opts::ReorderData.end(), + [&](const std::string &SectionName) { + return (Section && Section->getName() == SectionName); + }); return Itr != opts::ReorderData.end(); } @@ -412,9 +411,10 @@ Error RewriteInstance::setProfile(StringRef Filename) { if (ProfileReader) { // Already exists - return make_error( - Twine("multiple profiles specified: ") + ProfileReader->getFilename() + - " and " + Filename, inconvertibleErrorCode()); + return make_error(Twine("multiple profiles specified: ") + + ProfileReader->getFilename() + " and " + + Filename, + inconvertibleErrorCode()); } // Spawn a profile reader based on file contents. @@ -489,7 +489,7 @@ void RewriteInstance::discoverStorage() { StringRef SectionContents = cantFail(Section.getContents()); BC->OldTextSectionOffset = - SectionContents.data() - InputFile->getData().data(); + SectionContents.data() - InputFile->getData().data(); } if (!opts::HeatmapMode && @@ -529,8 +529,9 @@ void RewriteInstance::discoverStorage() { } else { NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress; } - assert(NextAvailableOffset == NextAvailableAddress - BC->FirstAllocAddress - && "PHDR table address calculation error"); + assert(NextAvailableOffset == + NextAvailableAddress - BC->FirstAllocAddress && + "PHDR table address calculation error"); outs() << "BOLT-INFO: creating new program header table at address 0x" << Twine::utohexstr(NextAvailableAddress) << ", offset 0x" @@ -544,7 +545,7 @@ void RewriteInstance::discoverStorage() { Phnum += 3; NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy); - NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy); + NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy); } // Align at cache line. @@ -803,7 +804,7 @@ void RewriteInstance::discoverFileObjects() { // For local symbols we want to keep track of associated FILE symbol name for // disambiguation by combined name. - StringRef FileSymbolName; + StringRef FileSymbolName; bool SeenFileName = false; struct SymbolRefHash { size_t operator()(SymbolRef const &S) const { @@ -858,29 +859,27 @@ void RewriteInstance::discoverFileObjects() { }; std::vector SortedFileSymbols; std::copy_if(InputFile->symbol_begin(), InputFile->symbol_end(), - std::back_inserter(SortedFileSymbols), - isSymbolInMemory); + std::back_inserter(SortedFileSymbols), isSymbolInMemory); - std::stable_sort(SortedFileSymbols.begin(), SortedFileSymbols.end(), - [](const SymbolRef &A, const SymbolRef &B) { - // FUNC symbols have the highest precedence, while SECTIONs - // have the lowest. - uint64_t AddressA = cantFail(A.getAddress()); - uint64_t AddressB = cantFail(B.getAddress()); - if (AddressA != AddressB) - return AddressA < AddressB; + std::stable_sort( + SortedFileSymbols.begin(), SortedFileSymbols.end(), + [](const SymbolRef &A, const SymbolRef &B) { + // FUNC symbols have the highest precedence, while SECTIONs + // have the lowest. + uint64_t AddressA = cantFail(A.getAddress()); + uint64_t AddressB = cantFail(B.getAddress()); + if (AddressA != AddressB) + return AddressA < AddressB; - SymbolRef::Type AType = cantFail(A.getType()); - SymbolRef::Type BType = cantFail(B.getType()); - if (AType == SymbolRef::ST_Function && - BType != SymbolRef::ST_Function) - return true; - if (BType == SymbolRef::ST_Debug && - AType != SymbolRef::ST_Debug) - return true; + SymbolRef::Type AType = cantFail(A.getType()); + SymbolRef::Type BType = cantFail(B.getType()); + if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function) + return true; + if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug) + return true; - return false; - }); + return false; + }); // For aarch64, the ABI defines mapping symbols so we identify data in the // code section (see IHI0056B). $d identifies data contents. @@ -985,8 +984,8 @@ void RewriteInstance::discoverFileObjects() { auto registerName = [&](uint64_t FinalSize) { // Register names even if it's not a function, e.g. for an entry point. - BC->registerNameAtAddress(UniqueName, Address, FinalSize, - SymbolAlignment, SymbolFlags); + BC->registerNameAtAddress(UniqueName, Address, FinalSize, SymbolAlignment, + SymbolFlags); if (!AlternativeName.empty()) BC->registerNameAtAddress(AlternativeName, Address, FinalSize, SymbolAlignment, SymbolFlags); @@ -1043,8 +1042,7 @@ void RewriteInstance::discoverFileObjects() { } } - if (PreviousFunction && - PreviousFunction->containsAddress(Address) && + if (PreviousFunction && PreviousFunction->containsAddress(Address) && PreviousFunction->getAddress() != Address) { if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) { if (opts::Verbosity >= 1) { @@ -1057,8 +1055,8 @@ void RewriteInstance::discoverFileObjects() { registerName(0); - PreviousFunction-> - addEntryPointAtOffset(Address - PreviousFunction->getAddress()); + PreviousFunction->addEntryPointAtOffset(Address - + PreviousFunction->getAddress()); // Remove the symbol from FileSymRefs so that we can skip it from // in the future. @@ -1125,8 +1123,8 @@ void RewriteInstance::discoverFileObjects() { errs() << "BOLT-WARNING: size mismatch for duplicate entries " << *BF << " and " << UniqueName << '\n'; } - outs() << "BOLT-INFO: adjusting size of function " << *BF - << " old " << BF->getSize() << " new " << SymbolSize << "\n"; + outs() << "BOLT-INFO: adjusting size of function " << *BF << " old " + << BF->getSize() << " new " << SymbolSize << "\n"; } BF->setSize(std::max(SymbolSize, BF->getSize())); BC->setBinaryDataSize(Address, BF->getSize()); @@ -1137,8 +1135,7 @@ void RewriteInstance::discoverFileObjects() { // Skip symbols from invalid sections if (!Section) { errs() << "BOLT-WARNING: " << UniqueName << " (0x" - << Twine::utohexstr(Address) - << ") does not have any section\n"; + << Twine::utohexstr(Address) << ") does not have any section\n"; continue; } assert(Section && "section for functions must be registered"); @@ -1184,14 +1181,14 @@ void RewriteInstance::discoverFileObjects() { } if (opts::Verbosity >= 1) { - errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Address) - << ", 0x" << Twine::utohexstr(Address + FDE->getAddressRange()) + errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Address) << ", 0x" + << Twine::utohexstr(Address + FDE->getAddressRange()) << ") has no corresponding symbol table entry\n"; } ErrorOr Section = BC->getSectionForAddress(Address); assert(Section && "cannot get section for address from FDE"); std::string FunctionName = - "__BOLT_FDE_FUNCat" + Twine::utohexstr(Address).str(); + "__BOLT_FDE_FUNCat" + Twine::utohexstr(Address).str(); BC->createBinaryFunction(FunctionName, *Section, Address, FDE->getAddressRange()); } @@ -1443,8 +1440,7 @@ void RewriteInstance::relocateEHFrameSection() { EHFrameSection->addRelocation(Offset, nullptr, RelType, Value); }; - Error E = - EHFrameParser::parse(DE, EHFrameSection->getAddress(), createReloc); + Error E = EHFrameParser::parse(DE, EHFrameSection->getAddress(), createReloc); check_error(std::move(E), "failed to patch EH frame"); } @@ -1453,9 +1449,7 @@ ArrayRef RewriteInstance::getLSDAData() { LSDASection->getContents().size()); } -uint64_t RewriteInstance::getLSDAAddress() { - return LSDASection->getAddress(); -} +uint64_t RewriteInstance::getLSDAAddress() { return LSDASection->getAddress(); } void RewriteInstance::readSpecialSections() { NamedRegionTimer T("readSpecialSections", "read special sections", @@ -1507,7 +1501,7 @@ void RewriteInstance::readSpecialSections() { if (!opts::HeatmapMode) { if (std::error_code EC = BAT->parse(BATSec->getContents())) { errs() << "BOLT-ERROR: failed to parse BOLT address translation " - "table.\n"; + "table.\n"; exit(1); } } @@ -1524,8 +1518,8 @@ void RewriteInstance::readSpecialSections() { exit(1); } - BC->HasRelocations = HasTextRelocations && - (opts::RelocationMode != cl::BOU_FALSE); + BC->HasRelocations = + HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE); // Force non-relocation mode for heatmap generation if (opts::HeatmapMode) { @@ -1648,13 +1642,12 @@ void RewriteInstance::adjustCommandLineOptions() { opts::UseOldText = false; } - if (!opts::AlignText.getNumOccurrences()) { opts::AlignText = BC->PageAlign; } - if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && - !opts::StrictMode && !opts::UseOldText) { + if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode && + !opts::UseOldText) { opts::Lite = true; } @@ -1692,7 +1685,8 @@ int64_t getRelocationAddend(const ELFObjectFile *Obj, DataRefImpl Rel = RelRef.getRawDataRefImpl(); const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a)); switch (RelocationSection->sh_type) { - default: llvm_unreachable("unexpected relocation section type"); + default: + llvm_unreachable("unexpected relocation section type"); case ELF::SHT_REL: break; case ELF::SHT_RELA: { @@ -1706,7 +1700,7 @@ int64_t getRelocationAddend(const ELFObjectFile *Obj, } int64_t getRelocationAddend(const ELFObjectFileBase *Obj, - const RelocationRef &Rel) { + const RelocationRef &Rel) { if (auto *ELF32LE = dyn_cast(Obj)) return getRelocationAddend(ELF32LE, Rel); if (auto *ELF64LE = dyn_cast(Obj)) @@ -1736,9 +1730,7 @@ bool RewriteInstance::analyzeRelocation( if ((Skip = Relocation::skipRelocationProcess(RType, *Value))) return true; - ExtractedValue = Relocation::extractValue(RType, - *Value, - Rel.getOffset()); + ExtractedValue = Relocation::extractValue(RType, *Value, Rel.getOffset()); Addend = getRelocationAddend(InputFile, Rel); const bool IsPCRelative = Relocation::isPCRelative(RType); @@ -1798,8 +1790,8 @@ bool RewriteInstance::analyzeRelocation( } else if (!SymbolAddress) { assert(!IsSectionRelocation); if (ExtractedValue || Addend == 0 || IsPCRelative) { - SymbolAddress = truncateToSize(ExtractedValue - Addend + PCRelOffset, - RelSize); + SymbolAddress = + truncateToSize(ExtractedValue - Addend + PCRelOffset, RelSize); } else { // This is weird case. The extracted value is zero but the addend is // non-zero and the relocation is not pc-rel. Using the previous logic, @@ -1887,9 +1879,8 @@ void RewriteInstance::processRelocations() { void RewriteInstance::insertLKMarker(uint64_t PC, uint64_t SectionOffset, int32_t PCRelativeOffset, bool IsPCRelative, StringRef SectionName) { - BC->LKMarkers[PC].emplace_back( - LKInstructionMarkerInfo{SectionOffset, PCRelativeOffset, IsPCRelative, - SectionName}); + BC->LKMarkers[PC].emplace_back(LKInstructionMarkerInfo{ + SectionOffset, PCRelativeOffset, IsPCRelative, SectionName}); } void RewriteInstance::processLKSections() { @@ -1991,8 +1982,8 @@ void RewriteInstance::processLKPCIFixup() { BinaryFunction *HookupFunction = BC->getBinaryFunctionAtAddress(HookupAddress); assert(HookupFunction && "expected function for entry in .pci_fixup"); - BC->addRelocation(PC, HookupFunction->getSymbol(), - Relocation::getPC32(), 0, *Offset); + BC->addRelocation(PC, HookupFunction->getSymbol(), Relocation::getPC32(), 0, + *Offset); } } @@ -2062,7 +2053,6 @@ void RewriteInstance::processLKBugTable() { } } - /// .smp_locks section contains PC-relative references to instructions with LOCK /// prefix. The prefix can be converted to NOP at boot time on non-SMP systems. void RewriteInstance::processLKSMPLocks() { @@ -2255,19 +2245,15 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { const uint64_t Address = SymbolAddress + Addend; - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "; - printRelocationInfo(Rel, - SymbolName, - SymbolAddress, - Addend, - ExtractedValue)); + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "; printRelocationInfo( + Rel, SymbolName, SymbolAddress, Addend, ExtractedValue)); BinaryFunction *ContainingBF = nullptr; if (IsFromCode) { ContainingBF = - BC->getBinaryFunctionContainingAddress(Rel.getOffset(), - /*CheckPastEnd*/ false, - /*UseMaxSize*/ true); + BC->getBinaryFunctionContainingAddress(Rel.getOffset(), + /*CheckPastEnd*/ false, + /*UseMaxSize*/ true); assert(ContainingBF && "cannot find function for address in code"); if (!IsAArch64 && !ContainingBF->containsAddress(Rel.getOffset())) { if (opts::Verbosity >= 1) { @@ -2329,19 +2315,16 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { // It's possible we are referencing a function without referencing any // code, e.g. when taking a bitmask action on a function address. errs() << "BOLT-WARNING: non-standard function reference (e.g. " - "bitmask) detected against function " << *BF; + "bitmask) detected against function " + << *BF; if (IsFromCode) { errs() << " from function " << *ContainingBF << '\n'; } else { errs() << " from data section at 0x" << Twine::utohexstr(Rel.getOffset()) << '\n'; } - LLVM_DEBUG(printRelocationInfo(Rel, - SymbolName, - SymbolAddress, - Addend, - ExtractedValue) - ); + LLVM_DEBUG(printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend, + ExtractedValue)); ReferencedBF = BF; } } @@ -2400,11 +2383,11 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { if (RefFunctionOffset) { if (ContainingBF && ContainingBF != ReferencedBF) { ReferencedSymbol = - ReferencedBF->addEntryPointAtOffset(RefFunctionOffset); + ReferencedBF->addEntryPointAtOffset(RefFunctionOffset); } else { ReferencedSymbol = - ReferencedBF->getOrCreateLocalLabel(Address, - /*CreatePastEnd =*/ true); + ReferencedBF->getOrCreateLocalLabel(Address, + /*CreatePastEnd =*/true); ReferencedBF->registerReferencedOffset(RefFunctionOffset); } if (opts::Verbosity > 1 && @@ -2448,8 +2431,7 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { // of sections and whose symbol address is not really what should be // encoded in the instruction). So we essentially disabled this check // for AArch64 and live with bogus names for objects. - assert((IsAArch64 || - IsSectionRelocation || + assert((IsAArch64 || IsSectionRelocation || BD->nameStartsWith(SymbolName) || BD->nameStartsWith("PG" + SymbolName) || (BD->nameStartsWith("ANONYMOUS") && @@ -2469,14 +2451,14 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { } else { // These are mostly local data symbols but undefined symbols // in relocation sections can get through here too, from .plt. - assert((IsAArch64 || - IsSectionRelocation || - BC->getSectionNameForAddress(SymbolAddress)->startswith(".plt")) - && "known symbols should not resolve to anonymous locals"); + assert( + (IsAArch64 || IsSectionRelocation || + BC->getSectionNameForAddress(SymbolAddress)->startswith(".plt")) && + "known symbols should not resolve to anonymous locals"); if (IsSectionRelocation) { - ReferencedSymbol = BC->getOrCreateGlobalSymbol(SymbolAddress, - "SYMBOLat"); + ReferencedSymbol = + BC->getOrCreateGlobalSymbol(SymbolAddress, "SYMBOLat"); } else { SymbolRef Symbol = *Rel.getSymbol(); const uint64_t SymbolSize = @@ -2488,18 +2470,15 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { if (SymbolFlags & SymbolRef::SF_Global) { Name = SymbolName; } else { - if (StringRef(SymbolName).startswith( - BC->AsmInfo->getPrivateGlobalPrefix())) { + if (StringRef(SymbolName) + .startswith(BC->AsmInfo->getPrivateGlobalPrefix())) { Name = NR.uniquify("PG" + SymbolName); } else { Name = NR.uniquify(SymbolName); } } - ReferencedSymbol = BC->registerNameAtAddress(Name, - SymbolAddress, - SymbolSize, - SymbolAlignment, - SymbolFlags); + ReferencedSymbol = BC->registerNameAtAddress( + Name, SymbolAddress, SymbolSize, SymbolAlignment, SymbolFlags); } if (IsSectionRelocation) { @@ -2528,11 +2507,8 @@ void RewriteInstance::readRelocations(const SectionRef &Section) { ForceRelocation = true; if (IsFromCode) { - ContainingBF->addRelocation(Rel.getOffset(), - ReferencedSymbol, - RType, - Addend, - ExtractedValue); + ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, + Addend, ExtractedValue); } else if (IsToCode || ForceRelocation) { BC->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, Addend, ExtractedValue); @@ -2595,9 +2571,8 @@ void RewriteInstance::selectFunctionsToProcess() { outs() << "BOLT-INFO: limiting processing to functions with at least " << LiteThresholdExecCount << " invocations\n"; } - LiteThresholdExecCount = - std::max(LiteThresholdExecCount, - static_cast(opts::LiteThresholdCount)); + LiteThresholdExecCount = std::max( + LiteThresholdExecCount, static_cast(opts::LiteThresholdCount)); uint64_t NumFunctionsToProcess = 0; auto shouldProcess = [&](const BinaryFunction &Function) { @@ -2689,8 +2664,7 @@ void RewriteInstance::preprocessProfileData() { if (Error E = ProfileReader->preprocessProfile(*BC.get())) report_error("cannot pre-process profile", std::move(E)); - if (!BC->hasSymbolsWithFileName() && - ProfileReader->hasLocalsWithFileName() && + if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName() && !opts::AllowStripped) { errs() << "BOLT-ERROR: input binary does not have local file symbols " "but profile data includes function names with embedded file " @@ -2756,7 +2730,7 @@ void RewriteInstance::disassembleFunctions() { // Offset of the function in the file. const auto *FileBegin = - reinterpret_cast(InputFile->getData().data()); + reinterpret_cast(InputFile->getData().data()); Function.setFileOffset(FunctionData->begin() - FileBegin); if (!shouldDisassemble(Function)) { @@ -2821,8 +2795,8 @@ void RewriteInstance::disassembleFunctions() { if (BC->HasRelocations) { BC->exitWithBugReport("unable to fill CFI.", Function); } else { - errs() << "BOLT-WARNING: unable to fill CFI for function " - << Function << ". Skipping.\n"; + errs() << "BOLT-WARNING: unable to fill CFI for function " << Function + << ". Skipping.\n"; Function.setSimple(false); continue; } @@ -2852,10 +2826,9 @@ void RewriteInstance::buildFunctionsCFG() { BF.print(outs(), "while building cfg", true); }; - ParallelUtilities::PredicateTy SkipPredicate = - [&](const BinaryFunction &BF) { - return !shouldDisassemble(BF) || !BF.isSimple(); - }; + ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) { + return !shouldDisassemble(BF) || !BF.isSimple(); + }; ParallelUtilities::runOnEachFunctionWithUniqueAllocId( *BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, @@ -2907,6 +2880,7 @@ namespace { class BOLTSymbolResolver : public JITSymbolResolver { BinaryContext &BC; + public: BOLTSymbolResolver(BinaryContext &BC) : BC(BC) {} @@ -2979,7 +2953,7 @@ void RewriteInstance::emitAndLink() { check_error(EC, "cannot create output object file"); std::unique_ptr BOS = - std::make_unique(TempOut->os()); + std::make_unique(TempOut->os()); raw_pwrite_stream *OS = BOS.get(); // Implicitly MCObjectStreamer takes ownership of MCAsmBackend (MAB) @@ -3016,7 +2990,7 @@ void RewriteInstance::emitAndLink() { BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC); MCAsmLayout FinalLayout( - static_cast(Streamer.get())->getAssembler()); + static_cast(Streamer.get())->getAssembler()); RTDyld.reset(new decltype(RTDyld)::element_type(*BC->EFMM, Resolver)); RTDyld->setProcessAllSections(false); @@ -3051,8 +3025,8 @@ void RewriteInstance::emitAndLink() { // output sections and de-register sections used for emission. for (BinaryFunction *Function : BC->getAllBinaryFunctions()) { ErrorOr Section = Function->getCodeSection(); - if (Section && (Function->getImageAddress() == 0 || - Function->getImageSize() == 0)) { + if (Section && + (Function->getImageAddress() == 0 || Function->getImageSize() == 0)) { continue; } @@ -3440,8 +3414,7 @@ void RewriteInstance::mapFileSections(RuntimeDyld &RTDyld) { mapDataSections(RTDyld); } -std::vector -RewriteInstance::getCodeSections() { +std::vector RewriteInstance::getCodeSections() { std::vector CodeSections; for (BinarySection &Section : BC->textSections()) { if (Section.hasValidSectionID()) @@ -3543,9 +3516,8 @@ void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) { } else { errs() << "BOLT-WARNING: original .text too small to fit the new code" << " using 0x" << Twine::utohexstr(opts::AlignText) - << " alignment. " << CodeSize - << " bytes needed, have " << BC->OldTextSectionSize - << " bytes available.\n"; + << " alignment. " << CodeSize << " bytes needed, have " + << BC->OldTextSectionSize << " bytes available.\n"; opts::UseOldText = false; } } @@ -3666,7 +3638,7 @@ void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) { 16); Section.setOutputAddress(NewTextSectionStartAddress); Section.setOutputFileOffset( - getFileOffsetForAddress(NewTextSectionStartAddress)); + getFileOffsetForAddress(NewTextSectionStartAddress)); } } @@ -3684,8 +3656,8 @@ void RewriteInstance::mapDataSections(RuntimeDyld &RTDyld) { ErrorOr Section = BC->getUniqueSectionByName(SectionName); if (!Section || !Section->isAllocatable() || !Section->isFinalized()) continue; - NextAvailableAddress = alignTo(NextAvailableAddress, - Section->getAlignment()); + NextAvailableAddress = + alignTo(NextAvailableAddress, Section->getAlignment()); LLVM_DEBUG(dbgs() << "BOLT: mapping section " << SectionName << " (0x" << Twine::utohexstr(Section->getAllocAddress()) << ") to 0x" << Twine::utohexstr(NextAvailableAddress) @@ -3829,8 +3801,7 @@ void RewriteInstance::patchELFPHDRTable() { } else if (Phdr.p_type == ELF::PT_GNU_EH_FRAME) { ErrorOr EHFrameHdrSec = BC->getUniqueSectionByName(".eh_frame_hdr"); - if (EHFrameHdrSec && - EHFrameHdrSec->isAllocatable() && + if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() && EHFrameHdrSec->isFinalized()) { NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset(); NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress(); @@ -3854,8 +3825,7 @@ void RewriteInstance::patchELFPHDRTable() { if (!opts::UseGnuStack && !AddedSegment) { // Append the new header to the end of the table. ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); - OS.write(reinterpret_cast(&NewTextPhdr), - sizeof(NewTextPhdr)); + OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); } assert((!opts::UseGnuStack || ModdedGnuStack) && @@ -3866,8 +3836,7 @@ namespace { /// Write padding to \p OS such that its current \p Offset becomes aligned /// at \p Alignment. Return new (aligned) offset. -uint64_t appendPadding(raw_pwrite_stream &OS, - uint64_t Offset, +uint64_t appendPadding(raw_pwrite_stream &OS, uint64_t Offset, uint64_t Alignment) { if (!Alignment) return Offset; @@ -3911,7 +3880,7 @@ void RewriteInstance::rewriteNoteSections() { // Insert padding as needed. NextAvailableOffset = - appendPadding(OS, NextAvailableOffset, Section.sh_addralign); + appendPadding(OS, NextAvailableOffset, Section.sh_addralign); // New section size. uint64_t Size = 0; @@ -3950,8 +3919,7 @@ void RewriteInstance::rewriteNoteSections() { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: " << (Size ? "appending" : "writing") << " contents to section " << SectionName << '\n'); - OS.write(reinterpret_cast(SectionData), - BSec->getOutputSize()); + OS.write(reinterpret_cast(SectionData), BSec->getOutputSize()); Size += BSec->getOutputSize(); } @@ -3984,8 +3952,8 @@ void RewriteInstance::rewriteNoteSections() { assert(!Section.hasPendingRelocations() && "cannot have pending relocs"); - NextAvailableOffset = appendPadding(OS, NextAvailableOffset, - Section.getAlignment()); + NextAvailableOffset = + appendPadding(OS, NextAvailableOffset, Section.getAlignment()); Section.setOutputFileOffset(NextAvailableOffset); LLVM_DEBUG( @@ -4075,7 +4043,7 @@ void RewriteInstance::encodeBATSection() { /*IsReadOnly=*/true, ELF::SHT_NOTE); } -template +template std::string RewriteInstance::getOutputSectionName(const ELFObjType &Obj, const ELFShdrTy &Section) { if (Section.sh_type == ELF::SHT_NULL) @@ -4109,8 +4077,9 @@ bool RewriteInstance::shouldStrip(const ELFShdrTy &Section, } template -std::vector RewriteInstance::getOutputSections( - ELFObjectFile *File, std::vector &NewSectionIndex) { +std::vector +RewriteInstance::getOutputSections(ELFObjectFile *File, + std::vector &NewSectionIndex) { const ELFFile &Obj = File->getELFFile(); typename ELFT::ShdrRange Sections = cantFail(Obj.sections()); @@ -4149,8 +4118,8 @@ std::vector RewriteInstance::getOutputSections( } if (opts::Verbosity >= 1) - outs() << "BOLT-INFO: writing section header for " - << Section.getName() << '\n'; + outs() << "BOLT-INFO: writing section header for " << Section.getName() + << '\n'; ELFShdrTy NewSection; NewSection.sh_type = ELF::SHT_PROGBITS; NewSection.sh_addr = Section.getOutputAddress(); @@ -4188,8 +4157,9 @@ std::vector RewriteInstance::getOutputSections( outs() << "BOLT-INFO: adjusting size for section " << PrevSectionName << '\n'; } - PrevSection->sh_size = Section.sh_addr > PrevSection->sh_addr ? - Section.sh_addr - PrevSection->sh_addr : 0; + PrevSection->sh_size = Section.sh_addr > PrevSection->sh_addr + ? Section.sh_addr - PrevSection->sh_addr + : 0; } PrevSection = &Section; @@ -4234,8 +4204,8 @@ std::vector RewriteInstance::getOutputSections( continue; if (opts::Verbosity >= 1) { - outs() << "BOLT-INFO: writing section header for " - << Section.getName() << '\n'; + outs() << "BOLT-INFO: writing section header for " << Section.getName() + << '\n'; } ELFShdrTy NewSection; NewSection.sh_type = Section.getELFType(); @@ -4346,9 +4316,7 @@ void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile *File) { OS.pwrite(reinterpret_cast(&NewEhdr), sizeof(NewEhdr), 0); } -template void RewriteInstance::updateELFSymbolTable( ELFObjectFile *File, @@ -4358,7 +4326,7 @@ void RewriteInstance::updateELFSymbolTable( WriteFuncTy Write, StrTabFuncTy AddToStrTab) { const ELFFile &Obj = File->getELFFile(); - using ELFSymTy = typename ELFObjectFile::Elf_Sym; + using ELFSymTy = typename ELFObjectFile::Elf_Sym; StringRef StringSection = cantFail(Obj.getStringTableForSymtab(SymTabSection)); @@ -4403,9 +4371,9 @@ void RewriteInstance::updateELFSymbolTable( ELFSymTy ICFSymbol = FunctionSymbol; SmallVector Buf; ICFSymbol.st_name = - AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection))) - .concat(".icf.0") - .toStringRef(Buf)); + AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection))) + .concat(".icf.0") + .toStringRef(Buf)); ICFSymbol.st_value = ICFParent->getOutputAddress(); ICFSymbol.st_size = ICFParent->getOutputSize(); ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex(); @@ -4415,9 +4383,9 @@ void RewriteInstance::updateELFSymbolTable( ELFSymTy NewColdSym = FunctionSymbol; SmallVector Buf; NewColdSym.st_name = - AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection))) - .concat(".cold.0") - .toStringRef(Buf)); + AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection))) + .concat(".cold.0") + .toStringRef(Buf)); NewColdSym.st_shndx = Function.getColdCodeSection()->getIndex(); NewColdSym.st_value = Function.cold().getAddress(); NewColdSym.st_size = Function.cold().getImageSize(); @@ -4506,7 +4474,7 @@ void RewriteInstance::updateELFSymbolTable( // SHN_UNDEF and SHN_ABS. if (!IsDynSym && Function && Symbol.st_shndx != - Function->getOriginSection()->getSectionRef().getIndex()) + Function->getOriginSection()->getSectionRef().getIndex()) Function = nullptr; // Create a new symbol based on the existing symbol. @@ -4529,11 +4497,12 @@ void RewriteInstance::updateELFSymbolTable( } else { // Check if the function symbol matches address inside a function, i.e. // it marks a secondary entry point. - Function = (Symbol.getType() == ELF::STT_FUNC) - ? BC->getBinaryFunctionContainingAddress(Symbol.st_value, - /*CheckPastEnd=*/false, - /*UseMaxSize=*/true) - : nullptr; + Function = + (Symbol.getType() == ELF::STT_FUNC) + ? BC->getBinaryFunctionContainingAddress(Symbol.st_value, + /*CheckPastEnd=*/false, + /*UseMaxSize=*/true) + : nullptr; if (Function && Function->isEmitted()) { const uint64_t OutputAddress = @@ -4542,15 +4511,16 @@ void RewriteInstance::updateELFSymbolTable( NewSymbol.st_value = OutputAddress; // Force secondary entry points to have zero size. NewSymbol.st_size = 0; - NewSymbol.st_shndx = OutputAddress >= Function->cold().getAddress() && - OutputAddress < Function->cold().getImageSize() - ? Function->getColdCodeSection()->getIndex() - : Function->getCodeSection()->getIndex(); + NewSymbol.st_shndx = + OutputAddress >= Function->cold().getAddress() && + OutputAddress < Function->cold().getImageSize() + ? Function->getColdCodeSection()->getIndex() + : Function->getCodeSection()->getIndex(); } else { // Check if the symbol belongs to moved data object and update it. BinaryData *BD = opts::ReorderData.empty() - ? nullptr - : BC->getBinaryDataAtAddress(Symbol.st_value); + ? nullptr + : BC->getBinaryDataAtAddress(Symbol.st_value); if (BD && BD->isMoved() && !BD->isJumpTable()) { assert((!BD->getSize() || !Symbol.st_size || Symbol.st_size == BD->getSize()) && @@ -4576,8 +4546,7 @@ void RewriteInstance::updateELFSymbolTable( // and that were preserved by the linker to support relocations against // .text. Remove them from the symtab. if (Symbol.getType() == ELF::STT_NOTYPE && - Symbol.getBinding() == ELF::STB_LOCAL && - Symbol.st_size == 0) { + Symbol.getBinding() == ELF::STB_LOCAL && Symbol.st_size == 0) { if (BC->getBinaryFunctionContainingAddress(Symbol.st_value, /*CheckPastEnd=*/false, /*UseMaxSize=*/true)) { @@ -4602,12 +4571,12 @@ void RewriteInstance::updateELFSymbolTable( ++IsUpdated; }; - if (opts::HotText && (*SymbolName == "__hot_start" || - *SymbolName == "__hot_end")) + if (opts::HotText && + (*SymbolName == "__hot_start" || *SymbolName == "__hot_end")) updateSymbolValue(*SymbolName, NumHotTextSymsUpdated); - if (opts::HotData && (*SymbolName == "__hot_data_start" || - *SymbolName == "__hot_data_end")) + if (opts::HotData && + (*SymbolName == "__hot_data_start" || *SymbolName == "__hot_data_end")) updateSymbolValue(*SymbolName, NumHotDataSymsUpdated); if (*SymbolName == "_end") { @@ -4633,9 +4602,10 @@ void RewriteInstance::updateELFSymbolTable( for (BinaryFunction *Function : BC->getInjectedBinaryFunctions()) { ELFSymTy NewSymbol; BinarySection *OriginSection = Function->getOriginSection(); - NewSymbol.st_shndx = OriginSection ? - getNewSectionIndex(OriginSection->getSectionRef().getIndex()) : - Function->getCodeSection()->getIndex(); + NewSymbol.st_shndx = + OriginSection + ? getNewSectionIndex(OriginSection->getSectionRef().getIndex()) + : Function->getCodeSection()->getIndex(); NewSymbol.st_value = Function->getOutputAddress(); NewSymbol.st_name = AddToStrTab(Function->getOneName()); NewSymbol.st_size = Function->getOutputSize(); @@ -4648,7 +4618,7 @@ void RewriteInstance::updateELFSymbolTable( NewColdSym.setType(ELF::STT_NOTYPE); SmallVector Buf; NewColdSym.st_name = AddToStrTab( - Twine(Function->getPrintName()).concat(".cold.0").toStringRef(Buf)); + Twine(Function->getPrintName()).concat(".cold.0").toStringRef(Buf)); NewColdSym.st_value = Function->cold().getAddress(); NewColdSym.st_size = Function->cold().getImageSize(); Symbols.emplace_back(NewColdSym); @@ -4704,7 +4674,7 @@ template void RewriteInstance::patchELFSymTabs(ELFObjectFile *File) { const ELFFile &Obj = File->getELFFile(); using ELFShdrTy = typename ELFObjectFile::Elf_Shdr; - using ELFSymTy = typename ELFObjectFile::Elf_Sym; + using ELFSymTy = typename ELFObjectFile::Elf_Sym; // Compute a preview of how section indices will change after rewriting, so // we can properly update the symbol table based on new section indices. @@ -4823,7 +4793,8 @@ RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile *File) { Elf_Rela NewRelA = *RelA; NewRelA.r_addend = NewAddress; OS.pwrite(reinterpret_cast(&NewRelA), sizeof(NewRelA), - reinterpret_cast(RelA) - File->getData().data()); + reinterpret_cast(RelA) - + File->getData().data()); } } } @@ -4848,16 +4819,17 @@ void RewriteInstance::patchELFGOT(ELFObjectFile *File) { StringRef GOTContents = cantFail(GOTSection.getContents()); for (const uint64_t *GOTEntry = - reinterpret_cast(GOTContents.data()); + reinterpret_cast(GOTContents.data()); GOTEntry < reinterpret_cast(GOTContents.data() + - GOTContents.size()); + GOTContents.size()); ++GOTEntry) { if (uint64_t NewAddress = getNewFunctionAddress(*GOTEntry)) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x" << Twine::utohexstr(*GOTEntry) << " with 0x" << Twine::utohexstr(NewAddress) << '\n'); OS.pwrite(reinterpret_cast(&NewAddress), sizeof(NewAddress), - reinterpret_cast(GOTEntry) - File->getData().data()); + reinterpret_cast(GOTEntry) - + File->getData().data()); } } } @@ -4871,7 +4843,7 @@ void RewriteInstance::patchELFDynamic(ELFObjectFile *File) { raw_fd_ostream &OS = Out->os(); using Elf_Phdr = typename ELFFile::Elf_Phdr; - using Elf_Dyn = typename ELFFile::Elf_Dyn; + using Elf_Dyn = typename ELFFile::Elf_Dyn; // Locate DYNAMIC by looking through program headers. uint64_t DynamicOffset = 0; @@ -4958,7 +4930,7 @@ void RewriteInstance::readELFDynamic(ELFObjectFile *File) { const ELFFile &Obj = File->getELFFile(); using Elf_Phdr = typename ELFFile::Elf_Phdr; - using Elf_Dyn = typename ELFFile::Elf_Dyn; + using Elf_Dyn = typename ELFFile::Elf_Dyn; // Locate DYNAMIC by looking through program headers. const Elf_Phdr *DynamicPhdr = 0; @@ -4977,7 +4949,7 @@ void RewriteInstance::readELFDynamic(ELFObjectFile *File) { } assert(DynamicPhdr->p_memsz == DynamicPhdr->p_filesz && - "dynamic section sizes should match"); + "dynamic section sizes should match"); // Go through all dynamic entries to locate entries of interest. typename ELFT::DynRange DynamicEntries = @@ -5016,7 +4988,6 @@ void RewriteInstance::readELFDynamic(ELFObjectFile *File) { PLTRelocationsSize = 0; } - uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) { const BinaryFunction *Function = BC->getBinaryFunctionAtAddress(OldAddress); if (!Function) @@ -5030,7 +5001,7 @@ uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) { void RewriteInstance::rewriteFile() { std::error_code EC; Out = std::make_unique(opts::OutputFilename, EC, - sys::fs::OF_None); + sys::fs::OF_None); check_error(EC, "cannot create output executable file"); raw_fd_ostream &OS = Out->os(); @@ -5064,8 +5035,8 @@ void RewriteInstance::rewriteFile() { errs() << "BOLT-WARNING: new function size (0x" << Twine::utohexstr(Function->getImageSize()) << ") is larger than maximum allowed size (0x" - << Twine::utohexstr(Function->getMaxSize()) - << ") for function " << *Function << '\n'; + << Twine::utohexstr(Function->getMaxSize()) << ") for function " + << *Function << '\n'; } // Remove jump table sections that this function owns in non-reloc mode // because we don't want to write them anymore. @@ -5089,8 +5060,7 @@ void RewriteInstance::rewriteFile() { outs() << "BOLT: rewriting function \"" << *Function << "\"\n"; } OS.pwrite(reinterpret_cast(Function->getImageAddress()), - Function->getImageSize(), - Function->getFileOffset()); + Function->getImageSize(), Function->getFileOffset()); // Write nops at the end of the function. if (Function->getMaxSize() != std::numeric_limits::max()) { @@ -5117,13 +5087,12 @@ void RewriteInstance::rewriteFile() { outs() << "BOLT: rewriting function \"" << *Function << "\" (cold part)\n"; } - OS.pwrite(reinterpret_cast(Function->cold().getImageAddress()), + OS.pwrite(reinterpret_cast(Function->cold().getImageAddress()), Function->cold().getImageSize(), Function->cold().getFileOffset()); ++CountOverwrittenFunctions; - if (opts::MaxFunctions && - CountOverwrittenFunctions == opts::MaxFunctions) { + if (opts::MaxFunctions && CountOverwrittenFunctions == opts::MaxFunctions) { outs() << "BOLT: maximum number of functions reached\n"; break; } @@ -5131,11 +5100,11 @@ void RewriteInstance::rewriteFile() { // Print function statistics for non-relocation mode. if (!BC->HasRelocations) { - outs() << "BOLT: " << CountOverwrittenFunctions - << " out of " << BC->getBinaryFunctions().size() + outs() << "BOLT: " << CountOverwrittenFunctions << " out of " + << BC->getBinaryFunctions().size() << " functions were overwritten.\n"; if (BC->TotalScore != 0) { - double Coverage = OverwrittenScore / (double) BC->TotalScore * 100.0; + double Coverage = OverwrittenScore / (double)BC->TotalScore * 100.0; outs() << format("BOLT-INFO: rewritten functions cover %.2lf", Coverage) << "% of the execution count of simple functions of " "this binary\n"; @@ -5164,19 +5133,17 @@ void RewriteInstance::rewriteFile() { if (opts::Verbosity >= 1) { outs() << "BOLT: writing new section " << Section.getName() << "\n data at 0x" << Twine::utohexstr(Section.getAllocAddress()) - << "\n of size " << Section.getOutputSize() - << "\n at offset " << Section.getOutputFileOffset() << '\n'; + << "\n of size " << Section.getOutputSize() << "\n at offset " + << Section.getOutputFileOffset() << '\n'; } - OS.pwrite(reinterpret_cast(Section.getOutputData()), - Section.getOutputSize(), - Section.getOutputFileOffset()); + OS.pwrite(reinterpret_cast(Section.getOutputData()), + Section.getOutputSize(), Section.getOutputFileOffset()); } for (BinarySection &Section : BC->allocatableSections()) { - Section.flushPendingRelocations(OS, - [this] (const MCSymbol *S) { - return getNewValueForSymbol(S->getName()); - }); + Section.flushPendingRelocations(OS, [this](const MCSymbol *S) { + return getNewValueForSymbol(S->getName()); + }); } // If .eh_frame is present create .eh_frame_hdr. @@ -5223,8 +5190,7 @@ void RewriteInstance::rewriteFile() { } Out->keep(); - EC = sys::fs::setPermissions(opts::OutputFilename, - sys::fs::perms::all_all); + EC = sys::fs::setPermissions(opts::OutputFilename, sys::fs::perms::all_all); check_error(EC, "cannot set permissions of output file"); } @@ -5253,7 +5219,7 @@ void RewriteInstance::writeEHFrameHeader() { LLVM_DEBUG(dbgs() << "BOLT: writing a new .eh_frame_hdr\n"); NextAvailableAddress = - appendPadding(Out->os(), NextAvailableAddress, EHFrameHdrAlign); + appendPadding(Out->os(), NextAvailableAddress, EHFrameHdrAlign); const uint64_t EHFrameHdrOutputAddress = NextAvailableAddress; const uint64_t EHFrameHdrFileOffset = @@ -5296,7 +5262,7 @@ void RewriteInstance::writeEHFrameHeader() { } uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) { - uint64_t Value = RTDyld->getSymbol(Name).getAddress(); + uint64_t Value = RTDyld->getSymbol(Name).getAddress(); if (Value != 0) return Value; @@ -5324,7 +5290,7 @@ uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const { Address >= SegmentInfo.Address + SegmentInfo.FileSize) return 0; - return SegmentInfo.FileOffset + Address - SegmentInfo.Address; + return SegmentInfo.FileOffset + Address - SegmentInfo.Address; } bool RewriteInstance::willOverwriteSection(StringRef SectionName) { diff --git a/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp index 77642b4786c9..0a387526fdc2 100644 --- a/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp @@ -181,8 +181,7 @@ void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC, if (BC.isMachO()) { MCSection *TablesSection = BC.Ctx->getMachOSection( - "__BOLT", "__tables", MachO::S_REGULAR, - SectionKind::getData()); + "__BOLT", "__tables", MachO::S_REGULAR, SectionKind::getData()); TablesSection->setAlignment(llvm::Align(BC.RegularPageSize)); Streamer.SwitchSection(TablesSection); emitString("__bolt_instr_tables", buildTables(BC)); diff --git a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp index cf811edffcc3..d9a4640aa21c 100644 --- a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp @@ -15,7 +15,6 @@ #include "llvm/Object/Archive.h" #include "llvm/Support/Path.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt-rtlib" using namespace llvm; @@ -31,8 +30,7 @@ std::string RuntimeLibrary::getLibPath(StringRef ToolPath, if (!llvm::sys::fs::exists(LibPath)) { // In some cases we install bolt binary into one level deeper in bin/, // we need to go back one more level to find lib directory. - LibPath = - llvm::sys::path::parent_path(llvm::sys::path::parent_path(Dir)); + LibPath = llvm::sys::path::parent_path(llvm::sys::path::parent_path(Dir)); llvm::sys::path::append(LibPath, "lib"); } llvm::sys::path::append(LibPath, LibFileName); @@ -63,8 +61,8 @@ void RuntimeLibrary::loadLibrary(StringRef LibPath, RuntimeDyld &RTDyld) { } else if (Magic == file_magic::elf_relocatable || Magic == file_magic::elf_shared_object) { std::unique_ptr Obj = cantFail( - object::ObjectFile::createObjectFile(B.get()->getMemBufferRef()), - "error creating in-memory object"); + object::ObjectFile::createObjectFile(B.get()->getMemBufferRef()), + "error creating in-memory object"); RTDyld.loadObject(*Obj); } else { errs() << "BOLT-ERROR: unrecognized library format: " << LibPath << "\n"; diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index fcc4086547bc..c651f58aff6c 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -32,30 +32,26 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { public: AArch64MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, const MCRegisterInfo *RegInfo) - : MCPlusBuilder(Analysis, Info, RegInfo) {} + : MCPlusBuilder(Analysis, Info, RegInfo) {} bool equals(const MCTargetExpr &A, const MCTargetExpr &B, CompFuncTy Comp) const override { const auto &AArch64ExprA = cast(A); const auto &AArch64ExprB = cast(B); - if (AArch64ExprA.getKind() != AArch64ExprB.getKind()) + if (AArch64ExprA.getKind() != AArch64ExprB.getKind()) return false; return MCPlusBuilder::equals(*AArch64ExprA.getSubExpr(), *AArch64ExprB.getSubExpr(), Comp); } - bool hasEVEXEncoding(const MCInst &) const override { - return false; - } + bool hasEVEXEncoding(const MCInst &) const override { return false; } bool isMacroOpFusionPair(ArrayRef Insts) const override { return false; } - bool shortenInstruction(MCInst &) const override { - return false; - } + bool shortenInstruction(MCInst &) const override { return false; } bool isADRP(const MCInst &Inst) const override { return Inst.getOpcode() == AArch64::ADRP; @@ -184,8 +180,8 @@ public: if (!Operand.isReg()) continue; unsigned Reg = Operand.getReg(); - if (Reg == AArch64::SP || Reg == AArch64::WSP || - Reg == AArch64::FP || Reg == AArch64::W29) + if (Reg == AArch64::SP || Reg == AArch64::WSP || Reg == AArch64::FP || + Reg == AArch64::W29) return true; } return false; @@ -208,9 +204,7 @@ public: return Inst.getOpcode() == AArch64::BLR; } - MCPhysReg getNoRegister() const override { - return AArch64::NoRegister; - } + MCPhysReg getNoRegister() const override { return AArch64::NoRegister; } bool hasPCRelOperand(const MCInst &Inst) const override { // ADRP is blacklisted and is an exception. Even though it has a @@ -237,8 +231,8 @@ public: const MCOperand &Label = Inst.getOperand(1); if (!Label.isImm()) { - assert (Label.isExpr() && "Unexpected ADR operand"); - assert (DispExpr && "DispExpr must be set"); + assert(Label.isExpr() && "Unexpected ADR operand"); + assert(DispExpr && "DispExpr must be set"); *DispExpr = Label.getExpr(); return false; } @@ -251,10 +245,8 @@ public: return true; } - bool evaluateAArch64MemoryOperand(const MCInst &Inst, - int64_t &DispImm, - const MCExpr **DispExpr = nullptr) - const { + bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm, + const MCExpr **DispExpr = nullptr) const { if (isADR(Inst) || isADRP(Inst)) return evaluateADR(Inst, DispImm, DispExpr); @@ -280,7 +272,7 @@ public: bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, uint64_t Address, uint64_t Size) const override { - int64_t DispValue; + int64_t DispValue; const MCExpr *DispExpr = nullptr; if (!evaluateAArch64MemoryOperand(Inst, DispValue, &DispExpr)) return false; @@ -333,7 +325,7 @@ public: // RewriteInstance::readRelocations(). return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); } else { - switch(RelType) { + switch (RelType) { case ELF::R_AARCH64_ADD_ABS_LO12_NC: case ELF::R_AARCH64_LD64_GOT_LO12_NC: case ELF::R_AARCH64_LDST8_ABS_LO12_NC: @@ -622,8 +614,7 @@ public: } DenseMap> - computeLocalUDChain(const MCInst *CurInstr, - InstructionIterator Begin, + computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin, InstructionIterator End) const { DenseMap RegAliasTable; DenseMap> Uses; @@ -635,7 +626,7 @@ public: if (!Instr.getOperand(OpNum).isReg()) continue; unsigned Reg = Instr.getOperand(OpNum).getReg(); - MCInst* AliasInst = RegAliasTable[Reg]; + MCInst *AliasInst = RegAliasTable[Reg]; Uses[&Instr].push_back(AliasInst); LLVM_DEBUG({ dbgs() << "Adding reg operand " << Reg << " refs "; @@ -798,13 +789,9 @@ public: } } - int getShortJmpEncodingSize() const override { - return 33; - } + int getShortJmpEncodingSize() const override { return 33; } - int getUncondBranchEncodingSize() const override { - return 28; - } + int getUncondBranchEncodingSize() const override { return 28; } bool createTailCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx) override { @@ -853,14 +840,10 @@ public: return true; } - bool isStore(const MCInst &Inst) const override { - return false; - } + bool isStore(const MCInst &Inst) const override { return false; } - bool analyzeBranch(InstructionIterator Begin, - InstructionIterator End, - const MCSymbol *&TBB, - const MCSymbol *&FBB, + bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, + const MCSymbol *&TBB, const MCSymbol *&FBB, MCInst *&CondBranch, MCInst *&UncondBranch) const override { auto I = End; @@ -1066,29 +1049,17 @@ public: return true; } - bool isMoveMem2Reg(const MCInst &Inst) const override { - return false; - } + bool isMoveMem2Reg(const MCInst &Inst) const override { return false; } - bool isADD64rr(const MCInst &Inst) const override { - return false; - } + bool isADD64rr(const MCInst &Inst) const override { return false; } - bool isLeave(const MCInst &Inst) const override { - return false; - } + bool isLeave(const MCInst &Inst) const override { return false; } - bool isPop(const MCInst &Inst) const override { - return false; - } + bool isPop(const MCInst &Inst) const override { return false; } - bool isPrefix(const MCInst &Inst) const override { - return false; - } + bool isPrefix(const MCInst &Inst) const override { return false; } - bool deleteREPPrefix(MCInst &Inst) const override { - return false; - } + bool deleteREPPrefix(MCInst &Inst) const override { return false; } bool createReturn(MCInst &Inst) const override { Inst.setOpcode(AArch64::RET); @@ -1122,7 +1093,6 @@ public: } // end anonymous namespace - namespace llvm { namespace bolt { @@ -1132,5 +1102,5 @@ MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis, return new AArch64MCPlusBuilder(Analysis, Info, RegInfo); } -} -} +} // namespace bolt +} // namespace llvm diff --git a/bolt/tools/driver/llvm-bolt.cpp b/bolt/tools/driver/llvm-bolt.cpp index 6bafc2333c55..1e44f9718861 100644 --- a/bolt/tools/driver/llvm-bolt.cpp +++ b/bolt/tools/driver/llvm-bolt.cpp @@ -26,7 +26,6 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" -#undef DEBUG_TYPE #define DEBUG_TYPE "bolt" using namespace llvm; @@ -126,7 +125,7 @@ void heatmapMode(int argc, char **argv) { std::unique_ptr FakeArgv; if (argc == 1 || strcmp(argv[1], "heatmap")) { ++argc; - FakeArgv.reset(new char *[argc+1]); + FakeArgv.reset(new char *[argc + 1]); FakeArgv[0] = argv[0]; FakeArgv[1] = const_cast("heatmap"); for (int I = 2; I < argc; ++I) @@ -205,7 +204,7 @@ int main(int argc, char **argv) { sys::PrintStackTraceOnErrorSignal(argv[0]); PrettyStackTraceProgram X(argc, argv); - llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. + llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. std::string ToolPath = GetExecutablePath(argv[0]); @@ -236,7 +235,6 @@ int main(int argc, char **argv) { else boltMode(argc, argv); - if (!sys::fs::exists(opts::InputFilename)) report_error(opts::InputFilename, errc::no_such_file_or_directory); @@ -253,8 +251,10 @@ int main(int argc, char **argv) { if (!opts::PerfData.empty()) { if (!opts::AggregateOnly) { errs() << ToolName - << ": WARNING: reading perf data directly is unsupported, please use " - "-aggregate-only or perf2bolt.\n!!! Proceed on your own risk. !!!\n"; + << ": WARNING: reading perf data directly is unsupported, " + "please use " + "-aggregate-only or perf2bolt.\n!!! Proceed on your own " + "risk. !!!\n"; } if (Error E = RI.setProfile(opts::PerfData)) report_error(opts::PerfData, std::move(E)); diff --git a/bolt/tools/merge-fdata/merge-fdata.cpp b/bolt/tools/merge-fdata/merge-fdata.cpp index 0f4aa1fc4da8..8e3668b5e202 100644 --- a/bolt/tools/merge-fdata/merge-fdata.cpp +++ b/bolt/tools/merge-fdata/merge-fdata.cpp @@ -85,9 +85,8 @@ void mergeProfileHeaders(BinaryProfileHeader &MergedHeader, } if (!MergedHeader.FileName.empty() && MergedHeader.FileName != Header.FileName) { - errs() << "WARNING: merging profile from a binary for " - << Header.FileName << " into a profile for binary " - << MergedHeader.FileName << '\n'; + errs() << "WARNING: merging profile from a binary for " << Header.FileName + << " into a profile for binary " << MergedHeader.FileName << '\n'; } if (MergedHeader.Id.empty()) { MergedHeader.Id = Header.Id; @@ -286,7 +285,7 @@ int main(int argc, char **argv) { sys::PrintStackTraceOnErrorSignal(argv[0]); PrettyStackTraceProgram X(argc, argv); - llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. + llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. cl::HideUnrelatedOptions(opts::MergeFdataCategory); @@ -349,19 +348,14 @@ int main(int argc, char **argv) { BinaryProfile MergedProfile; MergedProfile.Header = MergedHeader; MergedProfile.Functions.resize(MergedBFs.size()); - std::transform(MergedBFs.begin(), - MergedBFs.end(), - MergedProfile.Functions.begin(), - [] (StringMapEntry &V) { - return V.second; - }); + std::transform( + MergedBFs.begin(), MergedBFs.end(), MergedProfile.Functions.begin(), + [](StringMapEntry &V) { return V.second; }); // For consistency, sort functions by their IDs. std::sort(MergedProfile.Functions.begin(), MergedProfile.Functions.end(), - [] (const BinaryFunctionProfile &A, - const BinaryFunctionProfile &B) { - return A.Id < B.Id; - }); + [](const BinaryFunctionProfile &A, + const BinaryFunctionProfile &B) { return A.Id < B.Id; }); YamlOut << MergedProfile; } @@ -372,39 +366,33 @@ int main(int argc, char **argv) { if (opts::PrintFunctionList != opts::ST_NONE) { // List of function names with execution count. std::vector> FunctionList(MergedBFs.size()); - using CountFuncType = - std::function( - const StringMapEntry &)>; + using CountFuncType = std::function( + const StringMapEntry &)>; CountFuncType ExecCountFunc = [](const StringMapEntry &V) { - return std::make_pair(V.second.ExecCount, - StringRef(V.second.Name)); - }; + return std::make_pair(V.second.ExecCount, StringRef(V.second.Name)); + }; CountFuncType BranchCountFunc = [](const StringMapEntry &V) { - // Return total branch count. - uint64_t BranchCount = 0; - for (const BinaryBasicBlockProfile &BI : V.second.Blocks) { - for (const SuccessorInfo &SI : BI.Successors) { - BranchCount += SI.Count; - } - } - return std::make_pair(BranchCount, - StringRef(V.second.Name)); - }; + // Return total branch count. + uint64_t BranchCount = 0; + for (const BinaryBasicBlockProfile &BI : V.second.Blocks) { + for (const SuccessorInfo &SI : BI.Successors) { + BranchCount += SI.Count; + } + } + return std::make_pair(BranchCount, StringRef(V.second.Name)); + }; CountFuncType CountFunc = (opts::PrintFunctionList == opts::ST_EXEC_COUNT) - ? ExecCountFunc - : BranchCountFunc; - std::transform(MergedBFs.begin(), - MergedBFs.end(), - FunctionList.begin(), + ? ExecCountFunc + : BranchCountFunc; + std::transform(MergedBFs.begin(), MergedBFs.end(), FunctionList.begin(), CountFunc); std::stable_sort(FunctionList.rbegin(), FunctionList.rend()); errs() << "Functions sorted by " - << (opts::PrintFunctionList == opts::ST_EXEC_COUNT - ? "execution" - : "total branch") + << (opts::PrintFunctionList == opts::ST_EXEC_COUNT ? "execution" + : "total branch") << " count:\n"; for (std::pair &FI : FunctionList) { errs() << FI.second << " : " << FI.first << '\n';