diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 982dc65e8822..2645a460dd93 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -59,24 +59,24 @@ Fragment linkage header ### Functions table Header: -| Entry | Width | Description | +| Entry | Encoding | Description | | ------ | ----- | ----------- | -| `NumFuncs` | 4B | Number of functions in the functions table | +| `NumFuncs` | ULEB128 | Number of functions in the functions table | The header is followed by Functions table with `NumFuncs` entries. -| Entry | Width | Description | +| Entry | Encoding | Description | | ------ | ------| ----------- | -| `Address` | 8B | Function address in the output binary | -| `NumEntries` | 4B | Number of address translation entries for a function | +| `Address` | ULEB128 | Function address in the output binary | +| `NumEntries` | ULEB128 | Number of address translation entries for a function | Function header is followed by `NumEntries` pairs of offsets for current function. ### Address translation table -| Entry | Width | Description | +| Entry | Encoding | Description | | ------ | ------| ----------- | -| `OutputAddr` | 4B | Function offset in output binary | -| `InputAddr` | 4B | Function offset in input binary with `BRANCHENTRY` top bit | +| `OutputAddr` | ULEB128 | Function offset in output binary | +| `InputAddr` | ULEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target @@ -86,12 +86,12 @@ function. Following Functions table, fragment linkage table is encoded to link split cold fragments with main (hot) fragment. Header: -| Entry | Width | Description | +| Entry | Encoding | Description | | ------ | ------------ | ----------- | -| `NumColdEntries` | 4B | Number of split functions in the functions table | +| `NumColdEntries` | ULEB128 | Number of split functions in the functions table | `NumColdEntries` pairs of addresses follow: -| Entry | Width | Description | +| Entry | Encoding | Description | | ------ | ------| ----------- | -| `ColdAddress` | 8B | Cold fragment address in output binary | -| `HotAddress` | 8B | Hot fragment address in output binary | +| `ColdAddress` | ULEB128 | Cold fragment address in output binary | +| `HotAddress` | ULEB128 | Hot fragment address in output binary | diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 07e4b283211c..5439412cb572 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -125,7 +125,7 @@ private: /// Identifies the address of a control-flow changing instructions in a /// translation map entry - const static uint32_t BRANCHENTRY = 0x80000000; + const static uint32_t BRANCHENTRY = 0x1; }; } // namespace bolt diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index e004309e0e21..3744598f7c7b 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -10,6 +10,8 @@ #include "bolt/Core/BinaryFunction.h" #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LEB128.h" #define DEBUG_TYPE "bolt-bat" @@ -44,7 +46,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, // and this deleted block will both share the same output address (the same // key), and we need to map back. We choose here to privilege the successor by // allowing it to overwrite the previously inserted key in the map. - Map[BBOutputOffset] = BBInputOffset; + Map[BBOutputOffset] = BBInputOffset << 1; const auto &IOAddressMap = BB.getFunction()->getBinaryContext().getIOAddressMap(); @@ -61,8 +63,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(OutputOffset) << " Val: " << Twine::utohexstr(InputOffset) << " (branch)\n"); - Map.insert( - std::pair(OutputOffset, InputOffset | BRANCHENTRY)); + Map.insert(std::pair(OutputOffset, + (InputOffset << 1) | BRANCHENTRY)); } } @@ -102,7 +104,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { } const uint32_t NumFuncs = Maps.size(); - OS.write(reinterpret_cast(&NumFuncs), 4); + encodeULEB128(NumFuncs, OS); LLVM_DEBUG(dbgs() << "Writing " << NumFuncs << " functions for BAT.\n"); for (auto &MapEntry : Maps) { const uint64_t Address = MapEntry.first; @@ -110,20 +112,20 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << ".\n"); - OS.write(reinterpret_cast(&Address), 8); - OS.write(reinterpret_cast(&NumEntries), 4); + encodeULEB128(Address, OS); + encodeULEB128(NumEntries, OS); for (std::pair &KeyVal : Map) { - OS.write(reinterpret_cast(&KeyVal.first), 4); - OS.write(reinterpret_cast(&KeyVal.second), 4); + encodeULEB128(KeyVal.first, OS); + encodeULEB128(KeyVal.second, OS); } } const uint32_t NumColdEntries = ColdPartSource.size(); LLVM_DEBUG(dbgs() << "Writing " << NumColdEntries << " cold part mappings.\n"); - OS.write(reinterpret_cast(&NumColdEntries), 4); + encodeULEB128(NumColdEntries, OS); for (std::pair &ColdEntry : ColdPartSource) { - OS.write(reinterpret_cast(&ColdEntry.first), 8); - OS.write(reinterpret_cast(&ColdEntry.second), 8); + encodeULEB128(ColdEntry.first, OS); + encodeULEB128(ColdEntry.second, OS); LLVM_DEBUG(dbgs() << " " << Twine::utohexstr(ColdEntry.first) << " -> " << Twine::utohexstr(ColdEntry.second) << "\n"); } @@ -152,26 +154,19 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) { if (Name.substr(0, 4) != "BOLT") return make_error_code(llvm::errc::io_error); - if (Buf.size() - Offset < 4) - return make_error_code(llvm::errc::io_error); - - const uint32_t NumFunctions = DE.getU32(&Offset); + Error Err(Error::success()); + const uint32_t NumFunctions = DE.getULEB128(&Offset, &Err); LLVM_DEBUG(dbgs() << "Parsing " << NumFunctions << " functions\n"); for (uint32_t I = 0; I < NumFunctions; ++I) { - if (Buf.size() - Offset < 12) - return make_error_code(llvm::errc::io_error); - - const uint64_t Address = DE.getU64(&Offset); - const uint32_t NumEntries = DE.getU32(&Offset); + const uint64_t Address = DE.getULEB128(&Offset, &Err); + const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); MapTy Map; LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); - if (Buf.size() - Offset < 8 * NumEntries) - return make_error_code(llvm::errc::io_error); for (uint32_t J = 0; J < NumEntries; ++J) { - const uint32_t OutputAddr = DE.getU32(&Offset); - const uint32_t InputAddr = DE.getU32(&Offset); + const uint32_t OutputAddr = DE.getULEB128(&Offset, &Err); + const uint32_t InputAddr = DE.getULEB128(&Offset, &Err); Map.insert(std::pair(OutputAddr, InputAddr)); LLVM_DEBUG(dbgs() << Twine::utohexstr(OutputAddr) << " -> " << Twine::utohexstr(InputAddr) << "\n"); @@ -179,16 +174,11 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) { Maps.insert(std::pair(Address, Map)); } - if (Buf.size() - Offset < 4) - return make_error_code(llvm::errc::io_error); - - const uint32_t NumColdEntries = DE.getU32(&Offset); + const uint32_t NumColdEntries = DE.getULEB128(&Offset, &Err); LLVM_DEBUG(dbgs() << "Parsing " << NumColdEntries << " cold part mappings\n"); for (uint32_t I = 0; I < NumColdEntries; ++I) { - if (Buf.size() - Offset < 16) - return make_error_code(llvm::errc::io_error); - const uint32_t ColdAddress = DE.getU64(&Offset); - const uint32_t HotAddress = DE.getU64(&Offset); + const uint32_t ColdAddress = DE.getULEB128(&Offset, &Err); + const uint32_t HotAddress = DE.getULEB128(&Offset, &Err); ColdPartSource.insert( std::pair(ColdAddress, HotAddress)); LLVM_DEBUG(dbgs() << Twine::utohexstr(ColdAddress) << " -> " @@ -198,7 +188,7 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) { outs() << "BOLT-INFO: Parsed " << NumColdEntries << " BAT cold-to-hot entries\n"; - return std::error_code(); + return errorToErrorCode(std::move(Err)); } void BoltAddressTranslation::dump(raw_ostream &OS) { @@ -209,7 +199,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { OS << "BB mappings:\n"; for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; - const uint32_t Val = Entry.second & ~BRANCHENTRY; + const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit OS << "0x" << Twine::utohexstr(Entry.first) << " -> " << "0x" << Twine::utohexstr(Val); if (IsBranch) @@ -244,7 +234,7 @@ uint64_t BoltAddressTranslation::translate(uint64_t FuncAddress, --KeyVal; - const uint32_t Val = KeyVal->second & ~BRANCHENTRY; + const uint32_t Val = KeyVal->second >> 1; // dropping BRANCHENTRY bit // Branch source addresses are translated to the first instruction of the // source BB to avoid accounting for modifications BOLT may have made in the // BB regarding deletion/addition of instructions. diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index c5813b411d87..023eb05fda04 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 BAT cold-to-hot entries -# CHECK: BOLT-INFO: BAT section size (bytes): 1436 +# CHECK: BOLT-INFO: BAT section size (bytes): 680 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions