[lld][macho] Support order cstrings with -order_file (#140307)

Expand the `-order_file` also accept cstrings to order. The purpose is to order hot cstrings for performance (implemented in this diff), and then later on we can also order cold cstrings for compression size win. Due to the speciality of cstrings, there's no way to pass in symbol names in the order file as the existing -order_file, so we expect `<hash of cstring literal content>` to represent/identify each cstring. ``` // An order file has one entry per line, in the following format: // // <cpu>:<object file>:[<symbol name> | CStringEntryPrefix <cstring hash>] // // <cpu> and <object file> are optional. // If not specified, then that entry tries to match either, // // 1) any symbol of the <symbol name>; // Parsing this format is not quite straightforward because the symbol name // itself can contain colons, so when encountering a colon, we consider the // preceding characters to decide if it can be a valid CPU type or file path. // If a symbol is matched by multiple entries, then it takes the // lowest-ordered entry (the one nearest to the front of the list.) // // or 2) any cstring literal with the given hash, if the entry has the // CStringEntryPrefix prefix defined below in the file. <cstring hash> is the // hash of cstring literal content. // // Cstring literals are not symbolized, we can't identify them by name // However, cstrings are deduplicated, hence unique, so we use the hash of // the content of cstring literals to identify them and assign priority to it. // We use the same hash as used in StringPiece, i.e. 31 bit: // xxh3_64bits(string) & 0x7fffffff // ``` The ordering of cstring has to happen during/before the finalizing of the cstring section content in the `finalizeContents()` function, which happens before the writer is run --------- Co-authored-by: Sharon Xu <sharonxu@fb.com>
2025-06-05 10:24:54 -07:00
parent 5990624383
commit 40933fd410
4 changed files with 331 additions and 33 deletions
--- a/lld/MachO/SectionPriorities.cpp
+++ b/lld/MachO/SectionPriorities.cpp
@@ -247,15 +247,13 @@ DenseMap<const InputSection *, int> CallGraphSort::run() {
 }

 std::optional<int>
-macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
-  if (sym->isAbsolute())
-    return std::nullopt;
+macho::PriorityBuilder::getSymbolOrCStringPriority(const StringRef key,
+                                                   InputFile *f) {

-  auto it = priorities.find(utils::getRootSymbol(sym->getName()));
+  auto it = priorities.find(key);
  if (it == priorities.end())
    return std::nullopt;
  const SymbolPriorityEntry &entry = it->second;
-  const InputFile *f = sym->isec()->getFile();
  if (!f)
    return entry.anyObjectFile;
  // We don't use toString(InputFile *) here because it returns the full path
@@ -269,6 +267,14 @@ macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
  return std::min(entry.objectFiles.lookup(filename), entry.anyObjectFile);
 }

+std::optional<int>
+macho::PriorityBuilder::getSymbolPriority(const Defined *sym) {
+  if (sym->isAbsolute())
+    return std::nullopt;
+  return getSymbolOrCStringPriority(utils::getRootSymbol(sym->getName()),
+                                    sym->isec()->getFile());
+}
+
 void macho::PriorityBuilder::extractCallGraphProfile() {
  TimeTraceScope timeScope("Extract call graph profile");
  bool hasOrderFile = !priorities.empty();
@@ -301,7 +307,7 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
  int prio = std::numeric_limits<int>::min();
  MemoryBufferRef mbref = *buffer;
  for (StringRef line : args::getLines(mbref)) {
-    StringRef objectFile, symbol;
+    StringRef objectFile, symbolOrCStrHash;
    line = line.take_until([](char c) { return c == '#'; }); // ignore comments
    line = line.ltrim();

@@ -316,7 +322,6 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {

    if (cpuType != CPU_TYPE_ANY && cpuType != target->cpuType)
      continue;
-
    // Drop the CPU type as well as the colon
    if (cpuType != CPU_TYPE_ANY)
      line = line.drop_until([](char c) { return c == ':'; }).drop_front();
@@ -331,10 +336,20 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) {
        break;
      }
    }
-    symbol = utils::getRootSymbol(line.trim());

-    if (!symbol.empty()) {
-      SymbolPriorityEntry &entry = priorities[symbol];
+    // The rest of the line is either <symbol name> or
+    // CStringEntryPrefix<cstring hash>
+    line = line.trim();
+    if (line.starts_with(CStringEntryPrefix)) {
+      StringRef possibleHash = line.drop_front(CStringEntryPrefix.size());
+      uint32_t hash = 0;
+      if (to_integer(possibleHash, hash))
+        symbolOrCStrHash = possibleHash;
+    } else
+      symbolOrCStrHash = utils::getRootSymbol(line);
+
+    if (!symbolOrCStrHash.empty()) {
+      SymbolPriorityEntry &entry = priorities[symbolOrCStrHash];
      if (!objectFile.empty())
        entry.objectFiles.insert(std::make_pair(objectFile, prio));
      else
@@ -389,3 +404,41 @@ macho::PriorityBuilder::buildInputSectionPriorities() {

  return sectionPriorities;
 }
+
+std::vector<StringPiecePair> macho::PriorityBuilder::buildCStringPriorities(
+    ArrayRef<CStringInputSection *> inputs) {
+  // Split the input strings into hold and cold sets.
+  // Order hot set based on -order_file_cstring for performance improvement;
+  // TODO: Order cold set of cstrings for compression via BP.
+  std::vector<std::pair<int, StringPiecePair>>
+      hotStringPrioritiesAndStringPieces;
+  std::vector<StringPiecePair> coldStringPieces;
+  std::vector<StringPiecePair> orderedStringPieces;
+
+  for (CStringInputSection *isec : inputs) {
+    for (const auto &[stringPieceIdx, piece] : llvm::enumerate(isec->pieces)) {
+      if (!piece.live)
+        continue;
+
+      std::optional<int> priority = getSymbolOrCStringPriority(
+          std::to_string(piece.hash), isec->getFile());
+      if (!priority)
+        coldStringPieces.emplace_back(isec, stringPieceIdx);
+      else
+        hotStringPrioritiesAndStringPieces.emplace_back(
+            *priority, std::make_pair(isec, stringPieceIdx));
+    }
+  }
+
+  // Order hot set for perf
+  llvm::stable_sort(hotStringPrioritiesAndStringPieces);
+  for (auto &[priority, stringPiecePair] : hotStringPrioritiesAndStringPieces)
+    orderedStringPieces.push_back(stringPiecePair);
+
+  // TODO: Order cold set for compression
+
+  orderedStringPieces.insert(orderedStringPieces.end(),
+                             coldStringPieces.begin(), coldStringPieces.end());
+
+  return orderedStringPieces;
+}
--- a/lld/MachO/SectionPriorities.h
+++ b/lld/MachO/SectionPriorities.h
@@ -16,6 +16,7 @@
 namespace lld::macho {

 using SectionPair = std::pair<const InputSection *, const InputSection *>;
+using StringPiecePair = std::pair<CStringInputSection *, size_t>;

 class PriorityBuilder {
 public:
@@ -28,17 +29,28 @@ public:
  //
  // An order file has one entry per line, in the following format:
  //
-  //   <cpu>:<object file>:<symbol name>
+  //   <cpu>:<object file>:[<symbol name> | CStringEntryPrefix <cstring hash>]
  //
-  // <cpu> and <object file> are optional. If not specified, then that entry
-  // matches any symbol of that name. Parsing this format is not quite
-  // straightforward because the symbol name itself can contain colons, so when
-  // encountering a colon, we consider the preceding characters to decide if it
-  // can be a valid CPU type or file path.
+  // <cpu> and <object file> are optional.
+  // If not specified, then that entry tries to match either,
  //
+  // 1) any symbol of the <symbol name>;
+  // Parsing this format is not quite straightforward because the symbol name
+  // itself can contain colons, so when encountering a colon, we consider the
+  // preceding characters to decide if it can be a valid CPU type or file path.
  // If a symbol is matched by multiple entries, then it takes the
  // lowest-ordered entry (the one nearest to the front of the list.)
  //
+  // or 2) any cstring literal with the given hash, if the entry has the
+  // CStringEntryPrefix prefix defined below in the file. <cstring hash> is the
+  // hash of cstring literal content.
+  //
+  // Cstring literals are not symbolized, we can't identify them by name
+  // However, cstrings are deduplicated, hence unique, so we use the hash of
+  // the content of cstring literals to identify them and assign priority to it.
+  // We use the same hash as used in StringPiece, i.e. 31 bit:
+  // xxh3_64bits(string) & 0x7fffffff
+  //
  // The file can also have line comments that start with '#'.
  void parseOrderFile(StringRef path);

@@ -54,6 +66,8 @@ public:
  // Each section gets assigned the priority of the highest-priority symbol it
  // contains.
  llvm::DenseMap<const InputSection *, int> buildInputSectionPriorities();
+  std::vector<StringPiecePair>
+      buildCStringPriorities(ArrayRef<CStringInputSection *>);

 private:
  // The symbol with the smallest priority should be ordered first in the output
@@ -65,8 +79,11 @@ private:
    // The priority given to a matching symbol from a particular object file.
    llvm::DenseMap<llvm::StringRef, int> objectFiles;
  };
+  const llvm::StringRef CStringEntryPrefix = "CSTR;";

  std::optional<int> getSymbolPriority(const Defined *sym);
+  std::optional<int> getSymbolOrCStringPriority(const StringRef key,
+                                                InputFile *f);
  llvm::DenseMap<llvm::StringRef, SymbolPriorityEntry> priorities;
  llvm::MapVector<SectionPair, uint64_t> callGraphProfile;
 };
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -14,6 +14,7 @@
 #include "InputFiles.h"
 #include "ObjC.h"
 #include "OutputSegment.h"
+#include "SectionPriorities.h"
 #include "SymbolTable.h"
 #include "Symbols.h"

@@ -1760,27 +1761,26 @@ void DeduplicatedCStringSection::finalizeContents() {
    }
  }

-  // Assign an offset for each string and save it to the corresponding
+  // Sort the strings for performance and compression size win, and then
+  // assign an offset for each string and save it to the corresponding
  // StringPieces for easy access.
-  for (CStringInputSection *isec : inputs) {
-    for (const auto &[i, piece] : llvm::enumerate(isec->pieces)) {
-      if (!piece.live)
-        continue;
+  for (auto &[isec, i] : priorityBuilder.buildCStringPriorities(inputs)) {
+    auto &piece = isec->pieces[i];
    auto s = isec->getCachedHashStringRef(i);
    auto it = stringOffsetMap.find(s);
    assert(it != stringOffsetMap.end());
-      StringOffset &offsetInfo = it->second;
+    lld::macho::DeduplicatedCStringSection::StringOffset &offsetInfo =
+        it->second;
    if (offsetInfo.outSecOff == UINT64_MAX) {
      offsetInfo.outSecOff =
          alignToPowerOf2(size, 1ULL << offsetInfo.trailingZeros);
-        size =
-            offsetInfo.outSecOff + s.size() + 1; // account for null terminator
+      size = offsetInfo.outSecOff + s.size() + 1; // account for null terminator
    }
    piece.outSecOff = offsetInfo.outSecOff;
  }
+  for (CStringInputSection *isec : inputs)
    isec->isFinal = true;
 }
-}

 void DeduplicatedCStringSection::writeTo(uint8_t *buf) const {
  for (const auto &p : stringOffsetMap) {
--- a/lld/test/MachO/ordre-file-cstring.s
+++ b/lld/test/MachO/ordre-file-cstring.s
@@ -0,0 +1,228 @@
+# REQUIRES: aarch64
+
+# RUN: rm -rf %t; split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin  %t/test.s -o %t/test.o
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/more-cstrings.s -o %t/more-cstrings.o
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-0 %t/test.o %t/more-cstrings.o
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-1 %t/test.o %t/more-cstrings.o -order_file %t/ord-1
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-1 | FileCheck %s --check-prefix=ONE_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-1 | FileCheck %s --check-prefix=ONE_SEC
+
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-2 %t/test.o %t/more-cstrings.o -order_file %t/ord-2
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-2 | FileCheck %s --check-prefix=TWO_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-2 | FileCheck %s --check-prefix=TWO_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-3 %t/test.o %t/more-cstrings.o -order_file %t/ord-3
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-3 | FileCheck %s --check-prefix=THREE_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-3 | FileCheck %s --check-prefix=THREE_SEC
+
+# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-4 %t/test.o %t/more-cstrings.o -order_file %t/ord-4
+# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-4 | FileCheck %s --check-prefix=FOUR_SYM
+# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC
+# RUN: llvm-readobj --string-dump=__cstring %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC_ESCAPE
+
+
+# We expect:
+# 1) Covered cstring symbols are reordered
+# 2) the rest of the cstring symbols remain original relative order within the cstring section
+
+# ORIGIN_SYM: _local_foo1
+# ORIGIN_SYM: _globl_foo2
+# ORIGIN_SYM: _local_foo2
+# ORIGIN_SYM: _bar
+# ORIGIN_SYM: _baz
+# ORIGIN_SYM: _baz_dup
+# ORIGIN_SYM: _bar2
+# ORIGIN_SYM: _globl_foo3
+
+# ORIGIN_SEC: foo1
+# ORIGIN_SEC: foo2
+# ORIGIN_SEC: bar
+# ORIGIN_SEC: baz
+# ORIGIN_SEC: bar2
+# ORIGIN_SEC: foo3
+
+# original order, but only parital covered
+#--- ord-1
+#foo2
+CSTR;1433942677
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+#foo3
+CSTR;1343999025
+
+# ONE_SYM: _globl_foo2
+# ONE_SYM: _local_foo2
+# ONE_SYM: _bar
+# ONE_SYM: _bar2
+# ONE_SYM: _globl_foo3
+# ONE_SYM: _local_foo1
+# ONE_SYM: _baz
+# ONE_SYM: _baz_dup
+
+# ONE_SEC: foo2
+# ONE_SEC: bar
+# ONE_SEC: bar2
+# ONE_SEC: foo3
+# ONE_SEC: foo1
+# ONE_SEC: baz
+
+
+# TWO_SYM: _globl_foo2
+# TWO_SYM: _local_foo2
+# TWO_SYM: _local_foo1
+# TWO_SYM: _baz
+# TWO_SYM: _baz_dup
+# TWO_SYM: _bar
+# TWO_SYM: _bar2
+# TWO_SYM: _globl_foo3
+
+# TWO_SEC: foo2
+# TWO_SEC: foo1
+# TWO_SEC: baz
+# TWO_SEC: bar
+# TWO_SEC: bar2
+# TWO_SEC: foo3
+
+
+# THREE_SYM: _local_foo1
+# THREE_SYM: _baz
+# THREE_SYM: _baz_dup
+# THREE_SYM: _bar
+# THREE_SYM: _bar2
+# THREE_SYM: _globl_foo2
+# THREE_SYM: _local_foo2
+# THREE_SYM: _globl_foo3
+
+# THREE_SEC: foo1
+# THREE_SEC: baz
+# THREE_SEC: bar
+# THREE_SEC: bar2
+# THREE_SEC: foo2
+# THREE_SEC: foo3
+
+
+# FOUR_SYM: _local_escape_white_space
+# FOUR_SYM: _globl_foo2
+# FOUR_SYM: _local_foo2
+# FOUR_SYM: _local_escape
+# FOUR_SYM: _globl_foo3
+# FOUR_SYM: _bar
+# FOUR_SYM: _local_foo1
+# FOUR_SYM: _baz
+# FOUR_SYM: _baz_dup
+# FOUR_SYM: _bar2
+
+# FOUR_SEC: \t\n
+# FOUR_SEC: foo2
+# FOUR_SEC: @\"NSDictionary\"
+# FOUR_SEC: foo3
+# FOUR_SEC: bar
+# FOUR_SEC: foo1
+# FOUR_SEC: baz
+# FOUR_SEC: bar2
+
+# FOUR_SEC_ESCAPE: ..
+# FOUR_SEC_ESCAPE: foo2
+# FOUR_SEC_ESCAPE: @"NSDictionary"
+# FOUR_SEC_ESCAPE: foo3
+# FOUR_SEC_ESCAPE: bar
+# FOUR_SEC_ESCAPE: foo1
+# FOUR_SEC_ESCAPE: baz
+# FOUR_SEC_ESCAPE: bar2
+
+
+# change order, parital covered
+#--- ord-2
+#foo2
+CSTR;1433942677
+#foo1
+CSTR;1663475769
+#baz
+CSTR;862947621
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+
+# change order, parital covered, with mismatches, duplicates
+#--- ord-3
+foo2222
+CSTR;0x11111111
+#bar (mismatched cpu and file name)
+fakeCPU:fake-file-name.o:CSTR;540201826
+#not a hash
+CSTR;xxx
+#foo1
+CSTR;1663475769
+#baz
+CSTR;862947621
+#bar
+CSTR;540201826
+#bar2
+CSTR;1496286555
+#baz
+CSTR;862947621
+
+# test escape strings
+#--- ord-4
+#\t\n
+CSTR;1035903177
+#foo2
+CSTR;1433942677
+#@\"NSDictionary\"
+CSTR;1202669430
+#foo3
+CSTR;1343999025
+#bar
+CSTR;540201826
+
+
+#--- test.s
+.text
+.globl _main
+
+_main:
+  ret
+
+.cstring
+.p2align 2
+_local_foo1:
+  .asciz "foo1"
+_local_foo2:
+  .asciz "foo2"
+L_.foo1_dup:
+  .asciz "foo1"
+L_.foo2_dup:
+  .asciz "foo2"
+_local_escape:
+  .asciz "@\"NSDictionary\""
+_local_escape_white_space:
+  .asciz "\t\n"
+
+_bar:
+  .asciz "bar"
+_baz:
+  .asciz "baz"
+_bar2:
+  .asciz "bar2"
+_baz_dup:
+  .asciz "baz"
+
+.subsections_via_symbols
+
+#--- more-cstrings.s
+.globl _globl_foo1, _globl_foo3
+.cstring
+.p2align 4
+_globl_foo3:
+  .asciz "foo3"
+_globl_foo2:
+  .asciz "foo2"