[Debuginfod] [Symbolizer] Break debuginfod out of libLLVM.

Debuginfod can pull in libcurl as a dependency, which isn't appropriate
for libLLVM. (See
https://gitlab.freedesktop.org/mesa/mesa/-/issues/5732).

This change breaks out debuginfod into a separate non-component library
that can be used directly in llvm-symbolizer. The tool can inject
debuginfod into the Symbolizer library via an abstract DebugInfoFetcher
interface, breaking the dependency of Symbolizer on debuinfod.

See https://github.com/llvm/llvm-project/issues/52731

Reviewed By: phosek

Differential Revision: https://reviews.llvm.org/D118413

llvm/include/llvm/DebugInfo/Symbolize/DIFetcher.h

@@ -0,0 +1,51 @@
+//===-- llvm/DebugInfo/Symbolize/DIFetcher.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file declares a DIFetcher abstraction for obtaining debug info from an
+/// arbitrary outside source.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H
+#define LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H
+
+#include <cstdint>
+#include <string>
+
+#include "llvm/ADT/ArrayRef.h"
+
+namespace llvm {
+namespace symbolize {
+
+/// The DIFetcher interface provides arbitrary mechanisms for obtaining debug
+/// info from an outside source.
+class DIFetcher {
+public:
+  virtual ~DIFetcher() = default;
+  virtual Optional<std::string>
+  fetchBuildID(ArrayRef<uint8_t> BuildID) const = 0;
+};
+
+/// LocalDIFetcher searches local cache directories for debug info.
+class LocalDIFetcher : public DIFetcher {
+public:
+  LocalDIFetcher(ArrayRef<std::string> DebugFileDirectory)
+      : DebugFileDirectory(DebugFileDirectory){};
+  virtual ~LocalDIFetcher() = default;
+
+  Optional<std::string> fetchBuildID(ArrayRef<uint8_t> BuildID) const override;
+
+private:
+  const ArrayRef<std::string> DebugFileDirectory;
+};
+
+} // end namespace symbolize
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_SYMBOLIZE_DIFETCHER_H

llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h

@@ -13,6 +13,7 @@
 #ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H
 #define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H
 
+#include "llvm/DebugInfo/Symbolize/DIFetcher.h"
 #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ELFObjectFile.h"
@@ -83,6 +84,10 @@ public:
   DemangleName(const std::string &Name,
                const SymbolizableModule *DbiModuleDescriptor);
 
+  void addDIFetcher(std::unique_ptr<DIFetcher> Fetcher) {
+    DIFetchers.push_back(std::move(Fetcher));
+  }
+
 private:
   // Bundles together object file with code/data and object file with
   // corresponding debug info. These objects can be the same.
@@ -126,6 +131,12 @@ private:
                                   const ELFObjectFileBase *Obj,
                                   const std::string &ArchName);
 
+  bool findDebugBinary(const std::string &OrigPath,
+                       const std::string &DebuglinkName, uint32_t CRCHash,
+                       std::string &Result);
+
+  bool findDebugBinary(const ArrayRef<uint8_t> BuildID, std::string &Result);
+
   /// Returns pair of pointers to object and debug object.
   Expected<ObjectPair> getOrCreateObjectPair(const std::string &Path,
                                              const std::string &ArchName);
@@ -152,6 +163,8 @@ private:
       ObjectForUBPathAndArch;
 
   Options Opts;
+
+  SmallVector<std::unique_ptr<DIFetcher>> DIFetchers;
 };
 
 } // end namespace symbolize

llvm/include/llvm/Debuginfod/DIFetcher.h

@@ -0,0 +1,34 @@
+//===- llvm/DebugInfod/DIFetcher.h - Debug info fetcher----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file declares a DIFetcher implementation for obtaining debug info from
+/// debuginfod.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFOD_DIFETCHER_H
+#define LLVM_DEBUGINFOD_DIFETCHER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/DebugInfo/Symbolize/DIFetcher.h"
+
+namespace llvm {
+
+class DebuginfodDIFetcher : public symbolize::DIFetcher {
+public:
+  virtual ~DebuginfodDIFetcher() = default;
+
+  /// Fetches the given Build ID using debuginfod and returns a local path to
+  /// the resulting debug binary.
+  Optional<std::string> fetchBuildID(ArrayRef<uint8_t> BuildID) const override;
+};
+
+} // namespace llvm
+
+#endif // LLVM_DEBUGINFOD_DIFETCHER_H

llvm/lib/DebugInfo/Symbolize/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_llvm_component_library(LLVMSymbolize
+  DIFetcher.cpp
   DIPrinter.cpp
   SymbolizableObjectFile.cpp
   Symbolize.cpp
@@ -9,7 +10,6 @@ add_llvm_component_library(LLVMSymbolize
   LINK_COMPONENTS
   DebugInfoDWARF
   DebugInfoPDB
-  Debuginfod
   Object
   Support
   Demangle

llvm/lib/DebugInfo/Symbolize/DIFetcher.cpp

@@ -0,0 +1,58 @@
+//===-- lib/DebugInfo/Symbolize/DIFetcher.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the implementation of the local debug info fetcher, which
+/// searches cache directories.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/Symbolize/DIFetcher.h"
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+namespace llvm {
+namespace symbolize {
+
+Optional<std::string>
+LocalDIFetcher::fetchBuildID(ArrayRef<uint8_t> BuildID) const {
+  auto GetDebugPath = [&](StringRef Directory) {
+    SmallString<128> Path{Directory};
+    sys::path::append(Path, ".build-id",
+                      llvm::toHex(BuildID[0], /*LowerCase=*/true),
+                      llvm::toHex(BuildID.slice(1), /*LowerCase=*/true));
+    Path += ".debug";
+    return Path;
+  };
+  if (DebugFileDirectory.empty()) {
+    SmallString<128> Path = GetDebugPath(
+#if defined(__NetBSD__)
+        // Try /usr/libdata/debug/.build-id/../...
+        "/usr/libdata/debug"
+#else
+        // Try /usr/lib/debug/.build-id/../...
+        "/usr/lib/debug"
+#endif
+    );
+    if (llvm::sys::fs::exists(Path))
+      return std::string(Path);
+  } else {
+    for (const auto &Directory : DebugFileDirectory) {
+      // Try <debug-file-directory>/.build-id/../...
+      SmallString<128> Path = GetDebugPath(Directory);
+      if (llvm::sys::fs::exists(Path))
+        return std::string(Path);
+    }
+  }
+  return None;
+}
+
+} // namespace symbolize
+} // namespace llvm

llvm/lib/DebugInfo/Symbolize/Symbolize.cpp

@@ -18,8 +18,8 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/DebugInfo/PDB/PDBContext.h"
+#include "llvm/DebugInfo/Symbolize/DIFetcher.h"
 #include "llvm/DebugInfo/Symbolize/SymbolizableObjectFile.h"
-#include "llvm/Debuginfod/Debuginfod.h"
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/MachO.h"
@@ -229,51 +229,6 @@ bool checkFileCRC(StringRef Path, uint32_t CRCHash) {
   return CRCHash == llvm::crc32(arrayRefFromStringRef(MB.get()->getBuffer()));
 }
 
-bool findDebugBinary(const std::string &OrigPath,
-                     const std::string &DebuglinkName, uint32_t CRCHash,
-                     const std::string &FallbackDebugPath,
-                     std::string &Result) {
-  SmallString<16> OrigDir(OrigPath);
-  llvm::sys::path::remove_filename(OrigDir);
-  SmallString<16> DebugPath = OrigDir;
-  // Try relative/path/to/original_binary/debuglink_name
-  llvm::sys::path::append(DebugPath, DebuglinkName);
-  if (checkFileCRC(DebugPath, CRCHash)) {
-    Result = std::string(DebugPath.str());
-    return true;
-  }
-  // Try relative/path/to/original_binary/.debug/debuglink_name
-  DebugPath = OrigDir;
-  llvm::sys::path::append(DebugPath, ".debug", DebuglinkName);
-  if (checkFileCRC(DebugPath, CRCHash)) {
-    Result = std::string(DebugPath.str());
-    return true;
-  }
-  // Make the path absolute so that lookups will go to
-  // "/usr/lib/debug/full/path/to/debug", not
-  // "/usr/lib/debug/to/debug"
-  llvm::sys::fs::make_absolute(OrigDir);
-  if (!FallbackDebugPath.empty()) {
-    // Try <FallbackDebugPath>/absolute/path/to/original_binary/debuglink_name
-    DebugPath = FallbackDebugPath;
-  } else {
-#if defined(__NetBSD__)
-    // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name
-    DebugPath = "/usr/libdata/debug";
-#else
-    // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name
-    DebugPath = "/usr/lib/debug";
-#endif
-  }
-  llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir),
-                          DebuglinkName);
-  if (checkFileCRC(DebugPath, CRCHash)) {
-    Result = std::string(DebugPath.str());
-    return true;
-  }
-  return false;
-}
-
 bool getGNUDebuglinkContents(const ObjectFile *Obj, std::string &DebugName,
                              uint32_t &CRCHash) {
   if (!Obj)
@@ -350,50 +305,6 @@ Optional<ArrayRef<uint8_t>> getBuildID(const ELFObjectFileBase *Obj) {
   return BuildID;
 }
 
-bool findDebugBinary(const std::vector<std::string> &DebugFileDirectory,
-                     const ArrayRef<uint8_t> BuildID, std::string &Result) {
-  auto getDebugPath = [&](StringRef Directory) {
-    SmallString<128> Path{Directory};
-    sys::path::append(Path, ".build-id",
-                      llvm::toHex(BuildID[0], /*LowerCase=*/true),
-                      llvm::toHex(BuildID.slice(1), /*LowerCase=*/true));
-    Path += ".debug";
-    return Path;
-  };
-  if (DebugFileDirectory.empty()) {
-    SmallString<128> Path = getDebugPath(
-#if defined(__NetBSD__)
-        // Try /usr/libdata/debug/.build-id/../...
-        "/usr/libdata/debug"
-#else
-        // Try /usr/lib/debug/.build-id/../...
-        "/usr/lib/debug"
-#endif
-    );
-    if (llvm::sys::fs::exists(Path)) {
-      Result = std::string(Path.str());
-      return true;
-    }
-  } else {
-    for (const auto &Directory : DebugFileDirectory) {
-      // Try <debug-file-directory>/.build-id/../...
-      SmallString<128> Path = getDebugPath(Directory);
-      if (llvm::sys::fs::exists(Path)) {
-        Result = std::string(Path.str());
-        return true;
-      }
-    }
-  }
-  // Try debuginfod client cache and known servers.
-  Expected<std::string> PathOrErr = getCachedOrDownloadDebuginfo(BuildID);
-  if (!PathOrErr) {
-    consumeError(PathOrErr.takeError());
-    return false;
-  }
-  Result = *PathOrErr;
-  return true;
-}
-
 } // end anonymous namespace
 
 ObjectFile *LLVMSymbolizer::lookUpDsymFile(const std::string &ExePath,
@@ -436,8 +347,7 @@ ObjectFile *LLVMSymbolizer::lookUpDebuglinkObject(const std::string &Path,
   std::string DebugBinaryPath;
   if (!getGNUDebuglinkContents(Obj, DebuglinkName, CRCHash))
     return nullptr;
-  if (!findDebugBinary(Path, DebuglinkName, CRCHash, Opts.FallbackDebugPath,
-                       DebugBinaryPath))
+  if (!findDebugBinary(Path, DebuglinkName, CRCHash, DebugBinaryPath))
     return nullptr;
   auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName);
   if (!DbgObjOrErr) {
@@ -457,7 +367,7 @@ ObjectFile *LLVMSymbolizer::lookUpBuildIDObject(const std::string &Path,
   if (BuildID->size() < 2)
     return nullptr;
   std::string DebugBinaryPath;
-  if (!findDebugBinary(Opts.DebugFileDirectory, *BuildID, DebugBinaryPath))
+  if (!findDebugBinary(*BuildID, DebugBinaryPath))
     return nullptr;
   auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName);
   if (!DbgObjOrErr) {
@@ -467,6 +377,71 @@ ObjectFile *LLVMSymbolizer::lookUpBuildIDObject(const std::string &Path,
   return DbgObjOrErr.get();
 }
 
+bool LLVMSymbolizer::findDebugBinary(const std::string &OrigPath,
+                                     const std::string &DebuglinkName,
+                                     uint32_t CRCHash, std::string &Result) {
+  SmallString<16> OrigDir(OrigPath);
+  llvm::sys::path::remove_filename(OrigDir);
+  SmallString<16> DebugPath = OrigDir;
+  // Try relative/path/to/original_binary/debuglink_name
+  llvm::sys::path::append(DebugPath, DebuglinkName);
+  if (checkFileCRC(DebugPath, CRCHash)) {
+    Result = std::string(DebugPath.str());
+    return true;
+  }
+  // Try relative/path/to/original_binary/.debug/debuglink_name
+  DebugPath = OrigDir;
+  llvm::sys::path::append(DebugPath, ".debug", DebuglinkName);
+  if (checkFileCRC(DebugPath, CRCHash)) {
+    Result = std::string(DebugPath.str());
+    return true;
+  }
+  // Make the path absolute so that lookups will go to
+  // "/usr/lib/debug/full/path/to/debug", not
+  // "/usr/lib/debug/to/debug"
+  llvm::sys::fs::make_absolute(OrigDir);
+  if (!Opts.FallbackDebugPath.empty()) {
+    // Try <FallbackDebugPath>/absolute/path/to/original_binary/debuglink_name
+    DebugPath = Opts.FallbackDebugPath;
+  } else {
+#if defined(__NetBSD__)
+    // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name
+    DebugPath = "/usr/libdata/debug";
+#else
+    // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name
+    DebugPath = "/usr/lib/debug";
+#endif
+  }
+  llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir),
+                          DebuglinkName);
+  if (checkFileCRC(DebugPath, CRCHash)) {
+    Result = std::string(DebugPath.str());
+    return true;
+  }
+  return false;
+}
+
+bool LLVMSymbolizer::findDebugBinary(const ArrayRef<uint8_t> BuildID,
+                                     std::string &Result) {
+  Optional<std::string> Path;
+  Path = LocalDIFetcher(Opts.DebugFileDirectory).fetchBuildID(BuildID);
+  if (Path) {
+    Result = std::move(*Path);
+    return true;
+  }
+
+  // Try caller-provided debug info fetchers.
+  for (const std::unique_ptr<DIFetcher> &Fetcher : DIFetchers) {
+    Path = Fetcher->fetchBuildID(BuildID);
+    if (Path) {
+      Result = std::move(*Path);
+      return true;
+    }
+  }
+
+  return false;
+}
+
 Expected<LLVMSymbolizer::ObjectPair>
 LLVMSymbolizer::getOrCreateObjectPair(const std::string &Path,
                                       const std::string &ArchName) {

llvm/lib/Debuginfod/CMakeLists.txt

@@ -3,8 +3,11 @@ if (LLVM_ENABLE_CURL)
   set(imported_libs CURL::libcurl)
 endif()
 
-add_llvm_component_library(LLVMDebuginfod
+# Note: This isn't a component, since that could potentially add a libcurl
+# dependency to libLLVM.
+add_llvm_library(LLVMDebuginfod
   Debuginfod.cpp
+  DIFetcher.cpp
   HTTPClient.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -16,18 +19,3 @@ add_llvm_component_library(LLVMDebuginfod
   LINK_COMPONENTS
   Support
   )
-
-# This block is only needed for llvm-config. When we deprecate llvm-config and
-# move to using CMake export, this block can be removed.
-if(LLVM_ENABLE_CURL)
-  # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators.
-  if(CMAKE_BUILD_TYPE)
-    string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
-    get_property(curl_library TARGET CURL::libcurl PROPERTY LOCATION_${build_type})
-  endif()
-  if(NOT curl_library)
-    get_property(curl_library TARGET CURL::libcurl PROPERTY LOCATION)
-  endif()
-  get_library_name(${curl_library} curl_library)
-  set(llvm_system_libs ${llvm_system_libs} "${curl_library}")
-endif()

llvm/lib/Debuginfod/DIFetcher.cpp

@@ -0,0 +1,28 @@
+//===- llvm/DebugInfod/DIFetcher.cpp - Debug info fetcher -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines a DIFetcher implementation for obtaining debug info
+/// from debuginfod.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/DIFetcher.h"
+
+#include "llvm/Debuginfod/Debuginfod.h"
+
+using namespace llvm;
+
+Optional<std::string>
+DebuginfodDIFetcher::fetchBuildID(ArrayRef<uint8_t> BuildID) const {
+  Expected<std::string> PathOrErr = getCachedOrDownloadDebuginfo(BuildID);
+  if (PathOrErr)
+    return *PathOrErr;
+  consumeError(PathOrErr.takeError());
+  return None;
+}

llvm/tools/llvm-debuginfod-find/CMakeLists.txt

@@ -1,10 +1,10 @@
 set(LLVM_LINK_COMPONENTS
-  Debuginfod
   Support
   )
 add_llvm_tool(llvm-debuginfod-find
   llvm-debuginfod-find.cpp
   )
+target_link_libraries(llvm-debuginfod-find PRIVATE LLVMDebuginfod)
 if(LLVM_INSTALL_BINUTILS_SYMLINKS)
   add_llvm_tool_symlink(debuginfod-find llvm-debuginfod-find)
 endif()

llvm/tools/llvm-symbolizer/CMakeLists.txt

@@ -10,7 +10,6 @@ add_public_tablegen_target(SymbolizerOptsTableGen)
 set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
   DebugInfoPDB
-  Debuginfod
   Demangle
   Object
   Option
@@ -20,10 +19,13 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_tool(llvm-symbolizer
   llvm-symbolizer.cpp
+
   DEPENDS
   SymbolizerOptsTableGen
   )
 
+target_link_libraries(llvm-symbolizer PRIVATE LLVMDebuginfod)
+
 add_llvm_tool_symlink(llvm-addr2line llvm-symbolizer)
 
 if(LLVM_INSTALL_BINUTILS_SYMLINKS)

llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp

@@ -19,6 +19,7 @@
 #include "llvm/Config/config.h"
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
+#include "llvm/Debuginfod/DIFetcher.h"
 #include "llvm/Debuginfod/HTTPClient.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
@@ -262,8 +263,6 @@ static FunctionNameKind decideHowToPrintFunctions(const opt::InputArgList &Args,
 
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
-  // The HTTPClient must be initialized for use by the debuginfod client.
-  HTTPClient::initialize();
   sys::InitializeCOMRAII COM(sys::COMThreadingMode::MultiThreaded);
 
   bool IsAddr2Line = sys::path::stem(argv[0]).contains("addr2line");
@@ -330,6 +329,12 @@ int main(int argc, char **argv) {
   }
 
   LLVMSymbolizer Symbolizer(Opts);
+
+  // Look up symbols using the debuginfod client.
+  Symbolizer.addDIFetcher(std::make_unique<DebuginfodDIFetcher>());
+  // The HTTPClient must be initialized for use by the debuginfod client.
+  HTTPClient::initialize();
+
   std::unique_ptr<DIPrinter> Printer;
   if (Style == OutputStyle::GNU)
     Printer = std::make_unique<GNUPrinter>(outs(), errs(), Config);

llvm/unittests/Debuginfod/CMakeLists.txt

@@ -1,10 +1,9 @@
-set(LLVM_LINK_COMPONENTS
-  Debuginfod
-  )
-
 add_llvm_unittest(DebuginfodTests
   HTTPClientTests.cpp
   DebuginfodTests.cpp
   )
 
-target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport)
+target_link_libraries(DebuginfodTests PRIVATE
+  LLVMDebuginfod
+  LLVMTestingSupport
+  )