[llvm] [Debuginfod] LLVM debuginfod server.

This implements a debuginfod server in llvm using the `DebuginfodCollection` and `DebuginfodServer` classes. This is tested with lit tests against the debuginfod-find client.

The server scans 0 or more local directories for artifacts. It serves the debuginfod protocol over HTTP. Only the `executable` and `debuginfo` endpoints are supported (no `/source` endpoint).
The server also uses the debuginfod client as a fallback, so it can hit the local debuginfod cache or federate to other known debuginfod servers.
The client behavior is controllable through the standard environment variables (`DEBUGINFOD_URLS`, `DEBUGINFOD_CACHE_PATH`, `DEBUGINFOD_TIMEOUT`)

The server implements on-demand collection updates as follows:
If the build-id is not found by a local lookup, rescan immediately and look up the build-id again before returning 404. To protect against DoS attacks, do not rescan more frequently than once per N seconds (specified by `-m`).

Lit tests are provided which test the `llvm-debuginfod-find` client against the `llvm-debuginfod` server.

Reviewed By: mysterymath

Differential Revision: https://reviews.llvm.org/D114846

llvm/test/CMakeLists.txt

@@ -6,6 +6,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_FFI
   LLVM_ENABLE_THREADS
   LLVM_ENABLE_CURL
+  LLVM_ENABLE_HTTPLIB
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_LIBXML2
   LLVM_INCLUDE_GO_TESTS

llvm/test/lit.cfg.py

@@ -159,7 +159,7 @@ tools = [
 tools.extend([
     'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
     'llvm-addr2line', 'llvm-bcanalyzer', 'llvm-bitcode-strip', 'llvm-config',
-    'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find',
+    'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find', 'llvm-debuginfod',
     'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis',
     'llvm-extract', 'llvm-isel-fuzzer', 'llvm-ifs',
     'llvm-install-name-tool', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib',
@@ -478,4 +478,3 @@ def exclude_unsupported_files_for_aix(dirname):
 if 'aix' in config.target_triple:
     for directory in ('/CodeGen/X86', '/DebugInfo', '/DebugInfo/X86', '/DebugInfo/Generic', '/LTO/X86', '/Linker'):
         exclude_unsupported_files_for_aix(config.test_source_root + directory)
-

llvm/test/lit.site.cfg.py.in

@@ -40,6 +40,7 @@ config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
+config.have_httplib = @LLVM_ENABLE_HTTPLIB@
 config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
 config.enable_ffi = @LLVM_ENABLE_FFI@
 config.build_examples = @LLVM_BUILD_EXAMPLES@

llvm/test/tools/llvm-debuginfod/llvm-debuginfod.test

@@ -0,0 +1,108 @@
+# REQUIRES: curl, httplib, thread_support
+
+#int main () {
+#  int x = 1;
+#  return x;
+#}
+#
+#Build as : clang -g main.c -o main-debug.exe
+#Then run : cp main-debug.exe main.exe && strip main.exe
+#resulting buildid: 2c39b7557c50162aaeb5a3148c9f76e6e46012e3
+
+# RUN: rm -rf %t
+# RUN: mkdir %t
+# # Query the debuginfod server for artifacts
+# RUN: DEBUGINFOD_CACHE_PATH=%t %python %s --server-cmd 'llvm-debuginfod -v -c 3 %S/Inputs' \
+# RUN:   --tool-cmd 'llvm-debuginfod-find --dump --executable 2c39b7557c50162aaeb5a3148c9f76e6e46012e3' | \
+# RUN:   diff - %S/Inputs/main.exe
+# RUN: DEBUGINFOD_CACHE_PATH=%t %python %s --server-cmd 'llvm-debuginfod -v -c 3 %S/Inputs' \
+# RUN:   --tool-cmd 'llvm-debuginfod-find --dump --debuginfo 2c39b7557c50162aaeb5a3148c9f76e6e46012e3' | \
+# RUN:   diff - %S/Inputs/main-debug.exe
+# Debuginfod server does not yet support source files
+
+# # The artifacts should still be present in the cache without needing to query
+# # the server.
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump \
+# RUN:   --executable 2c39b7557c50162aaeb5a3148c9f76e6e46012e3 | \
+# RUN:   diff - %S/Inputs/main.exe
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump \
+# RUN:   --debuginfo 2c39b7557c50162aaeb5a3148c9f76e6e46012e3 | \
+# RUN:   diff - %S/Inputs/main-debug.exe
+
+
+
+# This script is used to test the debuginfod client within a host tool against
+# the debuginfod server.
+# It first stands up the debuginfod server and then executes the tool.
+# This way the tool can make debuginfod HTTP requests to the debuginfod server.
+import argparse
+import threading
+import subprocess
+import sys
+import os
+import io
+
+# Starts the server and obtains the port number from the first line of stdout.
+# Waits until the server has completed one full directory scan before returning.
+def start_debuginfod_server(server_args):
+    process = subprocess.Popen(
+        server_args,
+        env=os.environ,
+        stdout=subprocess.PIPE)
+    port = -1
+    # Obtain the port.
+    stdout_reader = io.TextIOWrapper(process.stdout, encoding='ascii')
+    stdout_line = stdout_reader.readline()
+    port = int(stdout_line.split()[-1])
+    # Wait until a directory scan is completed.
+    while True:
+        stdout_line = stdout_reader.readline().strip()
+        print(stdout_line, file=sys.stderr)
+        if stdout_line == 'Updated collection':
+            break
+    return (process, port)
+
+# Starts the server with the specified args (if nonempty), then runs the tool
+# with specified args.
+# Sets the DEBUGINFOD_CACHE_PATH env var to point at the given cache_directory.
+# Sets the DEBUGINFOD_URLS env var to point at the local server.
+def test_tool(server_args, tool_args):
+    server_process = None
+    client_process = None
+    port = None
+    server_process, port = start_debuginfod_server(server_args)
+    try:
+        env = os.environ
+        if port is not None:
+            env['DEBUGINFOD_URLS'] = 'http://localhost:%s' % port
+        client_process = subprocess.Popen(
+            tool_args, env=os.environ)
+        client_code = client_process.wait()
+        if client_code != 0:
+            print('nontrivial client return code %s' % client_code, file=sys.stderr)
+            return 1
+        if server_process is not None:
+            server_process.terminate()
+            server_code = server_process.wait()
+            if server_code != -15:
+                print('nontrivial server return code %s' % server_code, file=sys.stderr)
+                return 1
+
+    finally:
+        if server_process is not None:
+            server_process.terminate()
+        if client_process is not None:
+            client_process.terminate()
+    return 0
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--server-cmd', default='', help='Command to start the server. If not present, no server is started.', type=str)
+    parser.add_argument('--tool-cmd', required=True, type=str)
+    args = parser.parse_args()
+    result = test_tool(args.server_cmd.split(),
+        args.tool_cmd.split())
+    sys.exit(result)
+
+if __name__ == '__main__':
+    main()

llvm/tools/llvm-debuginfod/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+  Debuginfod
+  Support
+  )
+add_llvm_tool(llvm-debuginfod
+  llvm-debuginfod.cpp
+  )
+if(LLVM_INSTALL_BINUTILS_SYMLINKS)
+  add_llvm_tool_symlink(debuginfod llvm-debuginfod)
+endif()

llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp

@@ -0,0 +1,103 @@
+//===-- llvm-debuginfod.cpp - federating debuginfod server ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the llvm-debuginfod tool, which serves the debuginfod
+/// protocol over HTTP. The tool periodically scans zero or more filesystem
+/// directories for ELF binaries to serve, and federates requests for unknown
+/// build IDs to the debuginfod servers set in the DEBUGINFOD_URLS environment
+/// variable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Debuginfod/HTTPClient.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/ThreadPool.h"
+
+using namespace llvm;
+
+cl::OptionCategory DebuginfodCategory("llvm-debuginfod Options");
+
+static cl::list<std::string> ScanPaths(cl::Positional,
+                                       cl::desc("<Directories to scan>"),
+                                       cl::cat(DebuginfodCategory));
+
+static cl::opt<unsigned>
+    Port("p", cl::init(0),
+         cl::desc("Port to listen on. Set to 0 to bind to any available port."),
+         cl::cat(DebuginfodCategory));
+
+static cl::opt<std::string>
+    HostInterface("i", cl::init("0.0.0.0"),
+                  cl::desc("Host interface to bind to."),
+                  cl::cat(DebuginfodCategory));
+
+static cl::opt<int>
+    ScanInterval("t", cl::init(300),
+                 cl::desc("Number of seconds to wait between subsequent "
+                          "automated scans of the filesystem."),
+                 cl::cat(DebuginfodCategory));
+
+static cl::opt<double> MinInterval(
+    "m", cl::init(10),
+    cl::desc(
+        "Minimum number of seconds to wait before an on-demand update can be "
+        "triggered by a request for a buildid which is not in the collection."),
+    cl::cat(DebuginfodCategory));
+
+static cl::opt<size_t>
+    MaxConcurrency("c", cl::init(0),
+                   cl::desc("Maximum number of files to scan concurrently. If "
+                            "0, use the hardware concurrency."),
+                   cl::cat(DebuginfodCategory));
+
+static cl::opt<bool> VerboseLogging("v", cl::init(false),
+                                    cl::desc("Enable verbose logging."),
+                                    cl::cat(DebuginfodCategory));
+
+ExitOnError ExitOnErr;
+
+int main(int argc, char **argv) {
+  InitLLVM X(argc, argv);
+  HTTPClient::initialize();
+  cl::HideUnrelatedOptions({&DebuginfodCategory});
+  cl::ParseCommandLineOptions(argc, argv);
+
+  SmallVector<StringRef, 1> Paths;
+  for (const std::string &Path : ScanPaths)
+    Paths.push_back(Path);
+
+  ThreadPool Pool(hardware_concurrency(MaxConcurrency));
+  DebuginfodLog Log;
+  DebuginfodCollection Collection(Paths, Log, Pool, MinInterval);
+  DebuginfodServer Server(Log, Collection);
+
+  if (!Port)
+    Port = ExitOnErr(Server.Server.bind(HostInterface.c_str()));
+  else
+    ExitOnErr(Server.Server.bind(Port, HostInterface.c_str()));
+
+  Log.push("Listening on port " + Twine(Port).str());
+
+  Pool.async([&]() { ExitOnErr(Server.Server.listen()); });
+  Pool.async([&]() {
+    while (1) {
+      DebuginfodLogEntry Entry = Log.pop();
+      if (VerboseLogging) {
+        outs() << Entry.Message << "\n";
+        outs().flush();
+      }
+    }
+  });
+  if (Paths.size())
+    ExitOnErr(Collection.updateForever(std::chrono::seconds(ScanInterval)));
+  Pool.wait();
+  llvm_unreachable("The ThreadPool should never finish running its tasks.");
+}