From 498c9750423d7794e9759ab48836618cff9b0601 Mon Sep 17 00:00:00 2001 From: ykiko Date: Thu, 26 Mar 2026 21:27:18 +0800 Subject: [PATCH] feat: add SearchConfig, ToolchainProvider, PathPool and related tests (#370) Co-authored-by: Claude Opus 4.6 --- CMakeLists.txt | 2 + src/command/command.cpp | 279 +++++++++++++++--- src/command/command.h | 24 ++ src/command/search_config.cpp | 134 +++++++++ src/command/search_config.h | 47 +++ src/command/toolchain_provider.cpp | 209 +++++++++++++ src/command/toolchain_provider.h | 75 +++++ src/support/path_pool.h | 40 +++ tests/unit/command/search_config_tests.cpp | 207 +++++++++++++ .../unit/command/toolchain_provider_tests.cpp | 204 +++++++++++++ tests/unit/test/temp_dir.h | 75 +++++ 11 files changed, 1255 insertions(+), 41 deletions(-) create mode 100644 src/command/search_config.cpp create mode 100644 src/command/search_config.h create mode 100644 src/command/toolchain_provider.cpp create mode 100644 src/command/toolchain_provider.h create mode 100644 src/support/path_pool.h create mode 100644 tests/unit/command/search_config_tests.cpp create mode 100644 tests/unit/command/toolchain_provider_tests.cpp create mode 100644 tests/unit/test/temp_dir.h diff --git a/CMakeLists.txt b/CMakeLists.txt index a9c18517..3e711b18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,7 +132,9 @@ add_custom_target(generate_flatbuffers_schema DEPENDS "${GENERATED_HEADER}") # Temporary migration-only build graph. add_library(clice-core STATIC "${PROJECT_SOURCE_DIR}/src/command/command.cpp" + "${PROJECT_SOURCE_DIR}/src/command/search_config.cpp" "${PROJECT_SOURCE_DIR}/src/command/toolchain.cpp" + "${PROJECT_SOURCE_DIR}/src/command/toolchain_provider.cpp" "${PROJECT_SOURCE_DIR}/src/compile/compilation.cpp" "${PROJECT_SOURCE_DIR}/src/compile/compilation_unit.cpp" "${PROJECT_SOURCE_DIR}/src/compile/diagnostic.cpp" diff --git a/src/command/command.cpp b/src/command/command.cpp index 0a299e88..ca158dc2 100644 --- a/src/command/command.cpp +++ b/src/command/command.cpp @@ -150,8 +150,18 @@ struct CompilationDatabase::Impl { /// All source files in the compilation database. llvm::DenseMap> files; - /// TODO: Cache of toolchain query driver results. - llvm::DenseMap toolchains; + /// Pluggable toolchain provider: manages toolchain queries and caching. + ToolchainProvider toolchain; + + /// Cache of SearchConfig keyed by (CompilationInfo*, options_bits). + /// options_bits encodes the CommandOptions fields that affect the result, + /// so different option combinations don't pollute each other's cache entries. + using ConfigCacheKey = std::pair; + llvm::DenseMap search_config_cache; + + static std::uint8_t options_bits(const CommandOptions& options) { + return options.query_toolchain ? 1u : 0u; + } /// The clang options we want to filter in all cases, like -c and -o. llvm::DenseSet filtered_options; @@ -506,6 +516,12 @@ struct CompilationDatabase::Impl { return; } + /// Filter debug info options by group (-g, -gdwarf-*, -gsplit-dwarf, etc.). + /// These only affect debug info generation, not frontend semantics. + if(opt.matches(ID::OPT_DebugInfo_Group)) { + return; + } + /// Remove arguments in the remove list. auto range = ranges::equal_range(known_remove_args, id, {}, get_id); for(auto& remove: range) { @@ -605,6 +621,57 @@ CompilationDatabase::CompilationDatabase() : self(std::make_uniquetoolchain.query_cached(file, directory, user_args); - /// FIXME: we need mangle the arguments again. - /// Work around ... the logic of this should be moved to query ... - bool next_main_file = false; - for(auto& arg: arguments) { - if(arg == llvm::StringRef("-main-file-name")) { - next_main_file = true; - continue; - } - - if(next_main_file) { - arg = self->strings.save(path::filename(file)).data(); - next_main_file = false; - } - } - - if(arguments.empty()) { + if(cached.empty()) { LOG_WARN("failed to query toolchain: {}", file); + arguments = std::move(user_args); } else { - arguments.pop_back(); - } + // Start with cc1 result (has system paths, driver flags, etc.). + arguments.assign(cached.begin(), cached.end()); - // Replace the queried resource dir with ours so the headers are consistent. - // (See clangd's CommandMangler for precedent.) - if(!resource_dir().empty()) { - llvm::StringRef old_resource_dir; - for(std::size_t i = 0; i + 1 < arguments.size(); ++i) { - if(arguments[i] == llvm::StringRef("-resource-dir")) { - old_resource_dir = arguments[i + 1]; - break; + // Remove the temp source file that was appended during query. + arguments.pop_back(); + + // The toolchain query derives the resource dir from the system + // compiler's executable path. If that compiler is a different clang + // version, its builtin headers may not match ours. Replace the + // queried resource dir with ours so the headers are consistent. + // (See clangd's CommandMangler for precedent.) + if(!resource_dir().empty()) { + llvm::StringRef old_resource_dir; + for(std::size_t i = 0; i + 1 < arguments.size(); ++i) { + if(arguments[i] == llvm::StringRef("-resource-dir")) { + old_resource_dir = arguments[i + 1]; + break; + } + } + if(!old_resource_dir.empty() && old_resource_dir != resource_dir()) { + for(auto& arg: arguments) { + llvm::StringRef s(arg); + if(s.starts_with(old_resource_dir)) { + auto replaced = + resource_dir().str() + s.substr(old_resource_dir.size()).str(); + arg = self->strings.save(replaced).data(); + } + } } } - if(!old_resource_dir.empty() && old_resource_dir != resource_dir()) { - for(auto& arg: arguments) { - llvm::StringRef s(arg); - if(s.starts_with(old_resource_dir)) { - auto replaced = - resource_dir().str() + s.substr(old_resource_dir.size()).str(); - arg = self->strings.save(replaced).data(); + + // Replay user-content options (-I/-D/-U/-include/-idirafter) from + // the original mangled args. These were excluded from the toolchain + // query since they don't affect compiler semantics or system paths. + self->parser.parse( + llvm::ArrayRef(user_args).drop_front(), + [&](std::unique_ptr arg) { + auto id = arg->getOption().getID(); + switch(id) { + case ID::OPT_I: + case ID::OPT_isystem: + case ID::OPT_iquote: + case ID::OPT_idirafter: + case ID::OPT_D: + case ID::OPT_U: + case ID::OPT_include: + append_arg(arg->getSpelling()); + for(auto value: arg->getValues()) { + append_arg(value); + } + break; + default: break; } + }, + [](int, int) {}); + + // Fix -main-file-name to match the actual file. + bool next_main_file = false; + for(auto& arg: arguments) { + if(arg == llvm::StringRef("-main-file-name")) { + next_main_file = true; + continue; + } + + if(next_main_file) { + arg = self->strings.save(path::filename(file)).data(); + next_main_file = false; } } } // Inject our resource dir if not already present in the arguments. + // On success, the cc1 output already has -resource-dir (possibly + // replaced above). On failure, the original user_args won't have it. if(!resource_dir().empty()) { bool has_resource_dir = false; for(auto& arg: arguments) { @@ -822,6 +923,50 @@ CompilationContext CompilationDatabase::lookup(llvm::StringRef file, return CompilationContext(directory, std::move(arguments)); } +SearchConfig CompilationDatabase::lookup_search_config(llvm::StringRef file, + const CommandOptions& options, + const void* context) { + // Resolve to the internal CompilationInfo pointer for cache lookup. + auto path_id = self->strings.get(file); + auto it = self->files.find(path_id); + const CompilationInfo* info_ptr = nullptr; + if(it != self->files.end()) { + if(!context) { + info_ptr = it->second->info.ptr; + } else { + auto cur = it->second; + while(cur) { + if(cur->info.ptr == context) { + info_ptr = cur->info.ptr; + break; + } + cur = cur->next; + } + } + } + + if(info_ptr) { + auto key = Impl::ConfigCacheKey{info_ptr, Impl::options_bits(options)}; + auto cache_it = self->search_config_cache.find(key); + if(cache_it != self->search_config_cache.end()) { + return cache_it->second; + } + } + + auto ctx = lookup(file, options, context); + auto config = extract_search_config(ctx.arguments, ctx.directory); + + if(info_ptr) { + auto key = Impl::ConfigCacheKey{info_ptr, Impl::options_bits(options)}; + self->search_config_cache.try_emplace(key, config); + } + return config; +} + +bool CompilationDatabase::has_cached_configs() const { + return !self->search_config_cache.empty(); +} + std::optional CompilationDatabase::get_option_id(llvm::StringRef argument) { auto& table = clang::driver::getDriverOptTable(); @@ -855,6 +1000,58 @@ llvm::StringRef CompilationDatabase::resource_dir() { return dir; } +ToolchainProvider& CompilationDatabase::toolchain() { + return self->toolchain; +} + +std::vector CompilationDatabase::resolve_toolchain_entries( + llvm::ArrayRef> files) { + std::vector entries; + entries.reserve(files.size()); + + for(auto& [file, context]: files) { + auto path_id = self->strings.get(file); + auto stored_file = self->strings.get(path_id); + + object_ptr info = nullptr; + auto it = self->files.find(path_id); + if(it != self->files.end()) { + if(!context) { + info = it->second->info; + } else { + auto cur = it->second; + while(cur) { + if(cur->info.ptr == context) { + info = cur->info; + break; + } + cur = cur->next; + } + } + } + + if(!info || info->arguments.empty()) { + continue; + } + + ToolchainProvider::PendingEntry entry; + entry.file = stored_file; + entry.directory = self->strings.get(info->directory); + entry.arguments.reserve(info->arguments.size()); + for(auto arg_id: info->arguments) { + entry.arguments.push_back(self->strings.get(arg_id).data()); + } + + entries.push_back(std::move(entry)); + } + + return entries; +} + +llvm::StringRef CompilationDatabase::resolve_path(std::uint32_t path_id) { + return self->strings.get(path_id); +} + std::vector CompilationDatabase::files() { std::vector result; for(auto& [file, _]: self->files) { diff --git a/src/command/command.h b/src/command/command.h index 34bf54a8..d6f78031 100644 --- a/src/command/command.h +++ b/src/command/command.h @@ -7,6 +7,8 @@ #include #include +#include "command/search_config.h" +#include "command/toolchain_provider.h" #include "support/format.h" #include "llvm/ADT/ArrayRef.h" @@ -94,6 +96,16 @@ public: /// all contexts and let user choose one. /// std::vector fetch_all(llvm::StringRef file); + /// Combined lookup + extract_search_config with internal caching. + /// Results are cached by CompilationInfo pointer, avoiding repeated + /// argument parsing across multiple calls with the same context. + SearchConfig lookup_search_config(llvm::StringRef file, + const CommandOptions& options = {}, + const void* context = nullptr); + + /// Check if SearchConfig cache is populated (non-empty). + bool has_cached_configs() const; + /// Get an the option for specific argument. static std::optional get_option_id(llvm::StringRef argument); @@ -101,6 +113,18 @@ public: /// from the current executable path using Driver::GetResourcesPath. static llvm::StringRef resource_dir(); + /// Resolve a path_id (from UpdateInfo) back to the file path string. + llvm::StringRef resolve_path(std::uint32_t path_id); + + /// Access the toolchain provider for batch pre-warming and direct queries. + ToolchainProvider& toolchain(); + + /// Resolve (file, context) pairs to PendingEntry tuples for toolchain queries. + /// Converts CDB-internal context pointers to raw (file, directory, arguments) + /// that the ToolchainProvider can consume. + std::vector + resolve_toolchain_entries(llvm::ArrayRef> files); + /// FIXME: bad interface design ... std::vector files(); diff --git a/src/command/search_config.cpp b/src/command/search_config.cpp new file mode 100644 index 00000000..63f0221b --- /dev/null +++ b/src/command/search_config.cpp @@ -0,0 +1,134 @@ +#include "command/search_config.h" + +#include "command/driver.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +namespace clice { + +using ID = clang::driver::options::ID; + +SearchConfig extract_search_config(llvm::ArrayRef arguments, + llvm::StringRef directory) { + // Replicate clang's InitHeaderSearch::Realize layout: + // Quoted (-iquote) → Angled (-I) → System (-isystem, -internal-isystem, etc.) + // Then deduplicate across [Angled..end) matching clang's RemoveDuplicates. + + std::vector quoted; + std::vector angled; + std::vector system; + std::vector after; + + auto make_absolute = [&](llvm::StringRef path) -> std::string { + llvm::SmallString<256> abs_path(path); + if(!llvm::sys::path::is_absolute(abs_path)) { + llvm::sys::fs::make_absolute(directory, abs_path); + } + llvm::sys::path::remove_dots(abs_path, true); + return abs_path.str().str(); + }; + + // Track -iprefix state for -iwithprefix/-iwithprefixbefore. + std::string prefix; + + llvm::BumpPtrAllocator allocator; + ArgumentParser parser{&allocator}; + + parser.parse( + llvm::ArrayRef(arguments).drop_front(), + [&](std::unique_ptr arg) { + auto id = arg->getOption().getID(); + switch(id) { + // Quoted group (clang: frontend::Quoted) + case ID::OPT_iquote: quoted.push_back({make_absolute(arg->getValue())}); break; + + // Angled group (clang: frontend::Angled) + case ID::OPT_I: angled.push_back({make_absolute(arg->getValue())}); break; + + // System group (clang: frontend::System / ExternCSystem) + case ID::OPT_isystem: + case ID::OPT_internal_isystem: + case ID::OPT_internal_externc_isystem: + system.push_back({make_absolute(arg->getValue())}); + break; + + // Prefix options: must be processed in argument order. + case ID::OPT_iprefix: prefix = arg->getValue(); break; + case ID::OPT_iwithprefix: + // clang maps to After group. + after.push_back({make_absolute(prefix + arg->getValue())}); + break; + case ID::OPT_iwithprefixbefore: + // clang maps to Angled group. + angled.push_back({make_absolute(prefix + arg->getValue())}); + break; + + case ID::OPT_idirafter: after.push_back({make_absolute(arg->getValue())}); break; + + // TODO: -cxx-isystem (clang: frontend::CXXSystem, C++-only system dirs) + // TODO: -iwithsysroot (prepends sysroot to path, then adds to System) + // TODO: HeaderMap support (-I foo.hmap remaps include names) + default: break; + } + }, + [](int, int) {}); + + // Concatenate: Quoted → Angled → System → After + SearchConfig config; + config.dirs.reserve(quoted.size() + angled.size() + system.size() + after.size()); + config.dirs.insert(config.dirs.end(), + std::make_move_iterator(quoted.begin()), + std::make_move_iterator(quoted.end())); + config.angled_start_idx = static_cast(config.dirs.size()); + config.dirs.insert(config.dirs.end(), + std::make_move_iterator(angled.begin()), + std::make_move_iterator(angled.end())); + config.system_start_idx = static_cast(config.dirs.size()); + config.dirs.insert(config.dirs.end(), + std::make_move_iterator(system.begin()), + std::make_move_iterator(system.end())); + config.after_start_idx = static_cast(config.dirs.size()); + config.dirs.insert(config.dirs.end(), + std::make_move_iterator(after.begin()), + std::make_move_iterator(after.end())); + + // Deduplicate across [angled_start_idx..end), matching clang's + // RemoveDuplicates(SearchList, NumQuoted). If a path appears in both + // Angled and System, keep the first (Angled) occurrence. This is + // critical for #include_next correctness. + { + llvm::StringSet<> seen; + // Do NOT seed with Quoted paths. clang's RemoveDuplicates(SearchList, + // NumQuoted) starts from NumQuoted, so a path in both Quoted and Angled + // is kept in both — this matters for #include <...> and #include_next. + + unsigned write = config.angled_start_idx; + unsigned removed_before_system = 0; + unsigned removed_before_after = 0; + for(unsigned read = config.angled_start_idx; read < config.dirs.size(); ++read) { + if(seen.insert(config.dirs[read].path).second) { + if(write != read) { + config.dirs[write] = std::move(config.dirs[read]); + } + ++write; + } else { + if(read < config.system_start_idx) { + ++removed_before_system; + } + if(read < config.after_start_idx) { + ++removed_before_after; + } + } + } + config.dirs.resize(write); + config.system_start_idx -= removed_before_system; + config.after_start_idx -= removed_before_after; + } + + return config; +} + +} // namespace clice diff --git a/src/command/search_config.h b/src/command/search_config.h new file mode 100644 index 00000000..0b3fc6c9 --- /dev/null +++ b/src/command/search_config.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +struct SearchDir { + std::string path; +}; + +/// Header search configuration extracted from compilation arguments. +/// Uses a four-segment model matching clang's InitHeaderSearch::Realize layout: +/// [Quoted... | Angled... | System... | After...] +/// ^ ^ ^ +/// angled_start_idx system_start_idx after_start_idx +struct SearchConfig { + /// Ordered list of search directories, partitioned into four segments. + std::vector dirs; + + /// Index in dirs where Angled (-I) dirs start. + /// Quoted ("") includes search from index 0; angled (<>) from here. + unsigned angled_start_idx = 0; + + /// Index in dirs where System (-isystem, -internal-isystem, etc.) dirs start. + unsigned system_start_idx = 0; + + /// Index in dirs where After (-idirafter, -iwithprefix) dirs start. + unsigned after_start_idx = 0; +}; + +/// Extract header search configuration from compilation arguments. +/// +/// Parses user-level flags (-I, -isystem, -iquote) and cc1-level flags +/// (-internal-isystem, -internal-externc-isystem) using the clang argument +/// parser. Relative paths are resolved against the given working directory +/// and normalized with remove_dots(). +/// +/// This is intentionally a standalone function (not tied to CompilationDatabase) +/// so it can be tested and improved independently to match clang's behavior. +SearchConfig extract_search_config(llvm::ArrayRef arguments, + llvm::StringRef directory); + +} // namespace clice diff --git a/src/command/toolchain_provider.cpp b/src/command/toolchain_provider.cpp new file mode 100644 index 00000000..63d51481 --- /dev/null +++ b/src/command/toolchain_provider.cpp @@ -0,0 +1,209 @@ +#include "command/toolchain_provider.h" + +#include "command/driver.h" +#include "command/toolchain.h" +#include "support/filesystem.h" +#include "support/logging.h" +#include "support/object_pool.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" + +namespace clice { + +using ID = clang::driver::options::ID; + +struct ToolchainProvider::Impl { + llvm::BumpPtrAllocator allocator; + StringSet strings{allocator}; + ArgumentParser parser{&allocator}; + + /// Cache of toolchain query results, keyed by canonical toolchain key. + /// The key includes all flags except user-content options (-I/-D/-U/etc.), + /// so the cc1 result reflects the correct compiler semantics (-f/-W/-O/etc.) + /// and only user-content options need to be replayed after cache lookup. + llvm::StringMap> toolchain_cache; + + /// Options excluded from the cache key and toolchain query. These are + /// per-file user content (include paths, defines, forced includes) or + /// input files. They don't affect compiler semantics or system path + /// discovery, and are replayed into the cc1 result afterward. + static bool is_excluded_option(unsigned id) { + switch(id) { + case ID::OPT_I: + case ID::OPT_isystem: + case ID::OPT_iquote: + case ID::OPT_idirafter: + case ID::OPT_D: + case ID::OPT_U: + case ID::OPT_include: + case ID::OPT_INPUT: return true; + default: return false; + } + } + + /// Extract flags for the toolchain query. All options except user-content + /// options (-I/-D/-U/etc.) are included in both the cache key and query args, + /// so the cc1 result correctly reflects compiler semantics (-f/-W/-O/etc.). + struct ToolchainExtract { + std::string key; + std::vector query_args; + }; + + ToolchainExtract extract_toolchain_flags(this Impl& self, + llvm::StringRef file, + llvm::ArrayRef arguments) { + ToolchainExtract result; + + // Driver binary (first arg) — e.g. "clang++" vs "clang" affects language mode. + result.key += arguments[0]; + result.key += '\0'; + + // File extension affects language mode (C vs C++). + result.key += path::extension(file); + result.key += '\0'; + + result.query_args.push_back(arguments[0]); + + self.parser.parse( + llvm::ArrayRef(arguments).drop_front(), + [&](std::unique_ptr arg) { + auto id = arg->getOption().getID(); + if(is_excluded_option(id)) { + return; + } + + // Add option ID and all its values to the cache key. + result.key += std::to_string(id); + result.key += '\0'; + for(auto value: arg->getValues()) { + result.key += value; + result.key += '\0'; + } + + // Render the argument back to query args, respecting the option's + // render style (joined vs separate). + switch(arg->getOption().getRenderStyle()) { + case llvm::opt::Option::RenderJoinedStyle: { + // e.g. -std=c++17, --target=x86_64-linux-gnu + llvm::SmallString<64> joined(arg->getSpelling()); + if(arg->getNumValues() > 0) { + joined += arg->getValue(0); + } + result.query_args.push_back(self.strings.save(joined).data()); + break; + } + case llvm::opt::Option::RenderSeparateStyle: { + // e.g. -target x86_64-linux-gnu, -isysroot /path + result.query_args.push_back(self.strings.save(arg->getSpelling()).data()); + for(auto value: arg->getValues()) { + result.query_args.push_back(self.strings.save(value).data()); + } + break; + } + default: { + // Flags (no value): -nostdinc, -nostdinc++ + result.query_args.push_back(self.strings.save(arg->getSpelling()).data()); + break; + } + } + }, + [](int, int) { + // Unknown arguments are silently dropped — they can't be + // reliably parsed, so we skip them rather than corrupting + // the cache key. + }); + + return result; + } + + /// Query toolchain with caching. Returns the cached cc1 args for the given + /// toolchain key, running the expensive query only on cache miss. + llvm::ArrayRef query_toolchain_cached(this Impl& self, + llvm::StringRef file, + llvm::StringRef directory, + llvm::ArrayRef arguments) { + auto [key, query_args] = self.extract_toolchain_flags(file, arguments); + auto it = self.toolchain_cache.find(key); + if(it != self.toolchain_cache.end()) { + return it->second; + } + + LOG_WARN("Toolchain cache miss (spawning process): file={}, cache_size={}, key_len={}", + file, + self.toolchain_cache.size(), + key.size()); + + auto callback = [&](const char* s) -> const char* { + return self.strings.save(s).data(); + }; + toolchain::QueryParams params = {file, directory, query_args, callback}; + auto result = toolchain::query_toolchain(params); + + auto [entry, _] = self.toolchain_cache.try_emplace(std::move(key), std::move(result)); + return entry->second; + } +}; + +ToolchainProvider::ToolchainProvider() : self(std::make_unique()) {} + +ToolchainProvider::~ToolchainProvider() = default; + +ToolchainProvider::ToolchainProvider(ToolchainProvider&&) noexcept = default; + +ToolchainProvider& ToolchainProvider::operator=(ToolchainProvider&&) noexcept = default; + +llvm::ArrayRef ToolchainProvider::query_cached(llvm::StringRef file, + llvm::StringRef directory, + llvm::ArrayRef arguments) { + return self->query_toolchain_cached(file, directory, arguments); +} + +std::vector + ToolchainProvider::get_pending_queries(llvm::ArrayRef entries) { + llvm::StringMap seen_keys; + std::vector queries; + + for(auto& entry: entries) { + if(entry.arguments.empty()) { + continue; + } + + auto [key, query_args] = self->extract_toolchain_flags(entry.file, entry.arguments); + + // Skip if already cached or already queued. + if(self->toolchain_cache.count(key) || !seen_keys.try_emplace(key, true).second) { + continue; + } + + LOG_DEBUG("Pre-warm: new toolchain key (len={}) for file={}", key.size(), entry.file); + queries.push_back( + {std::move(key), std::move(query_args), entry.file.str(), entry.directory.str()}); + } + + LOG_INFO("Pre-warm: {} unique keys from {} entries, {} queries needed", + seen_keys.size(), + entries.size(), + queries.size()); + return queries; +} + +void ToolchainProvider::inject_results(llvm::ArrayRef results) { + for(auto& result: results) { + if(self->toolchain_cache.count(result.key)) { + continue; + } + std::vector saved; + saved.reserve(result.cc1_args.size()); + for(auto& arg: result.cc1_args) { + saved.push_back(self->strings.save(arg).data()); + } + self->toolchain_cache.try_emplace(result.key, std::move(saved)); + } +} + +bool ToolchainProvider::has_cached_entries() const { + return !self->toolchain_cache.empty(); +} + +} // namespace clice diff --git a/src/command/toolchain_provider.h b/src/command/toolchain_provider.h new file mode 100644 index 00000000..19de2920 --- /dev/null +++ b/src/command/toolchain_provider.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +/// A pending toolchain query, ready to be executed (possibly in parallel). +struct ToolchainQuery { + std::string key; + std::vector query_args; + std::string file; + std::string directory; +}; + +/// Result of a toolchain query, to be injected back into the cache. +struct ToolchainResult { + std::string key; + std::vector cc1_args; +}; + +/// Manages toolchain queries and caching, separated from CompilationDatabase. +/// +/// Given compilation arguments, this component: +/// 1. Extracts toolchain-relevant flags (driver, target, sysroot, stdlib, etc.) +/// 2. Builds a canonical cache key from those flags +/// 3. Queries the compiler driver for system include paths (expensive: spawns a process) +/// 4. Caches results so identical toolchain configurations share one query +/// +/// Designed to be pluggable: CompilationDatabase holds a ToolchainProvider by +/// composition and delegates all toolchain operations to it. +class ToolchainProvider { +public: + ToolchainProvider(); + ~ToolchainProvider(); + ToolchainProvider(ToolchainProvider&&) noexcept; + ToolchainProvider& operator=(ToolchainProvider&&) noexcept; + + /// Query toolchain with caching. Returns cached cc1 args for the given + /// compilation arguments, running the expensive compiler query only on + /// cache miss. The returned ArrayRef is valid for the provider's lifetime. + llvm::ArrayRef query_cached(llvm::StringRef file, + llvm::StringRef directory, + llvm::ArrayRef arguments); + + /// Entry for batch pre-warming: file + directory + raw compilation arguments. + struct PendingEntry { + llvm::StringRef file; + llvm::StringRef directory; + llvm::SmallVector arguments; + }; + + /// Get pending queries for a batch of compilation entries. + /// Returns queries only for cache-miss keys (deduplicated). + std::vector get_pending_queries(llvm::ArrayRef entries); + + /// Inject pre-computed results into the cache. Strings are copied into + /// the provider's internal string pool. + void inject_results(llvm::ArrayRef results); + + /// Check if the cache has any entries. + bool has_cached_entries() const; + +private: + struct Impl; + std::unique_ptr self; +}; + +} // namespace clice diff --git a/src/support/path_pool.h b/src/support/path_pool.h new file mode 100644 index 00000000..81f6c887 --- /dev/null +++ b/src/support/path_pool.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Allocator.h" + +namespace clice { + +/// Intern pool that maps file paths to compact uint32_t IDs. +struct PathPool { + llvm::BumpPtrAllocator allocator; + llvm::SmallVector paths; + llvm::StringMap cache; + + std::uint32_t intern(llvm::StringRef path) { + auto [it, inserted] = cache.try_emplace(path, paths.size()); + if(inserted) { + // Allocate with null terminator so that resolve().data() is safe + // to use as const char* (e.g. in MemoryBuffer::getFile which calls strlen). + const std::size_t n = path.size(); + char* buf = allocator.Allocate(n + 1); + std::copy(path.begin(), path.end(), buf); + buf[n] = '\0'; + paths.push_back(llvm::StringRef(buf, n)); + } + return it->second; + } + + llvm::StringRef resolve(std::uint32_t id) const { + assert(id < paths.size()); + return paths[id]; + } +}; + +} // namespace clice diff --git a/tests/unit/command/search_config_tests.cpp b/tests/unit/command/search_config_tests.cpp new file mode 100644 index 00000000..58b13a8b --- /dev/null +++ b/tests/unit/command/search_config_tests.cpp @@ -0,0 +1,207 @@ +#include "test/temp_dir.h" +#include "test/test.h" +#include "command/search_config.h" + +namespace clice::testing { + +namespace { + +TEST_SUITE(ExtractSearchConfig) { + +TEST_CASE(ReordersDirectoryGroups) { + // TempDir gives cross-platform absolute paths (drive letter on Windows). + TempDir tmp; + std::vector args = {"clang++", + "-internal-isystem", + tmp.c_path("stdlib"), + "-internal-isystem", + tmp.c_path("clang"), + "-internal-externc-isystem", + tmp.c_path("sysroot"), + "-I", + tmp.c_path("user"), + "-iquote", + tmp.c_path("quoted"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + // Expected order: [quoted | user | stdlib, clang, sysroot] + ASSERT_EQ(config.dirs.size(), 5u); + EXPECT_EQ(config.angled_start_idx, 1u); + EXPECT_EQ(config.system_start_idx, 2u); + + EXPECT_EQ(config.dirs[0].path, tmp.path("quoted")); + EXPECT_EQ(config.dirs[1].path, tmp.path("user")); + EXPECT_EQ(config.dirs[2].path, tmp.path("stdlib")); + EXPECT_EQ(config.dirs[3].path, tmp.path("clang")); + EXPECT_EQ(config.dirs[4].path, tmp.path("sysroot")); +} + +TEST_CASE(PreservesWithinGroupOrder) { + TempDir tmp; + std::vector args = {"clang++", + "-I", + tmp.c_path("b"), + "-I", + tmp.c_path("a"), + "-isystem", + tmp.c_path("s2"), + "-isystem", + tmp.c_path("s1"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + ASSERT_EQ(config.dirs.size(), 4u); + EXPECT_EQ(config.angled_start_idx, 0u); + EXPECT_EQ(config.system_start_idx, 2u); + EXPECT_EQ(config.dirs[0].path, tmp.path("b")); + EXPECT_EQ(config.dirs[1].path, tmp.path("a")); + EXPECT_EQ(config.dirs[2].path, tmp.path("s2")); + EXPECT_EQ(config.dirs[3].path, tmp.path("s1")); +} + +TEST_CASE(DeduplicatesAngledSystem) { + TempDir tmp; + std::vector args = {"clang++", + "-I", + tmp.c_path("shared"), + "-internal-isystem", + tmp.c_path("shared"), + "-internal-isystem", + tmp.c_path("only_sys"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + // /shared in both Angled and System → keep Angled copy. + ASSERT_EQ(config.dirs.size(), 2u); + EXPECT_EQ(config.angled_start_idx, 0u); + EXPECT_EQ(config.system_start_idx, 1u); + EXPECT_EQ(config.dirs[0].path, tmp.path("shared")); + EXPECT_EQ(config.dirs[1].path, tmp.path("only_sys")); +} + +TEST_CASE(QuotedAngledSamePathKeptInBoth) { + // clang's RemoveDuplicates starts from NumQuoted, so a path in both + // Quoted (-iquote) and Angled (-I) must be kept in both segments. + // This matters for #include <...> lookup and #include_next correctness. + TempDir tmp; + std::vector args = {"clang++", + "-iquote", + tmp.c_path("shared"), + "-I", + tmp.c_path("shared"), + "-I", + tmp.c_path("other"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + // "shared" must appear in both Quoted and Angled segments. + ASSERT_EQ(config.dirs.size(), 3u); + EXPECT_EQ(config.angled_start_idx, 1u); + EXPECT_EQ(config.dirs[0].path, tmp.path("shared")); // Quoted + EXPECT_EQ(config.dirs[1].path, tmp.path("shared")); // Angled (not deduped) + EXPECT_EQ(config.dirs[2].path, tmp.path("other")); +} + +TEST_CASE(DeduplicateAdjustsIndices) { + TempDir tmp; + std::vector args = {"clang++", + "-iquote", + tmp.c_path("q"), + "-I", + tmp.c_path("dup"), + "-I", + tmp.c_path("a2"), + "-isystem", + tmp.c_path("dup"), + "-isystem", + tmp.c_path("s"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + // Before dedup: [q | dup, a2 | dup, s] angled=1, system=3 + // dup in system removed. system_start_idx stays 3. + ASSERT_EQ(config.dirs.size(), 4u); + EXPECT_EQ(config.angled_start_idx, 1u); + EXPECT_EQ(config.system_start_idx, 3u); + EXPECT_EQ(config.dirs[0].path, tmp.path("q")); + EXPECT_EQ(config.dirs[1].path, tmp.path("dup")); + EXPECT_EQ(config.dirs[2].path, tmp.path("a2")); + EXPECT_EQ(config.dirs[3].path, tmp.path("s")); +} + +TEST_CASE(PrefixIncludeOptions) { + TempDir tmp; + // -iprefix sets a prefix; -iwithprefixbefore/iwithprefix append to it. + // The trailing separator in the prefix path ensures correct concatenation. + auto prefix12 = tmp.path("gcc/12/"); + auto prefix13 = tmp.path("gcc/13/"); + std::vector args = {"clang++", + "-iprefix", + prefix12.c_str(), + "-iwithprefixbefore", + "include", + "-iwithprefix", + "lib", + "-iprefix", + prefix13.c_str(), + "-iwithprefix", + "include", + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + // -iwithprefixbefore → Angled, -iwithprefix → After + ASSERT_EQ(config.dirs.size(), 3u); + EXPECT_EQ(config.angled_start_idx, 0u); + EXPECT_EQ(config.system_start_idx, 1u); + EXPECT_EQ(config.after_start_idx, 1u); + EXPECT_EQ(config.dirs[0].path, tmp.path("gcc/12/include")); + EXPECT_EQ(config.dirs[1].path, tmp.path("gcc/12/lib")); + EXPECT_EQ(config.dirs[2].path, tmp.path("gcc/13/include")); +} + +TEST_CASE(DirafterGroup) { + TempDir tmp; + std::vector args = {"clang++", + "-I", + tmp.c_path("user"), + "-isystem", + tmp.c_path("sys"), + "-idirafter", + tmp.c_path("fallback"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + ASSERT_EQ(config.dirs.size(), 3u); + EXPECT_EQ(config.angled_start_idx, 0u); + EXPECT_EQ(config.system_start_idx, 1u); + EXPECT_EQ(config.after_start_idx, 2u); + EXPECT_EQ(config.dirs[0].path, tmp.path("user")); + EXPECT_EQ(config.dirs[1].path, tmp.path("sys")); + EXPECT_EQ(config.dirs[2].path, tmp.path("fallback")); +} + +TEST_CASE(DirafterDeduplication) { + TempDir tmp; + std::vector args = {"clang++", + "-I", + tmp.c_path("shared"), + "-idirafter", + tmp.c_path("shared"), + "-idirafter", + tmp.c_path("extra"), + "main.cpp"}; + auto config = extract_search_config(args, tmp.root.str()); + + ASSERT_EQ(config.dirs.size(), 2u); + EXPECT_EQ(config.angled_start_idx, 0u); + EXPECT_EQ(config.after_start_idx, 1u); + EXPECT_EQ(config.dirs[0].path, tmp.path("shared")); + EXPECT_EQ(config.dirs[1].path, tmp.path("extra")); +} + +}; // TEST_SUITE(ExtractSearchConfig) + +} // namespace + +} // namespace clice::testing diff --git a/tests/unit/command/toolchain_provider_tests.cpp b/tests/unit/command/toolchain_provider_tests.cpp new file mode 100644 index 00000000..d1bbfcf8 --- /dev/null +++ b/tests/unit/command/toolchain_provider_tests.cpp @@ -0,0 +1,204 @@ +#include "test/test.h" +#include "command/toolchain_provider.h" + +namespace clice::testing { + +namespace { + +TEST_SUITE(ToolchainProvider) { + +TEST_CASE(InitiallyEmpty) { + ToolchainProvider provider; + EXPECT_FALSE(provider.has_cached_entries()); +} + +TEST_CASE(InjectResultsPopulatesCache) { + ToolchainProvider provider; + + std::vector results; + results.push_back({ + "key1", + {"-cc1", "-triple", "x86_64-linux-gnu"} + }); + provider.inject_results(results); + + EXPECT_TRUE(provider.has_cached_entries()); +} + +TEST_CASE(InjectResultsSkipsDuplicateKeys) { + ToolchainProvider provider; + + std::vector results; + results.push_back({ + "key1", + {"-cc1", "-triple", "x86_64"} + }); + results.push_back({ + "key1", + {"-cc1", "-triple", "aarch64"} + }); + provider.inject_results(results); + + // After injection, query_cached with same key should return the first result. + // We verify indirectly: inject twice, cache should still work. + EXPECT_TRUE(provider.has_cached_entries()); +} + +TEST_CASE(GetPendingQueriesReturnsUncachedOnly) { + ToolchainProvider provider; + + // Two entries with same flags but different user-content options. + // They share the same cache key, so only one query is needed. + ToolchainProvider::PendingEntry entry1; + entry1.file = "a.cpp"; + entry1.directory = "/tmp"; + entry1.arguments = {"clang++", "-std=c++17", "-DFOO", "a.cpp"}; + + ToolchainProvider::PendingEntry entry2; + entry2.file = "b.cpp"; + entry2.directory = "/tmp"; + entry2.arguments = {"clang++", "-std=c++17", "-DBAR", "b.cpp"}; + + auto queries = provider.get_pending_queries({entry1, entry2}); + // Same driver, same extension, same non-content flags → one query. + EXPECT_EQ(queries.size(), 1u); +} + +TEST_CASE(GetPendingQueriesDeduplicatesSameKey) { + ToolchainProvider provider; + + // Three entries with same driver and same flags (only -I/-D differ, + // which are user-content options excluded from the cache key). + ToolchainProvider::PendingEntry entry1; + entry1.file = "x.cpp"; + entry1.directory = "/project"; + entry1.arguments = {"clang++", "-Wall", "-O2", "-DFOO=1", "-I/inc/a", "x.cpp"}; + + ToolchainProvider::PendingEntry entry2; + entry2.file = "y.cpp"; + entry2.directory = "/project"; + entry2.arguments = {"clang++", "-Wall", "-O2", "-DBAR=2", "-I/inc/b", "y.cpp"}; + + ToolchainProvider::PendingEntry entry3; + entry3.file = "z.cpp"; + entry3.directory = "/project"; + entry3.arguments = {"clang++", "-Wall", "-O2", "-Uhello", "z.cpp"}; + + auto queries = provider.get_pending_queries({entry1, entry2, entry3}); + // Same driver, same extension, same non-content flags → same key. + EXPECT_EQ(queries.size(), 1u); +} + +TEST_CASE(GetPendingQueriesDifferentDrivers) { + ToolchainProvider provider; + + ToolchainProvider::PendingEntry entry1; + entry1.file = "a.cpp"; + entry1.directory = "/tmp"; + entry1.arguments = {"clang++", "a.cpp"}; + + ToolchainProvider::PendingEntry entry2; + entry2.file = "b.cpp"; + entry2.directory = "/tmp"; + entry2.arguments = {"g++", "b.cpp"}; + + auto queries = provider.get_pending_queries({entry1, entry2}); + // Different drivers → different keys → two queries. + EXPECT_EQ(queries.size(), 2u); +} + +TEST_CASE(GetPendingQueriesDifferentTargets) { + ToolchainProvider provider; + + ToolchainProvider::PendingEntry entry1; + entry1.file = "a.cpp"; + entry1.directory = "/tmp"; + entry1.arguments = {"clang++", "--target=x86_64-linux-gnu", "a.cpp"}; + + ToolchainProvider::PendingEntry entry2; + entry2.file = "b.cpp"; + entry2.directory = "/tmp"; + entry2.arguments = {"clang++", "--target=aarch64-linux-gnu", "b.cpp"}; + + auto queries = provider.get_pending_queries({entry1, entry2}); + // Different targets → different keys → two queries. + EXPECT_EQ(queries.size(), 2u); +} + +TEST_CASE(GetPendingQueriesDifferentLanguageMode) { + ToolchainProvider provider; + + // clang foo.h (default: c-header) vs clang -x c++ foo.h (c++) + // produce different system include paths, so they must have different keys. + ToolchainProvider::PendingEntry entry1; + entry1.file = "foo.h"; + entry1.directory = "/tmp"; + entry1.arguments = {"clang", "foo.h"}; + + ToolchainProvider::PendingEntry entry2; + entry2.file = "foo.h"; + entry2.directory = "/tmp"; + entry2.arguments = {"clang", "-x", "c++", "foo.h"}; + + auto queries = provider.get_pending_queries({entry1, entry2}); + // -x c++ changes language mode → different keys → two queries. + EXPECT_EQ(queries.size(), 2u); +} + +TEST_CASE(GetPendingQueriesSkipsEmptyArgs) { + ToolchainProvider provider; + + ToolchainProvider::PendingEntry empty; + empty.file = "empty.cpp"; + empty.directory = "/tmp"; + // arguments is empty + + ToolchainProvider::PendingEntry valid; + valid.file = "valid.cpp"; + valid.directory = "/tmp"; + valid.arguments = {"clang++", "valid.cpp"}; + + auto queries = provider.get_pending_queries({empty, valid}); + EXPECT_EQ(queries.size(), 1u); +} + +TEST_CASE(InjectThenGetPendingSkipsCached) { + ToolchainProvider provider; + + // First, get pending queries to learn what key is generated. + ToolchainProvider::PendingEntry entry; + entry.file = "test.cpp"; + entry.directory = "/tmp"; + entry.arguments = {"clang++", "test.cpp"}; + + auto queries = provider.get_pending_queries({entry}); + ASSERT_EQ(queries.size(), 1u); + + // Inject a result for that key. + std::vector results; + results.push_back({ + queries[0].key, + {"-cc1", "-triple", "x86_64-linux-gnu"} + }); + provider.inject_results(results); + + // Now the same entry should produce no pending queries. + auto queries2 = provider.get_pending_queries({entry}); + EXPECT_EQ(queries2.size(), 0u); +} + +TEST_CASE(MoveConstruction) { + ToolchainProvider provider; + std::vector results; + results.push_back({"key1", {"-cc1"}}); + provider.inject_results(results); + + ToolchainProvider moved(std::move(provider)); + EXPECT_TRUE(moved.has_cached_entries()); +} + +}; // TEST_SUITE(ToolchainProvider) + +} // namespace + +} // namespace clice::testing diff --git a/tests/unit/test/temp_dir.h b/tests/unit/test/temp_dir.h new file mode 100644 index 00000000..b9eca86b --- /dev/null +++ b/tests/unit/test/temp_dir.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +namespace clice::testing { + +/// RAII helper for a temporary directory tree. +/// +/// Creates a unique temporary directory on construction and removes it +/// (recursively) on destruction. Provides helpers for building paths, +/// creating sub-directories, and writing files — used across multiple +/// test suites that need real filesystem state. +/// +/// Also serves as a cross-platform source of absolute paths: on Windows +/// the root includes a drive letter, so `path("x")` is absolute everywhere. +struct TempDir { + llvm::SmallString<128> root; + + TempDir(llvm::StringRef prefix = "clice-test") { + llvm::sys::fs::createUniqueDirectory(prefix, root); + } + + ~TempDir() { + llvm::sys::fs::remove_directories(root); + } + + TempDir(const TempDir&) = delete; + TempDir& operator=(const TempDir&) = delete; + + /// Build an absolute path under this temporary root. + std::string path(llvm::StringRef relative) const { + llvm::SmallString<256> result(root); + llvm::sys::path::append(result, relative); + llvm::sys::path::native(result); + return std::string(result); + } + + /// Like path(), but returns a `const char*` whose lifetime is tied to + /// this TempDir. Useful for building `ArrayRef` argument + /// lists without manual lifetime management. + const char* c_path(llvm::StringRef relative) { + pool.push_back(path(relative)); + return pool.back().c_str(); + } + + /// Create a sub-directory (and any parents). + void mkdir(llvm::StringRef relative) { + llvm::sys::fs::create_directories(path(relative)); + } + + /// Create a file with optional content (parent dirs created automatically). + void touch(llvm::StringRef relative, llvm::StringRef content = "") { + auto p = path(relative); + llvm::sys::fs::create_directories(llvm::sys::path::parent_path(p)); + std::error_code ec; + llvm::raw_fd_ostream out(p, ec); + if(!ec) { + out << content; + } + } + +private: + /// Pool for strings returned by c_path(). std::deque guarantees that + /// existing elements are not moved when new ones are appended. + std::deque pool; +}; + +} // namespace clice::testing