From 8aff090a086d8e3e515f515ec0e72ea38835b855 Mon Sep 17 00:00:00 2001 From: ykiko Date: Sun, 23 Nov 2025 18:43:36 +0800 Subject: [PATCH] refactor: incremental update for compilation database and introduce query toolchain (#311) --- .github/workflows/cmake.yml | 4 + CMakeLists.txt | 6 +- bin/clice.cc | 10 +- include/Compiler/Command.h | 101 +-- include/Compiler/Compilation.h | 2 + include/Compiler/Toolchain.h | 68 +- include/Support/FileSystem.h | 2 + include/Support/Logging.h | 26 +- include/Support/ObjectPool.h | 224 ++++++ include/Test/Tester.h | 149 +--- src/AST/Selection.cpp | 26 +- src/Async/FileSystem.cpp | 2 +- src/Async/Network.cpp | 6 +- src/Async/libuv.cpp | 7 +- src/Compiler/Command.cpp | 1085 +++++++++++++++-------------- src/Compiler/Compilation.cpp | 48 +- src/Compiler/Tidy.cpp | 6 +- src/Compiler/Toolchain.cpp | 576 ++++++++++----- src/Feature/Diagnostic.cpp | 6 +- src/Feature/Formatting.cpp | 2 +- src/Server/Document.cpp | 62 +- src/Server/Feature.cpp | 2 + src/Server/Indexer.cpp | 23 +- src/Server/Lifecycle.cpp | 18 +- src/Server/Server.cpp | 14 +- tests/unit/AST/Selection.cpp | 53 +- tests/unit/Compiler/Command.cpp | 210 +++--- tests/unit/Compiler/Toolchain.cpp | 126 ++++ tests/unit/Index/USR.cpp | 4 +- tests/unit/Test/Tester.cpp | 165 +++++ xmake.lua | 6 +- 31 files changed, 1871 insertions(+), 1168 deletions(-) create mode 100644 include/Support/ObjectPool.h create mode 100644 tests/unit/Compiler/Toolchain.cpp create mode 100644 tests/unit/Test/Tester.cpp diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 32401bba..cdab7a14 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -103,5 +103,9 @@ jobs: EXE_EXT=".exe" fi + if [[ "${{ runner.os }}" == "macOS" ]]; then + export PATH="/opt/homebrew/opt/llvm@20/bin:/opt/homebrew/opt/lld@20/bin:$PATH" + fi + ./build/bin/unit_tests${EXE_EXT} --test-dir="./tests/data" uv run pytest -s --log-cli-level=INFO tests/integration --executable=./build/bin/clice${EXE_EXT} diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e907723..dd9552fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,8 +27,12 @@ include("${PROJECT_SOURCE_DIR}/cmake/package.cmake") add_library(clice_options INTERFACE) +if(CLICE_ENABLE_TEST) + target_compile_definitions(clice_options INTERFACE CLICE_ENABLE_TEST=1) +endif() + if(CLICE_CI_ENVIRONMENT) - target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT) + target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT=1) endif() if(WIN32) diff --git a/bin/clice.cc b/bin/clice.cc index eac56e11..1885fbc8 100644 --- a/bin/clice.cc +++ b/bin/clice.cc @@ -94,11 +94,11 @@ int main(int argc, const char** argv) { logging::stderr_logger("clice", logging::options); if(auto result = fs::init_resource_dir(argv[0]); !result) { - LOGGING_FATAL("Cannot find default resource directory, because {}", result.error()); + LOG_FATAL("Cannot find default resource directory, because {}", result.error()); } for(int i = 0; i < argc; ++i) { - LOGGING_INFO("argv[{}] = {}", i, argv[i]); + LOG_INFO("argv[{}] = {}", i, argv[i]); } async::init(); @@ -112,13 +112,13 @@ int main(int argc, const char** argv) { switch(mode) { case Mode::Pipe: { async::net::listen(loop); - LOGGING_INFO("Server starts listening on stdin/stdout"); + LOG_INFO("Server starts listening on stdin/stdout"); break; } case Mode::Socket: { async::net::listen(host.c_str(), port, loop); - LOGGING_INFO("Server starts listening on {}:{}", host.getValue(), port.getValue()); + LOG_INFO("Server starts listening on {}:{}", host.getValue(), port.getValue()); break; } @@ -130,7 +130,7 @@ int main(int argc, const char** argv) { async::run(); - LOGGING_INFO("clice exit normally!"); + LOG_INFO("clice exit normally!"); return 0; } diff --git a/include/Compiler/Command.h b/include/Compiler/Command.h index 4b78c78e..73fb3cf0 100644 --- a/include/Compiler/Command.h +++ b/include/Compiler/Command.h @@ -21,7 +21,7 @@ struct CommandOptions { bool resource_dir = false; /// Query the compiler driver for additional information, such as system includes and target. - bool query_driver = false; + bool query_toolchain = false; /// Suppress the warning log if failed to query driver info. /// Set true in unittests to avoid cluttering test output. @@ -35,36 +35,34 @@ struct CommandOptions { }; enum class UpdateKind : std::uint8_t { - Unchange, - Create, - Update, - Delete, -}; - -struct DriverInfo { - /// The target of this driver. - llvm::StringRef target; - - /// The default system includes of this driver. - llvm::ArrayRef system_includes; + Unchanged, + Inserted, + Deleted, }; struct UpdateInfo { /// The kind of update. UpdateKind kind; - llvm::StringRef file; + /// The updated file. + std::uint32_t path_id; + + /// The compilation context of this file command, which could + /// be used to identity the same file with different compilation + /// contexts. + const void* context; }; -struct LookupInfo { +struct CompilationContext { + /// The working directory of compilation. llvm::StringRef directory; + /// The compilation arguments. std::vector arguments; - - /// The include arguments indices in the arguments list. - std::vector include_indices; }; +std::string print_argv(llvm::ArrayRef args); + class CompilationDatabase { public: CompilationDatabase(); @@ -79,45 +77,48 @@ public: ~CompilationDatabase(); -private: - struct Impl; - - using Self = CompilationDatabase; - public: + /// Read the compilation database on the give file and return the + /// incremental update infos. + std::vector load_compile_database(llvm::StringRef file); + + /// Lookup the compilation context of specific file. If the context + /// param is provided, we will return the compilation context corresponding + /// to the handle. Otherwise we just return the first one(if the file have) + /// multiple compilation contexts. + CompilationContext lookup(llvm::StringRef file, + const CommandOptions& options = {}, + const void* context = nullptr); + + /// TODO: list all compilation context of the file, this is useful to show + /// all contexts and let user choose one. + /// std::vector fetch_all(llvm::StringRef file); + /// Get an the option for specific argument. static std::optional get_option_id(llvm::StringRef argument); - auto save_string(llvm::StringRef string) -> llvm::StringRef; - - /// Query the compiler driver and return its driver info. - auto query_driver(llvm::StringRef driver) - -> std::expected; - - /// Update with arguments. - auto update_command(llvm::StringRef directory, - llvm::StringRef file, - llvm::ArrayRef arguments) -> UpdateInfo; - - /// Update with full command. - auto update_command(llvm::StringRef directory, llvm::StringRef file, llvm::StringRef command) - -> UpdateInfo; - - /// Update commands from json file and return all updated file. - auto load_commands(llvm::StringRef json_content, llvm::StringRef workspace) - -> std::expected, std::string>; - - /// Load compile commands from given directories. If no valid commands are found, - /// search recursively from the workspace directory. - auto load_compile_database(llvm::ArrayRef compile_commands_dirs, - llvm::StringRef workspace) -> void; - - /// Get compile command from database. `file` should has relative path of workspace. - auto lookup(llvm::StringRef file, CommandOptions options = {}) -> LookupInfo; - + /// FIXME: bad interface design ... std::vector files(); + /// FIXME: remove this api? + auto save_string(llvm::StringRef string) -> llvm::StringRef; + +#ifdef CLICE_ENABLE_TEST + + void add_command(llvm::StringRef directory, + llvm::StringRef file, + llvm::ArrayRef arguments); + + void add_command(llvm::StringRef directory, llvm::StringRef file, llvm::StringRef command); + + /// FIXME: remove this + /// Update commands from json file and return all updated file. + std::expected, std::string> load_commands(llvm::StringRef json_content, + llvm::StringRef workspace); +#endif + private: + struct Impl; std::unique_ptr self; }; diff --git a/include/Compiler/Compilation.h b/include/Compiler/Compilation.h index e25406cc..73e9ff79 100644 --- a/include/Compiler/Compilation.h +++ b/include/Compiler/Compilation.h @@ -23,6 +23,8 @@ struct CompilationParams { std::string directory; + bool arguments_from_database = false; + /// Responsible for storing the arguments. std::vector arguments; diff --git a/include/Compiler/Toolchain.h b/include/Compiler/Toolchain.h index f429e174..c6e3b0eb 100644 --- a/include/Compiler/Toolchain.h +++ b/include/Compiler/Toolchain.h @@ -1,51 +1,49 @@ #pragma once -#include -#include "Support/Enum.h" -#include "Support/Format.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLExtras.h" namespace clice::toolchain { -struct QueryDriverError { - struct ErrorKind : refl::Enum { - enum Kind : std::uint8_t { - NotFoundInPATH, - NotImplemented, - FailToCreateTempFile, - InvokeDriverFail, - OutputFileNotReadable, - InvalidOutputFormat, - }; - - using Enum::Enum; - }; - - ErrorKind kind; - std::string detail; +enum class CompilerFamily { + Unknown, + GCC, // Covers gcc, g++, cc, c++, and versioned/arch variants + Clang, // Covers clang, clang++, and versioned variants (excluding clang-cl) + MSVC, // Covers cl + ClangCL, // Covers clang-cl explicitly + NVCC, // Covers nvcc + Intel, // Covers icc, icpc, icx, dpcpp + Zig, // Covers zig cc / zig c++ (assumed GCC/Clang compatible for query) }; -struct QueryResult { - std::string target; - llvm::SmallVector includes; +struct QueryParams { + llvm::StringRef file; + llvm::StringRef directory; + llvm::ArrayRef arguments; + llvm::function_ref callback; }; -enum class Kind {}; +/// Query the toolchain info and return the full arguments, the returned arguments should be +/// converted to `clang::CompilerInvocation::CreateFromArgs` directly. +std::vector query_toolchain(const QueryParams& params); -struct Toolchain {}; +CompilerFamily driver_family(llvm::StringRef driver); -/// Query toolchain info according to the original compilation arguments. -Toolchain query_toolchain(llvm::ArrayRef arguments); +/// Query g++ or mingw toolchain info. We detect the target and corresponding +/// gcc toolchain install path as default behavior. +std::vector query_gcc_toolchain(const QueryParams& params); -auto query_driver(llvm::StringRef driver) -> std::expected; +/// Query clang++ or any clang based toolchain, e.g. zig cc/c++. We query +/// the full cc1 command of clang toolchain as default. +/// TODO: Is armclang also compatible? +std::vector query_clang_toolchain(const QueryParams& params); + +/// Query the msvc or clang-cl toolchain, default behavior only adds the +/// target and includes info. +std::vector query_msvc_toolchain(const QueryParams& params); + +/// FIXME: To be implemented +std::vector query_nvcc_toolchain(const QueryParams& params); } // namespace clice::toolchain - -template <> -struct std::formatter : std::formatter { - template - auto format(const clice::toolchain::QueryDriverError& e, FormatContext& ctx) const { - return std::format_to(ctx.out(), "{} {}", e.kind.name(), e.detail); - } -}; diff --git a/include/Support/FileSystem.h b/include/Support/FileSystem.h index fa22c1b1..feeb45b0 100644 --- a/include/Support/FileSystem.h +++ b/include/Support/FileSystem.h @@ -50,6 +50,8 @@ inline std::expected init_resource_dir(llvm::StringRef execut return std::expected(); } +using llvm::sys::fs::createTemporaryFile; + inline std::expected createTemporaryFile(llvm::StringRef prefix, llvm::StringRef suffix) { llvm::SmallString<128> path; diff --git a/include/Support/Logging.h b/include/Support/Logging.h index f5893fa4..d2b4400f 100644 --- a/include/Support/Logging.h +++ b/include/Support/Logging.h @@ -92,14 +92,26 @@ void critical [[noreturn]] (logging_format fmt, Args&&... args) { } // namespace clice::logging -#define LOGGING_MESSAGE(name, fmt, ...) \ +#define LOG_MESSAGE(name, fmt, ...) \ if(clice::logging::options.level <= clice::logging::Level::name) { \ clice::logging::name(fmt __VA_OPT__(, ) __VA_ARGS__); \ } -#define LOGGING_TRACE(fmt, ...) LOGGING_MESSAGE(trace, fmt, __VA_ARGS__) -#define LOGGING_DEBUG(fmt, ...) LOGGING_MESSAGE(debug, fmt, __VA_ARGS__) -#define LOGGING_INFO(fmt, ...) LOGGING_MESSAGE(info, fmt, __VA_ARGS__) -#define LOGGING_WARN(fmt, ...) LOGGING_MESSAGE(warn, fmt, __VA_ARGS__) -#define LOGGING_ERROR(fmt, ...) LOGGING_MESSAGE(err, fmt, __VA_ARGS__) -#define LOGGING_FATAL(fmt, ...) clice::logging::critical(fmt __VA_OPT__(, ) __VA_ARGS__); +#define LOG_TRACE(fmt, ...) LOG_MESSAGE(trace, fmt, __VA_ARGS__) +#define LOG_DEBUG(fmt, ...) LOG_MESSAGE(debug, fmt, __VA_ARGS__) +#define LOG_INFO(fmt, ...) LOG_MESSAGE(info, fmt, __VA_ARGS__) +#define LOG_WARN(fmt, ...) LOG_MESSAGE(warn, fmt, __VA_ARGS__) +#define LOG_ERROR(fmt, ...) LOG_MESSAGE(err, fmt, __VA_ARGS__) +#define LOG_FATAL(fmt, ...) clice::logging::critical(fmt __VA_OPT__(, ) __VA_ARGS__); + +#define LOG_MESSAGE_RET(ret, name, fmt, ...) \ + do { \ + LOG_MESSAGE(name, fmt, __VA_ARGS__); \ + return ret; \ + } while(0); + +#define LOG_TRACE_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, trace, fmt, __VA_ARGS__) +#define LOG_DEBUG_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, debug, fmt, __VA_ARGS__) +#define LOG_INFO_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, info, fmt, __VA_ARGS__) +#define LOG_WARN_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, warn, fmt, __VA_ARGS__) +#define LOG_ERROR_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, err, fmt, __VA_ARGS__) diff --git a/include/Support/ObjectPool.h b/include/Support/ObjectPool.h new file mode 100644 index 00000000..21c86cc6 --- /dev/null +++ b/include/Support/ObjectPool.h @@ -0,0 +1,224 @@ +#pragma once + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Support/Allocator.h" + +namespace clice { + +class StringSet { +public: + using ID = std::uint32_t; + + StringSet(llvm::BumpPtrAllocator& allocator) : allocator(allocator) { + strings.emplace_back(); + } + + StringSet(const StringSet&) = delete; + + StringSet(StringSet&&) = delete; + + StringSet& operator= (const StringSet&) = delete; + + StringSet& operator= (StringSet&&) = delete; + + ~StringSet() = default; + + ID get(llvm::StringRef s) { + if(s.empty()) { + return ID(0); + } + + auto [it, success] = cache.try_emplace(s, ID(0)); + if(!success) { + return it->second; + } + + const auto size = s.size(); + auto p = allocator.Allocate(size + 1); + std::memcpy(p, s.data(), size); + p[size] = '\0'; + + it->first = llvm::StringRef(p, size); + it->second = ID(strings.size()); + strings.emplace_back(it->first); + return it->second; + } + + llvm::StringRef get(ID id) { + assert(id < strings.size()); + return strings[id]; + } + + llvm::StringRef save(llvm::StringRef s) { + return get(get(s)); + } + +private: + llvm::BumpPtrAllocator& allocator; + + std::vector strings; + + llvm::DenseMap cache; +}; + +template +struct object_ptr { + T* ptr = nullptr; + + object_ptr() noexcept = default; + + object_ptr(std::nullptr_t) noexcept : ptr(nullptr) {} + + explicit object_ptr(T* p) noexcept : ptr(p) {} + + T& operator* () const noexcept { + return *ptr; + } + + T* operator->() const noexcept { + return ptr; + } + + explicit operator bool() const noexcept { + return ptr != nullptr; + } + + std::strong_ordering operator<=> (const object_ptr&) const = default; +}; + +template +object_ptr(T*) -> object_ptr; + +template +class ObjectSet { +public: + using ID = std::uint32_t; + + ObjectSet(llvm::BumpPtrAllocator& allocator) : allocator(allocator) {} + + ObjectSet(const ObjectSet&) = delete; + + ObjectSet(ObjectSet&&) = delete; + + ObjectSet& operator= (const ObjectSet&) = delete; + + ObjectSet& operator= (ObjectSet&&) = delete; + + ~ObjectSet() { + if constexpr(!std::is_trivially_destructible_v) { + for(auto object: objects) { + if(object) { + std::destroy_at(object.ptr); + } + } + + for(auto& [object, _]: removed) { + if(object) { + std::destroy_at(object.ptr); + } + } + } + } + + ID get(const T& object) { + auto [it, success] = cache.try_emplace(object_ptr(const_cast(&object)), ID(0)); + if(!success) { + return it->second; + } + + if(!removed.empty()) [[unlikely]] { + auto [o, id] = removed.pop_back_val(); + + /// Reuse the old memory + std::destroy_at(o.ptr); + new (o.ptr) T(object); + + it->first = o; + it->second = id; + + /// Resume the object id. + objects[id] = o; + } else { + /// Alloc the new memory. + auto p = allocator.Allocate(1); + p = new (p) T(object); + + it->first = object_ptr{p}; + it->second = ID(objects.size()); + + objects.emplace_back(p); + } + + return it->second; + } + + object_ptr get(ID id) { + assert(id < objects.size()); + return objects[id]; + } + + object_ptr save(const T& object) { + return this->get(this->get(object)); + } + + void remove(object_ptr object) { + auto it = cache.find(object_ptr(object.ptr)); + if(it == cache.end()) { + return; + } + + auto id = it->second; + removed.emplace_back(object, id); + cache.erase(it); + objects[id] = nullptr; + } + +private: + llvm::BumpPtrAllocator& allocator; + + std::vector> objects; + + llvm::SmallVector, ID>> removed; + + llvm::DenseMap, ID> cache; +}; + +} // namespace clice + +namespace llvm { + +template +struct DenseMapInfo> { + using U = std::remove_cvref_t; + using O = clice::object_ptr; + + inline static O getEmptyKey() { + return O(DenseMapInfo::getEmptyKey()); + } + + inline static O getTombstoneKey() { + return O(DenseMapInfo::getTombstoneKey()); + } + + inline static unsigned getHashValue(O value) { + return DenseMapInfo::getHashValue(*value); + } + + inline static bool isEqual(O lhs, O rhs) { + if(lhs == rhs) { + return true; + }; + + const O Empty = getEmptyKey(); + const O Tombstone = getTombstoneKey(); + + if(lhs == Empty || rhs == Empty || lhs == Tombstone || rhs == Tombstone) { + return false; + } + + return DenseMapInfo::isEqual(*lhs, *rhs); + } +}; + +} // namespace llvm diff --git a/include/Test/Tester.h b/include/Test/Tester.h index 3cefd2c9..590f3304 100644 --- a/include/Test/Tester.h +++ b/include/Test/Tester.h @@ -5,6 +5,7 @@ #include "Protocol/Protocol.h" #include "Compiler/Command.h" #include "Compiler/Compilation.h" +#include "Support/Logging.h" namespace clice::testing { @@ -31,157 +32,23 @@ struct Tester { sources.add_sources(content); } - void prepare(llvm::StringRef standard = "-std=c++20") { - auto command = std::format("clang++ {} {} -fms-extensions", standard, src_path); + void prepare(llvm::StringRef standard = "-std=c++20"); - database.update_command("fake", src_path, command); - params.kind = CompilationUnit::Content; + bool compile(llvm::StringRef standard = "-std=c++20"); - CommandOptions options; - options.resource_dir = true; - options.query_driver = true; - options.suppress_logging = true; - params.arguments = database.lookup(src_path, options).arguments; - - for(auto& [file, source]: sources.all_files) { - if(file == src_path) { - params.add_remapped_file(file, source.content); - } else { - /// FIXME: This is a workaround. - std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); - params.add_remapped_file(path, source.content); - } - } - } - - bool compile(llvm::StringRef standard = "-std=c++20") { - prepare(standard); - - auto info = clice::compile(params); - if(!info) { - return false; - } - - this->unit.emplace(std::move(*info)); - return true; - } - - bool compile_with_pch(llvm::StringRef standard = "-std=c++20") { - params.diagnostics = std::make_shared>(); - auto command = std::format("clang++ {} {} -fms-extensions", standard, src_path); - - database.update_command("fake", src_path, command); - params.kind = CompilationUnit::Preamble; - - CommandOptions options; - options.resource_dir = true; - options.query_driver = true; - options.suppress_logging = true; - params.arguments = database.lookup(src_path, options).arguments; - - auto path = fs::createTemporaryFile("clice", "pch"); - if(!path) { - llvm::outs() << path.error().message() << "\n"; - } - - /// Build PCH - params.output_file = *path; - - for(auto& [file, source]: sources.all_files) { - if(file == src_path) { - auto bound = compute_preamble_bound(source.content); - params.add_remapped_file(file, source.content, bound); - } else { - /// FIXME: This is a workaround. - std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); - params.add_remapped_file(path, source.content); - } - } - - PCHInfo info; - { - auto unit = clice::compile(params, info); - if(!unit) { - llvm::outs() << unit.error() << "\n"; - for(auto& diag: *params.diagnostics) { - std::println("{}", diag.message); - } - return false; - } - } - - /// Build AST - params.output_file.clear(); - params.kind = CompilationUnit::Content; - params.pch = {info.path, info.preamble.size()}; - for(auto& [file, source]: sources.all_files) { - if(file == src_path) { - params.add_remapped_file(file, source.content); - } else { - /// FIXME: This is a workaround. - std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); - params.add_remapped_file(path, source.content); - } - } - - auto unit = clice::compile(params); - if(!unit) { - return false; - } - - this->unit.emplace(std::move(*unit)); - return true; - } + bool compile_with_pch(llvm::StringRef standard = "-std=c++20"); std::uint32_t operator[] (llvm::StringRef file, llvm::StringRef pos) { return sources.all_files.lookup(file).offsets.lookup(pos); } - std::uint32_t point(llvm::StringRef name = "", llvm::StringRef file = "") { - if(file.empty()) { - file = src_path; - } + std::uint32_t point(llvm::StringRef name = "", llvm::StringRef file = ""); - auto& offsets = sources.all_files[file].offsets; - if(name.empty()) { - assert(offsets.size() == 1); - return offsets.begin()->second; - } else { - assert(offsets.contains(name)); - return offsets.lookup(name); - } - } + llvm::ArrayRef nameless_points(llvm::StringRef file = ""); - llvm::ArrayRef nameless_points(llvm::StringRef file = "") { - if(file.empty()) { - file = src_path; - } + LocalSourceRange range(llvm::StringRef name = "", llvm::StringRef file = ""); - return sources.all_files[file].nameless_offsets; - } - - LocalSourceRange range(llvm::StringRef name = "", llvm::StringRef file = "") { - if(file.empty()) { - file = src_path; - } - - auto& ranges = sources.all_files[file].ranges; - if(name.empty()) { - assert(ranges.size() == 1); - return ranges.begin()->second; - } else { - assert(ranges.contains(name)); - return ranges.lookup(name); - } - } - - void clear() { - params = CompilationParams(); - database = CompilationDatabase(); - unit.reset(); - sources.all_files.clear(); - src_path.clear(); - } + void clear(); }; } // namespace clice::testing diff --git a/src/AST/Selection.cpp b/src/AST/Selection.cpp index 6b490582..5d8b817b 100644 --- a/src/AST/Selection.cpp +++ b/src/AST/Selection.cpp @@ -948,10 +948,10 @@ private: } if(!checker.may_hit(S)) { - LOGGING_DEBUG("{2}skip: {0} {1}", - print_node_to_string(N, print_policy), - S.printToString(SM), - indent()); + LOG_DEBUG("{2}skip: {0} {1}", + print_node_to_string(N, print_policy), + S.printToString(SM), + indent()); return true; } @@ -971,10 +971,10 @@ private: // Performs early hit detection for some nodes (on the earlySourceRange). void push(clang::DynTypedNode node) { clang::SourceRange Early = early_source_range(node); - LOGGING_DEBUG("{2}push: {0} {1}", - print_node_to_string(node, print_policy), - node.getSourceRange().printToString(SM), - indent()); + LOG_DEBUG("{2}push: {0} {1}", + print_node_to_string(node, print_policy), + node.getSourceRange().printToString(SM), + indent()); nodes.emplace_back(); nodes.back().data = std::move(node); nodes.back().parent = stack.top(); @@ -987,7 +987,7 @@ private: // Performs primary hit detection. void pop() { Node& N = *stack.top(); - LOGGING_DEBUG("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1)); + LOG_DEBUG("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1)); claim_tokens_for(N.data, N.selected); if(N.selected == no_tokens) { N.selected = SelectionTree::Unselected; @@ -1118,7 +1118,7 @@ private: } if(result && result != no_tokens) { - LOGGING_DEBUG("{1}hit selection: {0}", S.printToString(SM), indent()); + LOG_DEBUG("{1}hit selection: {0}", S.printToString(SM), indent()); } } @@ -1243,9 +1243,9 @@ SelectionTree::SelectionTree(CompilationUnit& unit, LocalSourceRange range) : print_policy.IncludeNewlines = false; auto [begin, end] = range; - LOGGING_DEBUG("Computing selection for {0}", - clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end)) - .printToString(SM)); + LOG_DEBUG("Computing selection for {0}", + clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end)) + .printToString(SM)); nodes = SelectionVisitor::collect(unit, print_policy, range, fid); m_root = nodes.empty() ? nullptr : &nodes.front(); diff --git a/src/Async/FileSystem.cpp b/src/Async/FileSystem.cpp index 89f2592a..a63e52cf 100644 --- a/src/Async/FileSystem.cpp +++ b/src/Async/FileSystem.cpp @@ -122,7 +122,7 @@ handle::~handle() { uv_fs_t request; int error = uv_fs_close(async::loop, &request, file, nullptr); if(error < 0) { - LOGGING_WARN("Failed to close file: {}", uv_strerror(error)); + LOG_WARN("Failed to close file: {}", uv_strerror(error)); } uv_fs_req_cleanup(&request); } diff --git a/src/Async/Network.cpp b/src/Async/Network.cpp index 6cdb7515..621b4f7e 100644 --- a/src/Async/Network.cpp +++ b/src/Async/Network.cpp @@ -30,7 +30,7 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) { /// If an error occurred while reading, we can't continue. if(nread < 0) [[unlikely]] { - LOGGING_FATAL("An error occurred while reading: {0}", uv_strerror(nread)); + LOG_FATAL("An error occurred while reading: {0}", uv_strerror(nread)); } /// We have at most one connection and use default event loop. So there is no data race @@ -57,7 +57,7 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) { task.dispose(); } else { /// If the message is invalid, we can't continue. - LOGGING_FATAL("Unexpected JSON input: {0}", result); + LOG_FATAL("Unexpected JSON input: {0}", result); } /// Remove the processed message from the buffer. @@ -130,7 +130,7 @@ struct write { uv_write(&req, writer, buf, 2, [](uv_write_t* req, int status) { if(status < 0) { - LOGGING_FATAL("An error occurred while writing: {0}", uv_strerror(status)); + LOG_FATAL("An error occurred while writing: {0}", uv_strerror(status)); } auto& awaiter = uv_cast(req); diff --git a/src/Async/libuv.cpp b/src/Async/libuv.cpp index 3527710b..179e6e5b 100644 --- a/src/Async/libuv.cpp +++ b/src/Async/libuv.cpp @@ -29,11 +29,8 @@ const std::error_category& category() { /// Use source_location to log the file, line, and function name where the error occurred. void uv_check_result(const int result, const std::source_location location) { if(result < 0) { - LOGGING_WARN("libuv error: {}", uv_strerror(result)); - LOGGING_WARN("At {}:{}:{}", - location.file_name(), - location.line(), - location.function_name()); + LOG_WARN("libuv error: {}", uv_strerror(result)); + LOG_WARN("At {}:{}:{}", location.file_name(), location.line(), location.function_name()); } } diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index dd8b3a27..8ac73d52 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -4,25 +4,103 @@ #include "Support/Logging.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Program.h" #include "Driver.h" +#include "Support/ObjectPool.h" + +namespace clice { + +namespace { + +using StringID = StringSet::ID; + +struct CompilationInfo { + /// The working directory of the compilation. + StringID directory = 0; + + /// The canonical compilation arguments(input file and output file are removed). + llvm::ArrayRef arguments; + + friend bool operator== (const CompilationInfo&, const CompilationInfo&) = default; +}; + +/// An item in the compilation database. +struct JSONItem { + /// The path of the source json file, so that we can know where this + /// json item from. + StringID json_src_path = 0; + + /// The file path of this json item. + StringID file_path = 0; + + /// The canonical compilation info of this item. + object_ptr info = {nullptr}; + + /// A file may have multiple compilation commands, we use + /// a chain to connect them. Note that this field does't + /// get involved in equality judgement or hash computing. + object_ptr next = {nullptr}; + + friend bool operator== (const JSONItem& lhs, const JSONItem& rhs) { + return lhs.json_src_path == rhs.json_src_path && lhs.file_path == rhs.file_path && + lhs.info == rhs.info; + } + + friend bool operator< (const JSONItem& lhs, const JSONItem& rhs) { + return std::tie(lhs.file_path, lhs.info) < std::tie(rhs.file_path, rhs.info); + } +}; + +struct JSONSource { + /// The path of the source json file. + StringID src_path; + + /// All json items in the json file, used for increment update. + std::vector> items; +}; + +using ID = clang::driver::options::ID; + +} // namespace + +} // namespace clice namespace llvm { template <> -struct DenseMapInfo> { - using T = llvm::ArrayRef; +struct DenseMapInfo { + using T = clice::CompilationInfo; inline static T getEmptyKey() { - return T(reinterpret_cast(~0), T::size_type(0)); + return T(llvm::DenseMapInfo::getEmptyKey()); } inline static T getTombstoneKey() { - return T(reinterpret_cast(~1), T::size_type(0)); + return T(llvm::DenseMapInfo::getTombstoneKey()); + } + + static unsigned getHashValue(const T& info) { + return llvm::hash_combine(info.directory, llvm::hash_combine_range(info.arguments)); + } + + static bool isEqual(const T& lhs, const T& rhs) { + return lhs == rhs; + } +}; + +template <> +struct DenseMapInfo { + using T = clice::JSONItem; + + inline static T getEmptyKey() { + return T(0, llvm::DenseMapInfo::getEmptyKey()); + } + + inline static T getTombstoneKey() { + return T(0, llvm::DenseMapInfo::getTombstoneKey()); } static unsigned getHashValue(const T& value) { - return llvm::hash_combine_range(value.begin(), value.end()); + return llvm::hash_combine(value.json_src_path, value.file_path, value.info.ptr); } static bool isEqual(const T& lhs, const T& rhs) { @@ -34,98 +112,266 @@ struct DenseMapInfo> { namespace clice { -namespace options = clang::driver::options; - -struct CommandInfo { - /// TODO: add sysroot or no stdinc command info. - llvm::StringRef directory; - - /// The canonical command list. - llvm::ArrayRef arguments; - - /// The extra command @... - llvm::StringRef response_file; - - /// The original index of the response file argument in the command list. - std::uint32_t response_file_index = 0; -}; - struct CompilationDatabase::Impl { - /// The memory pool to hold all cstring and command list. + /// The memory pool which holds all elements of compilation database. + /// We never try to release the memory until it destructs. So don't + /// worry about the lifetime of allocated elements. llvm::BumpPtrAllocator allocator; - /// A cache between input string and its cache cstring - /// in the allocator, make sure end with `\0`. - llvm::DenseSet string_cache; + /// Keep all strings. + StringSet strings = {allocator}; - /// A cache between input command and its cache array - /// in the allocator. - llvm::DenseSet> arguments_cache; + /// Keep all items in the `compile_commands.json`. + ObjectSet items = {allocator}; + + /// Keep all canonical command infos, most of file actually + /// have the same canonical command. + ObjectSet infos = {allocator}; + + /// All json source file. + llvm::SmallVector sources; + + /// All source files in the compilation database. + llvm::DenseMap> files; + + /// TODO: Cache of toolchain query driver results. + llvm::DenseMap toolchains; /// The clang options we want to filter in all cases, like -c and -o. llvm::DenseSet filtered_options; - /// A map between file path and its canonical command list. - llvm::DenseMap command_infos; - - /// A map between driver path and its query driver info. - llvm::DenseMap driver_infos; - ArgumentParser parser = {&allocator}; - using Self = CompilationDatabase::Impl; + auto save_compilation_info(this Impl& self, + llvm::StringRef file, + llvm::StringRef directory, + llvm::ArrayRef arguments) { + llvm::SmallVector stored_arguments; - auto save_string(this Self& self, llvm::StringRef string) -> llvm::StringRef { - assert(!string.empty() && "expected non empty string"); - auto it = self.string_cache.find(string); + self.parser.set_arguments(arguments); + /// We don't want to parse all arguments here, it is time-consuming. But we + /// want to remove output and input file from arguments. They are main reasons + /// causing different file have different commands. + for(unsigned it = 0; it != arguments.size(); it++) { + llvm::StringRef argument = arguments[it]; - /// If we already store the argument, reuse it. - if(it != self.string_cache.end()) { - return *it; + /// FIXME: Is it possible that file in command and field are different? + if(argument == file) { + continue; + } + + /// All possible output options prefix. + constexpr static std::string_view output_options[] = { + "-o", + "--output", + "/o", + "/Fo", + "/Fe", + }; + + /// FIXME: This is a heuristic approach that covers the vast majority of cases, but + /// theoretical corner cases exist. For example, `-oxx` might be an argument for another + /// command, and processing it this way would lead to its incorrect removal. To fix + /// these corner cases, it's necessary to parse the command line fully. Additionally, + /// detailed benchmarks should be conducted to determine the time required for parsing + /// command-line arguments in order to decide if it's worth doing so. + if(ranges::any_of(output_options, [&](llvm::StringRef option) { + return argument.starts_with(option); + })) { + auto prev = it; + auto arg = self.parser.parse_one(it); + + /// FIXME: How to handle parse error here? + if(!arg) { + it = prev; + continue; + } + + auto id = arg->getOption().getID(); + if(id == ID::OPT_o || id == ID::OPT_dxc_Fo || id == ID::OPT__SLASH_o || + id == ID::OPT__SLASH_Fo || id == ID::OPT__SLASH_Fe) { + /// It will point to the next argument start but it also increases + /// in the next loop. So decrease it for not skipping next argument. + it -= 1; + continue; + } + + /// This argument doesn't represent output file, just recovery it. + it = prev; + } + + /// FIXME: Handle response file. + if(argument.starts_with("@")) { + LOG_WARN( + "clice currently supports only one response file in the command, when loads {}", + file); + continue; + } + + stored_arguments.emplace_back(self.strings.get(argument)); } - /// Allocate for new string. - const auto size = string.size(); - auto ptr = self.allocator.Allocate(size + 1); - std::memcpy(ptr, string.data(), size); - ptr[size] = '\0'; + auto info_id = self.infos.get({ + self.strings.get(directory), + stored_arguments, + }); - /// Insert it to cache. - auto result = llvm::StringRef(ptr, size); - self.string_cache.insert(result); - return result; - } - - auto save_cstring_list(this Self& self, llvm::ArrayRef arguments) - -> llvm::ArrayRef { - auto it = self.arguments_cache.find(arguments); - - /// If we already store the argument, reuse it. - if(it != self.arguments_cache.end()) { - return *it; + /// Note: check whether the arguments data are same as stored arguments, + /// if so, we need allocate buffer for it to avoid dangling reference. + auto info = self.infos.get(info_id); + if(info->arguments.data() == stored_arguments.data()) { + auto result = self.allocator.Allocate(info->arguments.size()); + std::ranges::copy(info->arguments, result); + info->arguments = {result, info->arguments.size()}; } - /// Allocate for new array. - const auto size = arguments.size(); - auto ptr = self.allocator.Allocate(size); - ranges::copy(arguments, ptr); - - /// Insert it to cache. - auto result = llvm::ArrayRef(ptr, size); - self.arguments_cache.insert(result); - return result; + return info; } - auto process_command(this Self& self, - llvm::StringRef file, - const CommandInfo& info, - const CommandOptions& options) -> std::vector { + auto save_compilation_info(this Impl& self, + llvm::StringRef file, + llvm::StringRef directory, + llvm::StringRef command) { + llvm::BumpPtrAllocator local; + llvm::StringSaver saver(local); + + llvm::SmallVector arguments; + + /// FIXME: We need a better way to handle this. + if(command.contains("cl.exe") || command.contains("clang-cl")) { + llvm::cl::TokenizeWindowsCommandLineFull(command, saver, arguments); + } else { + llvm::cl::TokenizeGNUCommandLine(command, saver, arguments); + } + + return self.save_compilation_info(file, directory, arguments); + } + + void insert_item(this Impl& self, object_ptr item) { + auto [it, success] = self.files.try_emplace(item->file_path, item); + if(success) { + return; + } + + if(!it->second) { + it->second = item; + return; + } + + auto cur = it->second; + while(cur->next) { + cur = cur->next; + } + cur->next = item; + } + + void delete_item(this Impl& self, object_ptr item) { + auto it = self.files.find(item->file_path); + if(it == self.files.end()) { + return; + } + + if(it->second == item) { + it->second = item->next; + return; + } + + auto cur = it->second; + while(cur->next) { + if(cur->next == item) { + cur->next = item->next; + break; + } + cur = cur->next; + } + } + + auto update_source(this Impl& self, JSONSource& source) { + std::vector updates; + + /// We only need to sort the input source items, so that sources in self + /// are already sorted. + ranges::sort(source.items, [](object_ptr lhs, object_ptr rhs) { + return *lhs < *rhs; + }); + + auto it = ranges::find(self.sources, source.src_path, &JSONSource::src_path); + if(it == self.sources.end()) { + for(auto& item: source.items) { + self.insert_item(item); + updates.emplace_back(UpdateKind::Inserted, item->file_path, item->info.ptr); + } + + self.sources.emplace_back(std::move(source)); + } else { + auto& new_items = source.items; + auto& old_items = it->items; + + auto it_new = new_items.begin(); + auto it_old = old_items.begin(); + + while(it_new != new_items.end() && it_old != old_items.end()) { + const auto& new_item = **it_new; + const auto& old_item = **it_old; + + if(new_item == old_item) { + updates.emplace_back(UpdateKind::Unchanged, + new_item.file_path, + new_item.info.ptr); + ++it_new; + ++it_old; + } else if(new_item < old_item) { + self.insert_item(*it_new); + updates.emplace_back(UpdateKind::Inserted, + new_item.file_path, + new_item.info.ptr); + ++it_new; + } else { + self.delete_item(*it_old); + updates.emplace_back(UpdateKind::Deleted, + old_item.file_path, + old_item.info.ptr); + ++it_old; + } + } + + while(it_new != new_items.end()) { + self.insert_item(*it_new); + updates.emplace_back(UpdateKind::Inserted, + (*it_new)->file_path, + (*it_new)->info.ptr); + ++it_new; + } + + while(it_old != old_items.end()) { + self.delete_item(*it_old); + updates.emplace_back(UpdateKind::Deleted, + (*it_old)->file_path, + (*it_old)->info.ptr); + ++it_old; + } + + it->items = std::move(source.items); + } + + return updates; + } + + auto mangle_command(this Impl& self, + llvm::StringRef file, + const CompilationInfo& info, + const CommandOptions& options) { + llvm::StringRef directory = self.strings.get(info.directory); + llvm::SmallVector arguments; + for(auto arg: info.arguments) { + arguments.emplace_back(self.strings.get(arg).data()); + } /// Store the final result arguments. llvm::SmallVector final_arguments; auto add_string = [&](llvm::StringRef argument) { - auto saved = self.save_string(argument); + auto saved = self.strings.save(argument); final_arguments.emplace_back(saved.data()); }; @@ -172,20 +418,17 @@ struct CompilationDatabase::Impl { }; /// Append driver sperately - add_string(info.arguments.front()); + add_string(arguments.front()); using Arg = std::unique_ptr; auto on_error = [&](int index, int count) { - LOGGING_WARN("missing argument index: {}, count: {} when parse: {}", - index, - count, - file); + LOG_WARN("missing argument index: {}, count: {} when parse: {}", index, count, file); }; /// Prepare for removing arguments. llvm::SmallVector remove; for(auto& arg: options.remove) { - remove.push_back(self.save_string(arg).data()); + remove.push_back(self.strings.save(arg).data()); } /// FIXME: Handle unknow remove arguments. @@ -203,7 +446,7 @@ struct CompilationDatabase::Impl { /// FIXME: Append the commands from response file. self.parser.parse( - info.arguments.drop_front(), + llvm::ArrayRef(arguments).drop_front(), [&](Arg arg) { auto& opt = arg->getOption(); auto id = opt.getID(); @@ -233,11 +476,11 @@ struct CompilationDatabase::Impl { /// For arguments -I, convert directory to absolute path. /// i.e xmake will generate commands in this style. - if(id == options::OPT_I && arg->getNumValues() == 1) { + if(id == ID::OPT_I && arg->getNumValues() == 1) { add_string("-I"); llvm::StringRef value = arg->getValue(0); if(!value.empty() && !path::is_absolute(value)) { - add_string(path::join(info.directory, value)); + add_string(path::join(directory, value)); } else { add_string(value); } @@ -245,7 +488,7 @@ struct CompilationDatabase::Impl { } /// A workaround to remove extra PCH when cmake generate PCH flags for clang. - if(id == options::OPT_Xclang && arg->getNumValues() == 1) { + if(id == ID::OPT_Xclang && arg->getNumValues() == 1) { if(remove_pch) { remove_pch = false; return; @@ -269,84 +512,51 @@ struct CompilationDatabase::Impl { return llvm::ArrayRef(final_arguments).vec(); } - - auto guess_or_fallback(this Self& self, llvm::StringRef file) -> LookupInfo { - // Try to guess command from other file in same directory or parent directory - llvm::StringRef dir = path::parent_path(file); - - // Search up to 3 levels of parent directories - int up_level = 0; - while(!dir.empty() && up_level < 3) { - // If any file in the directory has a command, use that command - for(const auto& [other_file, info]: self.command_infos) { - llvm::StringRef other = other_file; - // Filter case that dir is /path/to/foo and there's another directory - // /path/to/foobar - if(other.starts_with(dir) && - (other.size() == dir.size() || path::is_separator(other[dir.size()]))) { - LOGGING_INFO("Guess command for:{}, from existed file: {}", file, other_file); - return LookupInfo{info.directory, info.arguments}; - } - } - dir = path::parent_path(dir); - up_level += 1; - } - - /// FIXME: use a better default case. - // Fallback to default case. - LookupInfo info; - constexpr const char* fallback[] = {"clang++", "-std=c++20"}; - for(const char* arg: fallback) { - info.arguments.emplace_back(self.save_string(arg).data()); - } - return info; - } -}; - -using ID = options::ID; -constexpr static std::array filtered_options = { - /// Remove the input file, we will add input file ourselves. - ID::OPT_INPUT, - - /// -c and -o are meaningless for frontend. - ID::OPT_c, - ID::OPT_o, - ID::OPT_dxc_Fc, - ID::OPT_dxc_Fo, - - /// Remove all ID related to PCH building. - ID::OPT_emit_pch, - ID::OPT_include_pch, - ID::OPT__SLASH_Yu, - ID::OPT__SLASH_Fp, - - /// Remove all ID related to dependency scan. - ID::OPT_E, - ID::OPT_M, - ID::OPT_MM, - ID::OPT_MD, - ID::OPT_MMD, - ID::OPT_MF, - ID::OPT_MT, - ID::OPT_MQ, - ID::OPT_MG, - ID::OPT_MP, - ID::OPT_show_inst, - ID::OPT_show_encoding, - ID::OPT_show_includes, - ID::OPT__SLASH_showFilenames, - ID::OPT__SLASH_showFilenames_, - ID::OPT__SLASH_showIncludes, - ID::OPT__SLASH_showIncludes_user, - - /// Remove all ID related to C++ module, we will - /// build module and set deps ourselves. - ID::OPT_fmodule_file, - ID::OPT_fmodule_output, - ID::OPT_fprebuilt_module_path, }; CompilationDatabase::CompilationDatabase() : self(std::make_unique()) { + constexpr static std::array filtered_options = { + /// Remove the input file, we will add input file ourselves. + ID::OPT_INPUT, + + /// -c and -o are meaningless for frontend. + ID::OPT_c, + ID::OPT_o, + ID::OPT_dxc_Fc, + ID::OPT_dxc_Fo, + + /// Remove all ID related to PCH building. + ID::OPT_emit_pch, + ID::OPT_include_pch, + ID::OPT__SLASH_Yu, + ID::OPT__SLASH_Fp, + + /// Remove all ID related to dependency scan. + ID::OPT_E, + ID::OPT_M, + ID::OPT_MM, + ID::OPT_MD, + ID::OPT_MMD, + ID::OPT_MF, + ID::OPT_MT, + ID::OPT_MQ, + ID::OPT_MG, + ID::OPT_MP, + ID::OPT_show_inst, + ID::OPT_show_encoding, + ID::OPT_show_includes, + ID::OPT__SLASH_showFilenames, + ID::OPT__SLASH_showFilenames_, + ID::OPT__SLASH_showIncludes, + ID::OPT__SLASH_showIncludes_user, + + /// Remove all ID related to C++ module, we will + /// build module and set deps ourselves. + ID::OPT_fmodule_file, + ID::OPT_fmodule_output, + ID::OPT_fprebuilt_module_path, + }; + for(auto opt: filtered_options) { self->filtered_options.insert(opt); } @@ -358,6 +568,171 @@ CompilationDatabase& CompilationDatabase::operator= (CompilationDatabase&& other CompilationDatabase::~CompilationDatabase() = default; +std::vector CompilationDatabase::load_compile_database(llvm::StringRef path) { + auto content = llvm::MemoryBuffer::getFile(path); + if(!content) { + LOG_ERROR("Failed to read compilation database from {}. Reason: {}", + path, + content.getError()); + return {}; + } + + auto json = json::parse(content.get()->getBuffer()); + if(!json) { + LOG_ERROR("Failed to parse compilation database from {}. Reason: {}", + path, + json.takeError()); + return {}; + } + + if(json->kind() != json::Value::Array) { + LOG_ERROR( + "Invalid compilation database format in {}. Reason: Root element must be an array.", + path); + return {}; + } + + JSONSource source; + source.src_path = self->strings.get(path); + + for(size_t i = 0; i < json->getAsArray()->size(); ++i) { + const auto& value = (*json->getAsArray())[i]; + if(value.kind() != json::Value::Object) { + LOG_ERROR( + "Invalid compilation database in {}. Skipping item at index {}. Reason: item is not an object.", + path, + i); + continue; + } + + auto& object = *value.getAsObject(); + auto directory = object.getString("directory"); + if(!directory) { + LOG_ERROR( + "Invalid compilation database in {}. Skipping item at index {}. Reason: 'directory' key is missing.", + path, + i); + continue; + } + + auto file = object.getString("file"); + if(!file) { + LOG_ERROR( + "Invalid compilation database in {}. Skipping item at index {}. Reason: 'file' key is missing.", + path, + i); + continue; + } + + auto arguments = object.getArray("arguments"); + auto command = object.getString("command"); + if(!arguments && !command) { + LOG_ERROR( + "Invalid compilation database in {}. Skipping item at index {}. Reason: neither 'arguments' nor 'command' key is present.", + path, + i); + continue; + } + + JSONItem item; + item.json_src_path = source.src_path; + item.file_path = self->strings.get(*file); + if(arguments) { + llvm::BumpPtrAllocator local; + llvm::StringSaver saver(local); + llvm::SmallVector agrs; + for(auto& argument: *arguments) { + if(argument.kind() == json::Value::String) { + agrs.emplace_back(saver.save(*argument.getAsString()).data()); + } + } + item.info = self->save_compilation_info(*file, *directory, agrs); + } else if(command) { + item.info = self->save_compilation_info(*file, *directory, *command); + } + source.items.emplace_back(self->items.save(item)); + } + + return self->update_source(source); +} + +CompilationContext CompilationDatabase::lookup(llvm::StringRef file, + const CommandOptions& options, + const void* context) { + object_ptr info = nullptr; + + auto path_id = self->strings.get(file); + file = self->strings.get(path_id); + + auto it = self->files.find(path_id); + if(it != self->files.end()) [[unlikely]] { + if(!context) { + /// If context is not provided, we just use the first. + info = it->second->info; + } else { + /// Otherwise find the corresponding one. + auto cur = it->second; + while(cur) { + if(cur->info.ptr == context) { + info = cur->info; + break; + } + cur = cur->next; + } + } + } + + llvm::StringRef directory; + std::vector arguments; + + if(info) { + directory = self->strings.get(info->directory); + arguments = self->mangle_command(file, *info, options); + } else { + arguments = {"clang++", "-std=c++20"}; + } + + auto append_arg = [&](llvm::StringRef s) { + arguments.emplace_back(self->strings.save(s).data()); + }; + + if(options.resource_dir) { + append_arg("-resource-dir"); + append_arg(fs::resource_dir); + } + + if(options.query_toolchain) { + auto callback = [&](const char* s) { + return save_string(s).data(); + }; + toolchain::QueryParams params = {file, directory, arguments, callback}; + + /// FIXME: querying is expensive, we want to cache this ... + arguments = toolchain::query_toolchain(params); + + /// FIXME: we need mangle the arguments again. + /// Work around ... the logic of this should be moved to query ... + bool next_main_file = false; + for(auto& arg: arguments) { + if(arg == llvm::StringRef("-main-file-name")) { + next_main_file = true; + continue; + } + + if(next_main_file) { + arg = self->strings.save(path::filename(file)).data(); + next_main_file = false; + } + } + + arguments.pop_back(); + } + + arguments.emplace_back(file.data()); + + return CompilationContext(directory, std::move(arguments)); +} + std::optional CompilationDatabase::get_option_id(llvm::StringRef argument) { auto& table = clang::driver::getDriverOptTable(); @@ -378,358 +753,54 @@ std::optional CompilationDatabase::get_option_id(llvm::StringRef } } -auto CompilationDatabase::save_string(llvm::StringRef string) -> llvm::StringRef { - return self->save_string(string); -} - -auto CompilationDatabase::query_driver(llvm::StringRef driver) - -> std::expected { - driver = self->save_string(driver).data(); - auto it = self->driver_infos.find(driver.data()); - if(it != self->driver_infos.end()) { - return it->second; - } - - auto driver_info = toolchain::query_driver(driver); - if(!driver_info) { - return std::unexpected(driver_info.error()); - } - - DriverInfo info; - info.target = self->save_string(driver_info->target); - - llvm::SmallVector includes; - for(llvm::StringRef include: driver_info->includes) { - llvm::SmallString<64> buffer; - - /// Make sure the path is absolute, otherwise it may be - /// "/usr/lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13", which - /// interferes with our determination of the resource directory - auto err = fs::real_path(include, buffer); - include = buffer; - - /// Remove resource dir of the driver. - if(err || - include.contains("lib/gcc") - /// FIXME: Only for windows, for Mac removing default resource dir - /// may result in unexpected error. Figure out it. - || include.contains("lib\\clang")) { - continue; - } - includes.emplace_back(self->save_string(include).data()); - } - - info.system_includes = self->save_cstring_list(includes); - self->driver_infos.try_emplace(driver.data(), info); - return info; -} - -auto CompilationDatabase::update_command(llvm::StringRef directory, - llvm::StringRef file, - llvm::ArrayRef arguments) -> UpdateInfo { - self->parser.set_arguments(arguments); - - file = self->save_string(file); - directory = self->save_string(directory); - - llvm::StringRef response_file; - std::uint32_t response_file_index = 0; - - llvm::SmallVector canonical_arguments; - - /// We don't want to parse all arguments here, it is time-consuming. But we - /// want to remove output and input file from arguments. They are main reasons - /// causing different file have different commands. - for(unsigned it = 0; it != arguments.size(); it++) { - llvm::StringRef argument = arguments[it]; - - /// FIXME: Is it possible that file in command and field are different? - if(argument == file) { - continue; - } - - /// All possible output options prefix. - constexpr static std::string_view output_options[] = { - "-o", - "--output", -#ifdef _WIN32 - "/o", - "/Fo", - "/Fe", -#endif - }; - - /// FIXME: This is a heuristic approach that covers the vast majority of cases, but - /// theoretical corner cases exist. For example, `-oxx` might be an argument for another - /// command, and processing it this way would lead to its incorrect removal. To fix - /// these corner cases, it's necessary to parse the command line fully. Additionally, - /// detailed benchmarks should be conducted to determine the time required for parsing - /// command-line arguments in order to decide if it's worth doing so. - if(ranges::any_of(output_options, - [&](llvm::StringRef option) { return argument.starts_with(option); })) { - auto prev = it; - auto arg = self->parser.parse_one(it); - - /// FIXME: How to handle parse error here? - if(!arg) { - it = prev; - continue; - } - - auto id = arg->getOption().getID(); - if(id == options::OPT_o || id == options::OPT_dxc_Fo || id == options::OPT__SLASH_o || - id == options::OPT__SLASH_Fo || id == options::OPT__SLASH_Fe) { - /// It will point to the next argument start but it also increases - /// in the next loop. So decrease it for not skipping next argument. - it -= 1; - continue; - } - - /// This argument doesn't represent output file, just recovery it. - it = prev; - } - - /// Handle response file. - if(argument.starts_with("@")) { - if(!response_file.empty()) { - LOGGING_WARN( - "clice currently supports only one response file in the command, when loads {}", - file); - } - response_file = self->save_string(argument); - response_file_index = it; - continue; - } - - canonical_arguments.push_back(self->save_string(argument).data()); - } - - /// Cache the canonical arguments - arguments = self->save_cstring_list(canonical_arguments); - - UpdateKind kind = UpdateKind::Unchange; - CommandInfo info = { - directory, - arguments, - response_file, - response_file_index, - }; - - auto [it, success] = self->command_infos.try_emplace(file.data(), info); - if(success) { - /// If successfully inserted, we are loading new file. - kind = UpdateKind::Create; - } else { - /// If failed to insert, compare whether need to update. Because we cache - /// all the ref structure here, so just comparing the pointer is fine. - auto& old_info = it->second; - if(old_info.directory.data() != info.directory.data() || - old_info.arguments.data() != info.arguments.data() || - old_info.response_file.data() != info.response_file.data() || - old_info.response_file_index != info.response_file_index) { - kind = UpdateKind::Update; - old_info = info; - } - } - - return UpdateInfo{kind, file}; -} - -auto CompilationDatabase::update_command(llvm::StringRef directory, - llvm::StringRef file, - llvm::StringRef command) -> UpdateInfo { - llvm::BumpPtrAllocator local; - llvm::StringSaver saver(local); - - llvm::SmallVector arguments; - auto [driver, _] = command.split(' '); - driver = path::filename(driver); - driver.consume_back(".exe"); - - /// FIXME: Use a better to handle this. - if(driver.ends_with("cl") || driver.starts_with("clang-cl")) { - llvm::cl::TokenizeWindowsCommandLineFull(command, saver, arguments); - } else { - llvm::cl::TokenizeGNUCommandLine(command, saver, arguments); - } - - return this->update_command(directory, file, arguments); -} - -auto CompilationDatabase::load_commands(llvm::StringRef json_content, llvm::StringRef workspace) - -> std::expected, std::string> { - std::vector infos; - - auto json = json::parse(json_content); - if(!json) { - return std::unexpected(std::format("parse json failed: {}", json.takeError())); - } - - if(json->kind() != json::Value::Array) { - return std::unexpected("compile_commands.json must be an array of object"); - } - - /// FIXME: warn illegal item. - for(auto& item: *json->getAsArray()) { - /// Ignore non-object item. - if(item.kind() != json::Value::Object) { - continue; - } - - auto& object = *item.getAsObject(); - - auto directory = object.getString("directory"); - if(!directory) { - continue; - } - - /// Always store absolute path of source file. - std::string source; - if(auto file = object.getString("file")) { - source = path::is_absolute(*file) ? file->str() : path::join(*directory, *file); - } else { - continue; - } - - if(auto arguments = object.getArray("arguments")) { - /// Construct cstring array. - llvm::BumpPtrAllocator local; - llvm::StringSaver saver(local); - llvm::SmallVector carguments; - - for(auto& argument: *arguments) { - if(argument.kind() == json::Value::String) { - carguments.emplace_back(saver.save(*argument.getAsString()).data()); - } - } - - auto info = this->update_command(*directory, source, carguments); - if(info.kind != UpdateKind::Unchange) { - infos.emplace_back(info); - } - } else if(auto command = object.getString("command")) { - auto info = this->update_command(*directory, source, *command); - if(info.kind != UpdateKind::Unchange) { - infos.emplace_back(info); - } - } - } - - return infos; -} - -auto CompilationDatabase::load_compile_database(llvm::ArrayRef compile_commands_dirs, - llvm::StringRef workspace) -> void { - auto try_load = [this, workspace](llvm::StringRef dir) { - std::string filepath = path::join(dir, "compile_commands.json"); - auto content = fs::read(filepath); - if(!content) { - LOGGING_WARN("Failed to read CDB file: {}, {}", filepath, content.error()); - return false; - } - - auto load = this->load_commands(*content, workspace); - if(!load) { - LOGGING_WARN("Failed to load CDB file: {}. {}", filepath, load.error()); - return false; - } - - LOGGING_INFO("Load CDB file: {} successfully, {} items loaded", filepath, load->size()); - return true; - }; - - if(std::ranges::any_of(compile_commands_dirs, try_load)) { - return; - } - - LOGGING_WARN( - "Can not found any valid CDB file from given directories, search recursively from workspace: {} ...", - workspace); - - return; - - /// FIXME: Recursive workspace scanning shouldn't be enabled by default, as it - /// might scan unintended directories. - std::error_code ec; - for(fs::recursive_directory_iterator it(workspace, ec), end; it != end && !ec; - it.increment(ec)) { - auto status = it->status(); - if(!status) { - continue; - } - - // Skip hidden directories. - llvm::StringRef filename = path::filename(it->path()); - if(fs::is_directory(*status) && filename.starts_with('.')) { - it.no_push(); - continue; - } - - if(fs::is_regular_file(*status) && filename == "compile_commands.json") { - if(try_load(path::parent_path(it->path()))) { - return; - } - } - } - - /// TODO: Add a default command in clice.toml. Or load commands from .clangd ? - LOGGING_WARN( - "Can not found any valid CDB file in current workspace, fallback to default mode."); -} - -auto CompilationDatabase::lookup(llvm::StringRef file, CommandOptions options) -> LookupInfo { - LookupInfo info; - - file = self->save_string(file); - auto it = self->command_infos.find(file.data()); - if(it != self->command_infos.end()) { - info.directory = it->second.directory; - info.arguments = self->process_command(file, it->second, options); - } else { - info = self->guess_or_fallback(file); - } - - auto record = [&info, this](llvm::StringRef argument) { - info.arguments.emplace_back(self->save_string(argument).data()); - }; - - if(options.query_driver) { - llvm::StringRef driver = info.arguments[0]; - /// FIXME: We may want to query with current includes, because some options - /// will affect the driver, e.g. --sysroot. - if(auto driver_info = this->query_driver(driver)) { - /// FIXME: Cache query result to avoid duplicate query. - record("-nostdlibinc"); - - if(!driver_info->target.empty()) { - record(std::format("--target={}", driver_info->target)); - } - - /// FIXME: Cache -I so that we can append directly, avoid duplicate lookup. - for(auto& system_header: driver_info->system_includes) { - record("-isystem"); - record(system_header); - } - } else if(!options.suppress_logging) { - LOGGING_WARN("Failed to query driver:{}, error:{}", driver, driver_info.error()); - } - } - - if(options.resource_dir) { - record(std::format("-resource-dir={}", fs::resource_dir)); - } - - info.arguments.emplace_back(file.data()); - /// TODO: apply rules in clice.toml. - return info; -} - std::vector CompilationDatabase::files() { std::vector result; - for(auto& [file, _]: self->command_infos) { - result.emplace_back(file); + for(auto& [file, _]: self->files) { + result.emplace_back(self->strings.get(file).data()); } return result; } +llvm::StringRef CompilationDatabase::save_string(llvm::StringRef string) { + return self->strings.save(string); +} + +#ifdef CLICE_ENABLE_TEST + +void CompilationDatabase::add_command(llvm::StringRef directory, + llvm::StringRef file, + + llvm::ArrayRef arguments) { + JSONItem item; + item.json_src_path = self->strings.get("fake"); + item.file_path = self->strings.get(file); + item.info = self->save_compilation_info(file, directory, arguments); + self->insert_item(self->items.save(item)); +} + +void CompilationDatabase::add_command(llvm::StringRef directory, + llvm::StringRef file, + llvm::StringRef command) { + JSONItem item; + item.json_src_path = self->strings.get("fake"); + item.file_path = self->strings.get(file); + item.info = self->save_compilation_info(file, directory, command); + self->insert_item(self->items.save(item)); +} + +#endif + +std::string print_argv(llvm::ArrayRef args) { + std::string s = "["; + if(!args.empty()) { + s += args.consume_front(); + for(auto arg: args) { + s += " "; + s += arg; + } + } + s += "]"; + return s; +} + } // namespace clice diff --git a/src/Compiler/Compilation.cpp b/src/Compiler/Compilation.cpp index 66945137..34732ff2 100644 --- a/src/Compiler/Compilation.cpp +++ b/src/Compiler/Compilation.cpp @@ -10,6 +10,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/MultiplexConsumer.h" +#include "Support/Logging.h" namespace clice { @@ -99,20 +100,40 @@ private: auto create_invocation(CompilationParams& params, llvm::IntrusiveRefCntPtr& diagnostic_engine) -> std::unique_ptr { + if(params.arguments.empty()) { + LOG_ERROR_RET(nullptr, "Fail to create invocation: empty argument list from database"); + } - /// Create clang invocation. - clang::CreateInvocationOptions options = { - .Diags = diagnostic_engine, - .VFS = params.vfs, + std::unique_ptr invocation; - /// Avoid replacing -include with -include-pch, also - /// see https://github.com/clangd/clangd/issues/856. - .ProbePrecompiled = false, - }; + /// Arguments from compilation database are already cc1 + if(params.arguments_from_database) { + invocation = std::make_unique(); + if(!clang::CompilerInvocation::CreateFromArgs(*invocation, + llvm::ArrayRef(params.arguments).drop_front(), + *diagnostic_engine, + params.arguments[0])) { + LOG_ERROR_RET(nullptr, + " Fail to create invocation, arguments list is: {}", + print_argv(params.arguments)); + } + } else { + /// Create clang invocation. + clang::CreateInvocationOptions options = { + .Diags = diagnostic_engine, + .VFS = params.vfs, - auto invocation = clang::createInvocation(params.arguments, options); - if(!invocation) { - return nullptr; + /// Avoid replacing -include with -include-pch, also + /// see https://github.com/clangd/clangd/issues/856. + .ProbePrecompiled = false, + }; + + invocation = clang::createInvocation(params.arguments, options); + if(!invocation) { + LOG_ERROR_RET(nullptr, + " Fail to create invocation, arguments list is: {}", + print_argv(params.arguments)); + } } auto& pp_opts = invocation->getPreprocessorOpts(); @@ -336,7 +357,10 @@ CompilationResult preprocess(CompilationParams& params) { } CompilationResult compile(CompilationParams& params) { - return run_clang(params); + return run_clang(params, [](clang::CompilerInstance& instance) { + /// Make sure the output file is empty. + instance.getFrontendOpts().OutputFile.clear(); + }); } CompilationResult compile(CompilationParams& params, PCHInfo& out) { diff --git a/src/Compiler/Tidy.cpp b/src/Compiler/Tidy.cpp index 07812033..10f1f2b8 100644 --- a/src/Compiler/Tidy.cpp +++ b/src/Compiler/Tidy.cpp @@ -334,11 +334,11 @@ std::unique_ptr configure(clang::CompilerInstance& instance, return nullptr; } auto file_name = input.getFile(); - LOGGING_INFO("Tidy configure file: {}", file_name); + LOG_INFO("Tidy configure file: {}", file_name); tidy::ClangTidyOptions opts = create_options(); if(opts.Checks) { - LOGGING_INFO("Tidy configure checks: {}", *opts.Checks); + LOG_INFO("Tidy configure checks: {}", *opts.Checks); } { @@ -391,7 +391,7 @@ std::unique_ptr configure(clang::CompilerInstance& instance, checker->context.setCurrentFile(file_name); checker->context.setSelfContainedDiags(true); checker->checks = factories.createChecksForLanguage(&checker->context); - LOGGING_INFO("Tidy configure checks: {}", checker->checks.size()); + LOG_INFO("Tidy configure checks: {}", checker->checks.size()); clang::Preprocessor* pp = &instance.getPreprocessor(); for(const auto& check: checker->checks) { check->registerPPCallbacks(instance.getSourceManager(), pp, pp); diff --git a/src/Compiler/Toolchain.cpp b/src/Compiler/Toolchain.cpp index 8a2ccdec..6a1cec11 100644 --- a/src/Compiler/Toolchain.cpp +++ b/src/Compiler/Toolchain.cpp @@ -1,4 +1,5 @@ #include "Compiler/Toolchain.h" +#include "Compiler/Command.h" #include "Support/FileSystem.h" #include "Support/Logging.h" #include "llvm/ADT/ScopeExit.h" @@ -6,96 +7,184 @@ #include "llvm/Support/Program.h" #include "llvm/Support/FileSystem.h" #include "clang/Driver/Driver.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Tool.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TargetParser/Host.h" + +#ifndef _WIN32 + +#include +extern char** environ; + +static llvm::ArrayRef envs() { + static std::vector storage; + static auto refs = [] { + std::vector refs; + if(environ) { + for(char** env = environ; *env != nullptr; ++env) { + llvm::StringRef s(*env); + if(!s.starts_with("LANG=")) { + storage.emplace_back(*env); + } + } + + storage.emplace_back("LANG=C"); + } + + /// Note that store the reference os strings in the vector + /// is not safe when vector grows capacity. But we store it + /// after all insertion are completed. It's safe here. + for(const auto& s: storage) { + refs.emplace_back(s); + } + + return refs; + }(); + return refs; +} + +#endif + +#ifdef _WIN32 +llvm::StringRef null_dev = "NUL"; +#else +llvm::StringRef null_dev = "/dev/null"; +#endif namespace clice::toolchain { -namespace opt = llvm::opt; -namespace driver = clang::driver; +namespace { -/// Checks if dash-dash (`--`) parsing is enabled. If enabled, all arguments -/// after a standalone `--` are treated as positional arguments (e.g., input files). -bool enable_dash_dash_parsing(const opt::OptTable& table); +std::optional execute_command(llvm::ArrayRef arguments, + bool capture_stdout = false) { + LOG_INFO("Execute command: {}", print_argv(arguments)); -/// Checks if grouped short options are enabled. If enabled, a short option group -/// like `-ab` is parsed as separate options `-a` and `-b`. -bool enable_grouped_short_options(const opt::OptTable& table); - -/// Get the specific toolchain of given target, we mainly use it to get msvc toolchain. -const driver::ToolChain& get_toolchain(driver::Driver& driver, - const opt::ArgList& Args, - const llvm::Triple& Target); - -template -struct Thief { - friend bool enable_dash_dash_parsing(const opt::OptTable& table) { - return table.*MP1; + llvm::SmallString<64> path; + if(auto e = fs::createTemporaryFile("query-toolchain", "clice", path)) { + LOG_ERROR_RET(std::nullopt, "Fail to create temporary file: {}", e); } - friend bool enable_grouped_short_options(const opt::OptTable& table) { - return table.*MP2; + auto _ = llvm::make_scope_exit([&path]() { + if(auto e = fs::remove(path)) { + LOG_ERROR("Fail to remove temporary file: {}", e); + } + }); + +#ifdef _WIN32 + /// If the env is `std::nullopt`, `ExecuteAndWait` will inherit env from parent process, + /// which is very important for msvc and clang on windows. Thay depend on the environment + /// variables to find correct standard library path. + constexpr auto env = std::nullopt; +#else + /// For linux, we should append or modify the "LANG=C" to the env, this is important + /// for gcc with locality. Otherwise, it will output non-ASCII char. We also want + /// to inherit the environment variables like windows. + auto env = envs(); +#endif + + std::optional redirects[3] = { + {null_dev}, // stdin + {capture_stdout ? path.str() : null_dev}, // stdout + {capture_stdout ? null_dev : path.str()}, // stderr + }; + + llvm::SmallVector argv(arguments.begin(), arguments.end()); + + std::string message; + if(int rc = llvm::sys::ExecuteAndWait(arguments[0], + argv, + env, + redirects, + /*SecondsToWait=*/0, + /*MemoryLimit=*/0, + &message)) { + /// FIXME: handle error when rc is positive. + LOG_ERROR_RET(std::nullopt, + "Fail to execute {}, return code is {}, because: {}", + arguments[0], + rc, + message); } - friend const driver::ToolChain& get_toolchain(driver::Driver& driver, - const opt::ArgList& args, - const llvm::Triple& target) { - return (driver.*MP3)(args, target); + auto file = llvm::MemoryBuffer::getFile(path); + if(!file) { + LOG_ERROR_RET(std::nullopt, "Fail to read redirect file: {}", file.getError()); } -}; -template struct Thief<&opt::OptTable::DashDashParsing, - &opt::OptTable::GroupedShortOptions, - &driver::Driver::getToolChain>; - -Toolchain query_toolchain(llvm::ArrayRef arguments) { - llvm::StringRef driver = arguments[0]; - - /// judge tool chain kind ... - - return {}; + return file->get()->getBuffer().str(); } -using ErrorKind = toolchain::QueryDriverError::ErrorKind; +bool query_driver( + llvm::ArrayRef arguments, + llvm::function_ref cc1_args)> callback) { + /// FIXME: collect diagnostic here ... + clang::DiagnosticOptions options; + clang::DiagnosticsEngine engine(new clang::DiagnosticIDs(), + options, + new clang::IgnoringDiagConsumer()); -auto unexpected(ErrorKind kind, std::string message) { - return std::unexpected({kind, std::move(message)}); -}; + llvm::SmallVector list; + list.emplace_back(arguments.consume_front()); + list.emplace_back("-fsyntax-only"); + list.append(arguments.begin(), arguments.end()); + arguments = list; -enum class CompilerFamily { - Unknown, - GCC, // Covers gcc, g++, cc, c++, and versioned/arch variants - Clang, // Covers clang, clang++, and versioned variants (excluding clang-cl) - MSVC, // Covers cl - ClangCL, // Covers clang-cl explicitly - NVCC, // Covers nvcc - Intel, // Covers icc, icpc, icx, dpcpp - Zig, // Covers zig cc / zig c++ (assumed GCC/Clang compatible for query) -}; + /// Note that clang use the `ClangExecutable` to determine the driver mode when + /// --driver-mode is not found in the arguments. and `TargetTriple` is used when + /// non --target argument is found in the arguments list. See + /// `clang::driver::BuildCompilation`. We use default arguments because we will + /// inject related commands before querying. + clang::driver::Driver driver(/*ClangExecutable=*/arguments[0], + /*TargetTriple=*/llvm::sys::getDefaultTargetTriple(), + /*Diags=*/engine); + driver.setCheckInputsExist(false); + driver.setProbePrecompiled(false); -CompilerFamily driver_family(llvm::StringRef driver) { - auto driver_name = llvm::sys::path::filename(driver); - driver_name.consume_back(".exe"); - if(driver_name == "cl") { - return CompilerFamily::MSVC; - } else if(driver_name == "nvcc") { - return CompilerFamily::NVCC; - } else if(driver_name.contains("clang-cl")) { - return CompilerFamily::ClangCL; - } else if(driver_name.contains("clang")) { - return CompilerFamily::Clang; - } else if(driver_name == "cc" || driver_name == "c++" || driver_name.contains("gcc") || - driver_name.contains("g++")) { - return CompilerFamily::GCC; - } else if(driver_name.contains("icpc") || driver_name.contains("icc") || - driver_name.contains("dpcpp") || driver_name.contains("icx")) { - return CompilerFamily::Intel; - } else if(driver_name.contains("zig")) { - return CompilerFamily::Zig; + std::unique_ptr compilation(driver.BuildCompilation(arguments)); + if(!compilation) { + LOG_ERROR_RET(false, "Fail to query driver"); } - return CompilerFamily::Unknown; + + // We expect to get back exactly one command job, if we didn't something + // failed. Offload compilation is an exception as it creates multiple jobs. If + // that's the case, we proceed with the first job. If caller needs a + // particular job, it should be controlled via options (e.g. + // --cuda-{host|device}-only for CUDA) passed to the driver. + const clang::driver::JobList& jobs = compilation->getJobs(); + bool offload_compilation = false; + if(jobs.size() > 1) { + for(auto& action: compilation->getActions()) { + // On MacOSX real actions may end up being wrapped in BindArchAction + if(llvm::isa(action)) { + action = *action->input_begin(); + } + + if(llvm::isa(action)) { + offload_compilation = true; + break; + } + } + } + + auto cmd = llvm::find_if(jobs, [](const clang::driver::Command& cmd) { + return cmd.getCreator().getName() == llvm::StringRef("clang"); + }); + if(cmd == jobs.end()) { + LOG_ERROR_RET(false, "Fail to query driver, clang job was not found!"); + } + + callback(arguments[0], cmd->getArguments()); + return true; } -auto parse_query_result(llvm::StringRef content, QueryResult& info) - -> std::expected { +struct QueryResult { + llvm::StringRef target; + std::vector includes; +}; + +/// TODO: use this to print the output of -v. +void parse_version_result(llvm::StringRef content, QueryResult& info) { const char* TS = "Target: "; const char* SIS = "#include <...> search starts here:"; const char* SIE = "End of search list."; @@ -134,18 +223,69 @@ auto parse_query_result(llvm::StringRef content, QueryResult& info) } if(!found_start_marker) { - return unexpected(ErrorKind::InvalidOutputFormat, "Start marker not found..."); + LOG_ERROR("Failed to parse version output: missing include search start marker"); + return; } if(in_includes_block) { - return unexpected(ErrorKind::InvalidOutputFormat, "End marker not found..."); + LOG_ERROR("Failed to parse version output: unclosed include search block"); + return; } - - return std::expected(); } -auto query_driver(llvm::StringRef driver) -> std::expected { - llvm::SmallString<128> path; +} // namespace + +CompilerFamily driver_family(llvm::StringRef driver) { + auto try_get = [](llvm::StringRef name) { + if(name == "cl") { + return CompilerFamily::MSVC; + } else if(name == "nvcc") { + return CompilerFamily::NVCC; + } else if(name.ends_with("clang") || name.ends_with("clang++")) { + return CompilerFamily::Clang; + } else if(name.ends_with("clang-cl")) { + return CompilerFamily::ClangCL; + } else if(name.ends_with("cc") || name.ends_with("c++") || name.ends_with("gcc") || + name.ends_with("g++")) { + return CompilerFamily::GCC; + } else if(name.contains("icpc") || name.contains("icc") || name.contains("dpcpp") || + name.contains("icx")) { + return CompilerFamily::Intel; + } else if(name.ends_with("zig")) { + return CompilerFamily::Zig; + } + return CompilerFamily::Unknown; + }; + + auto driver_name = llvm::sys::path::filename(driver); + auto family = try_get(driver_name); + if(family != CompilerFamily::Unknown) { + return family; + } + + // Stripping the executable suffix: clang++.exe -> clang++ + driver_name.consume_back(".exe"); + family = try_get(driver_name); + if(family != CompilerFamily::Unknown) { + return family; + } + + // Stripping any trailing version number: clang++3.5 -> clang++ + driver_name = driver_name.rtrim("0123456789.-"); + family = try_get(driver_name); + if(family != CompilerFamily::Unknown) { + return family; + } + + /// Stripping trailing -component. clang++-tot -> clang++ + driver_name = driver_name.slice(0, driver_name.rfind('-')); + family = try_get(driver_name); + return family; +} + +std::vector query_toolchain(const QueryParams& params) { + auto arguments = params.arguments; + llvm::StringRef driver = arguments[0]; /// Note: The name used to invoke the compiler driver affects its behavior. /// For example, `/usr/bin/clang++` is often a symbolic link to @@ -153,130 +293,198 @@ auto query_driver(llvm::StringRef driver) -> std::expected path; + if(!path::is_absolute(driver)) { /// If the path is not absolute path like g++, find it in the env vars. auto program = llvm::sys::findProgramByName(driver); if(!program) { - return unexpected(ErrorKind::NotFoundInPATH, program.getError().message()); + LOG_ERROR_RET({}, "Fail to query driver, cannot find the driver: {}", driver); } path = *program; - driver = path; + driver = path.c_str(); } - /// Check whether we can execute the driver. - if(!llvm::sys::fs::exists(driver) || !llvm::sys::fs::can_execute(driver)) { - /// FIXME: Add whitelisting, blacklisting (do not trust workspace executables), - /// and toolchain integrity checks. - return unexpected(ErrorKind::NotFoundInPATH, ""); + if(!fs::exists(driver) || !fs::can_execute(driver)) { + LOG_ERROR_RET({}, "Fail to query driver, driver: {} is not existent or executable", driver); } + auto params_copy = params; + llvm::SmallVector modified_arguments; + + /// Remove driver + arguments.consume_front(); + modified_arguments.emplace_back(driver.data()); + + /// Remove input file + auto ext = path::extension(params.file); + ext.consume_front("."); + + modified_arguments.append(arguments.begin(), arguments.end()); + + /// Create a file with same suffix of input file, because the input file may + /// not exist in the disk. + llvm::SmallString<64> src_path; + if(auto e = fs::createTemporaryFile("query-toolchain", ext, src_path)) { + LOG_ERROR_RET({}, "Fail to create temporary file: {}", e); + } + auto _ = llvm::make_scope_exit([&src_path]() { + if(auto e = fs::remove(src_path)) { + LOG_ERROR("Fail to remove temporary file: {}", e); + } + }); + modified_arguments.emplace_back(src_path.c_str()); + arguments = modified_arguments; + params_copy.arguments = arguments; + auto family = driver_family(driver); - - /// FIXME: Handle nvcc and intel compiler. - if(family == CompilerFamily::NVCC || family == CompilerFamily::Intel || - family == CompilerFamily::Zig || family == CompilerFamily::Unknown) [[unlikely]] { - /// FIXME: nvcc and intel compilers need further exploration. - /// zig is easy to handle, just use `zig cc` or `zig c++`, then - /// it will behave like clang. - return unexpected(ErrorKind::NotImplemented, ""); - } - - /// Query the compiler for includes information. - if(family == CompilerFamily::GCC || family == CompilerFamily::Clang) { - llvm::SmallString<128> output_path; - if(auto error = - llvm::sys::fs::createTemporaryFile("system-includes", "clice", output_path)) { - return unexpected(ErrorKind::FailToCreateTempFile, error.message()); + switch(family) { + case CompilerFamily::GCC: { + return query_gcc_toolchain(params_copy); } - // If we fail to get the driver infomation, keep the output file for user to debug. - bool keep_output_file = true; - auto clean_up = llvm::make_scope_exit([&output_path, &keep_output_file]() { - if(keep_output_file) { - LOGGING_WARN("Query driver failed, output file:{}", output_path); - return; - } - - if(auto errc = llvm::sys::fs::remove(output_path)) { - LOGGING_WARN("Fail to remove temporary file: {}", errc.message()); - } - }); - - /// FIXME: Is it possible that the output is not in stderr? - std::optional redirects[3] = { - {""}, - {""}, - {output_path.str()}, - }; - -#ifdef _WIN32 - /// If the env is `std::nullopt`, `ExecuteAndWait` will inherit env from parent process, - /// which is very important for msvc and clang on windows. Thay depend on the environment - /// variables to find correct standard library path. - constexpr auto env = std::nullopt; - - llvm::SmallVector argv = {driver, "-E", "-v", "-xc++", "NUL"}; -#else - /// FIXME: We should find a better way to convert "LANG=C", this is important - /// for gcc with locality. Otherwise, it will output non-ASCII char. We also - /// want to inherit the environment variables like windows. - llvm::SmallVector env = {"LANG=C"}; - llvm::SmallVector argv = {driver, "-E", "-v", "-xc++", "/dev/null"}; -#endif - - std::string message; - if(int RC = llvm::sys::ExecuteAndWait(driver, - argv, - env, - redirects, - /*SecondsToWait=*/0, - /*MemoryLimit=*/0, - &message)) { - return unexpected(ErrorKind::InvokeDriverFail, std::move(message)); + case CompilerFamily::Clang: + case CompilerFamily::Zig: { + return query_clang_toolchain(params_copy); + } + case CompilerFamily::MSVC: + case CompilerFamily::ClangCL: { + return query_msvc_toolchain(params_copy); } - auto file = llvm::MemoryBuffer::getFile(output_path); - if(!file) { - return unexpected(ErrorKind::OutputFileNotReadable, file.getError().message()); - } + case CompilerFamily::NVCC: + case CompilerFamily::Intel: + case CompilerFamily::Unknown: { + /// TODO: nvcc and intel compilers need further exploration. + LOG_ERROR("Fail to query driver, unknown supported driver kind: {}, driver is {}", + refl::enum_name(family), + driver); - QueryResult info; - if(auto r = parse_query_result(file.get()->getBuffer(), info)) { - keep_output_file = false; - return info; - } else { - return std::unexpected(r.error()); + std::vector result; + query_driver(params_copy.arguments, + [&](const char* driver, llvm::ArrayRef cc1_args) { + result.emplace_back(params.callback(driver)); + for(auto arg: cc1_args) { + result.emplace_back(params.callback(arg)); + } + }); + return result; } } - - /// For msvc and clang-cl, we don't need to query driver. Just use clang - /// tool chain to find the built includes. - if(family == CompilerFamily::MSVC || family == CompilerFamily::ClangCL) { - /// FIXME: target information? e.g. arm cross compilation. - llvm::StringRef target = "x86_64-pc-windows-msvc"; - - /// An workaround to use clang's toolchain to find vsinstall information - /// and related includes. - clang::DiagnosticOptions options; - thread_local clang::DiagnosticsEngine engine(new clang::DiagnosticIDs(), options); - clang::driver::Driver driver("", target, engine); - llvm::SmallVector args = {"", "-xc++", "NUL"}; - llvm::opt::InputArgList list(args.begin(), args.end()); - auto& toolchain = get_toolchain(driver, list, llvm::Triple(target)); - - /// FIXME: specify specific version of vs? - llvm::opt::ArgStringList includes; - toolchain.AddClangSystemIncludeArgs(list, includes); - - QueryResult info; - info.target = target; - for(auto& include: includes) { - info.includes.emplace_back(include); - } - return info; - } - - std::unreachable(); } +std::vector query_gcc_toolchain(const QueryParams& params) { + auto arguments = params.arguments; + llvm::SmallVector query_arguments; + + llvm::SmallString<64> target; + llvm::SmallString<64> install_path; + + query_arguments = {arguments[0], "-dumpmachine"}; + if(auto content = execute_command(query_arguments, true)) { + target = llvm::StringRef(*content).trim(); + } + + query_arguments = {arguments[0], "-print-search-dirs"}; + if(auto content = execute_command(query_arguments, true)) { + llvm::SmallVector lines; + llvm::StringRef(*content).split(lines, '\n', -1, /*KeepEmpty=*/false); + for(auto line: lines) { + line = line.trim(); + if(line.consume_front_insensitive("install:")) { + install_path = line.trim(); + break; + } + } + } + + target = std::format("--target={}", target); + install_path = std::format("--gcc-install-dir={}", install_path); + + query_arguments.clear(); + query_arguments.emplace_back(arguments.consume_front()); + query_arguments.emplace_back(target.c_str()); + query_arguments.emplace_back(install_path.c_str()); + query_arguments.append(arguments.begin(), arguments.end()); + + std::vector result; + query_driver(query_arguments, [&](const char* driver, llvm::ArrayRef cc1_args) { + result.emplace_back(params.callback(driver)); + for(auto arg: cc1_args) { + result.emplace_back(params.callback(arg)); + } + }); + return result; +} + +std::vector query_clang_toolchain(const QueryParams& params) { + auto arguments = params.arguments; + llvm::SmallVector query_arguments; + + if(driver_family(arguments[0]) == CompilerFamily::Zig) { + /// zig cc or zig c++ consumes two arguments. + query_arguments.emplace_back(arguments.consume_front()); + query_arguments.emplace_back(arguments.consume_front()); + } else { + query_arguments.emplace_back(arguments.consume_front()); + } + + query_arguments.emplace_back("-###"); + query_arguments.emplace_back("-fsyntax-only"); + query_arguments.append(arguments.begin(), arguments.end()); + + std::vector result; + if(auto content = execute_command(query_arguments, false)) { + llvm::SmallVector lines; + llvm::StringRef(*content).split(lines, '\n', -1, /*KeepEmpty=*/false); + + for(llvm::StringRef line: lines) { + line = line.trim(); + + if(line.empty() || line.front() != '"') { + continue; + } + + llvm::SmallVector args; + llvm::BumpPtrAllocator allocator; + llvm::StringSaver saver(allocator); + llvm::cl::TokenizeGNUCommandLine(line, saver, args); + + using namespace std::string_view_literals; + if(args.size() < 2 || args[1] != "-cc1"sv) { + continue; + } + + for(auto arg: args) { + if(arg == "-###"sv) { + continue; + } + result.emplace_back(params.callback(arg)); + } + } + } + return result; +} + +std::vector query_msvc_toolchain(const QueryParams& params) { + auto arguments = params.arguments; + llvm::SmallVector query_arguments; + + query_arguments.emplace_back(arguments.consume_front()); + /// When clang in cl mode, the target will be set to windows-msvc automatically. + /// We don't need to add extra flag. + query_arguments.emplace_back("--driver-mode=cl"); + query_arguments.append(arguments.begin(), arguments.end()); + + std::vector result; + query_driver(query_arguments, [&](const char* driver, llvm::ArrayRef cc) { + result.emplace_back(params.callback(driver)); + for(auto c: cc) { + result.emplace_back(params.callback(c)); + } + }); + return result; +} + +std::vector query_nvcc_toolchain(const QueryParams& params); + } // namespace clice::toolchain diff --git a/src/Feature/Diagnostic.cpp b/src/Feature/Diagnostic.cpp index 08fee0e6..5c11dcb4 100644 --- a/src/Feature/Diagnostic.cpp +++ b/src/Feature/Diagnostic.cpp @@ -32,9 +32,9 @@ json::Value diagnostics(PositionEncodingKind kind, PathMapping mapping, Compilat if(level == DiagnosticLevel::Note || level == DiagnosticLevel::Remark) { /// FIXME: figure out why it may be invalid. if(fid.isInvalid()) { - LOGGING_INFO("code: {}, message: {}", - raw_diagnostic.id.diagnostic_code(), - raw_diagnostic.message); + LOG_INFO("code: {}, message: {}", + raw_diagnostic.id.diagnostic_code(), + raw_diagnostic.message); continue; } diff --git a/src/Feature/Formatting.cpp b/src/Feature/Formatting.cpp index 2f39b2db..e0778920 100644 --- a/src/Feature/Formatting.cpp +++ b/src/Feature/Formatting.cpp @@ -41,7 +41,7 @@ std::vector document_format(llvm::StringRef file, range ? tooling::Range(range->begin, range->length()) : tooling::Range(0, content.size()); auto replacements = format(file, content, selection); if(!replacements) { - LOGGING_INFO("Fail to format for {}\n{}", file, replacements.error()); + LOG_INFO("Fail to format for {}\n{}", file, replacements.error()); return edits; } diff --git a/src/Server/Document.cpp b/src/Server/Document.cpp index 40bf0161..11d4232d 100644 --- a/src/Server/Document.cpp +++ b/src/Server/Document.cpp @@ -12,14 +12,14 @@ void Server::load_cache_info() { auto path = path::join(config.project.cache_dir, "cache.json"); auto file = llvm::MemoryBuffer::getFile(path); if(!file) { - LOGGING_WARN("Fail to load cache info, because: {}", file.getError()); + LOG_WARN("Fail to load cache info, because: {}", file.getError()); return; } llvm::StringRef content = file.get()->getBuffer(); auto json = json::parse(content); if(!json) { - LOGGING_WARN("Fail to load cache info, invalid json: {}", json.takeError()); + LOG_WARN("Fail to load cache info, invalid json: {}", json.takeError()); return; } @@ -30,7 +30,7 @@ void Server::load_cache_info() { auto version = object->getString("version"); if(!version) { - LOGGING_INFO("Fail to load cache info, the cache info is outdated"); + LOG_INFO("Fail to load cache info, the cache info is outdated"); return; } @@ -75,7 +75,7 @@ void Server::load_cache_info() { } } - LOGGING_INFO("Load cache info successfully"); + LOG_INFO("Load cache info successfully"); } void Server::save_cache_info() { @@ -105,20 +105,20 @@ void Server::save_cache_info() { llvm::SmallString<128> temp_path; if(auto error = llvm::sys::fs::createTemporaryFile("cache", "json", temp_path)) { - LOGGING_WARN("Fail to create temporary file for cache info: {}", error.message()); + LOG_WARN("Fail to create temporary file for cache info: {}", error.message()); return; } auto clean_up = llvm::make_scope_exit([&temp_path]() { if(auto errc = llvm::sys::fs::remove(temp_path)) { - LOGGING_WARN("Fail to remove temporary file: {}", errc.message()); + LOG_WARN("Fail to remove temporary file: {}", errc.message()); } }); std::error_code EC; llvm::raw_fd_ostream os(temp_path, EC, llvm::sys::fs::OF_None); if(EC) { - LOGGING_WARN("Fail to open temporary file for writing: {}", EC.message()); + LOG_WARN("Fail to open temporary file for writing: {}", EC.message()); return; } @@ -127,24 +127,26 @@ void Server::save_cache_info() { os.close(); if(os.has_error()) { - LOGGING_WARN("Fail to write cache info to temporary file"); + LOG_WARN("Fail to write cache info to temporary file"); return; } if(auto error = llvm::sys::fs::rename(temp_path, final_path)) { - LOGGING_WARN("Fail to rename temporary file to final cache file: {}", error.message()); + LOG_WARN("Fail to rename temporary file to final cache file: {}", error.message()); return; } clean_up.release(); - LOGGING_INFO("Save cache info successfully"); + LOG_INFO("Save cache info successfully"); } namespace { -bool - check_pch_update(llvm::StringRef content, std::uint32_t bound, LookupInfo& info, PCHInfo& pch) { +bool check_pch_update(llvm::StringRef content, + std::uint32_t bound, + CompilationContext& info, + PCHInfo& pch) { if(content.substr(0, bound) != pch.preamble) { return true; } @@ -168,7 +170,7 @@ bool } /// The actual PCH build task. -async::Task build_pch_task(LookupInfo& info, +async::Task build_pch_task(CompilationContext& info, std::string cache_dir, std::shared_ptr open_file, std::string path, @@ -178,7 +180,7 @@ async::Task build_pch_task(LookupInfo& info, if(!fs::exists(cache_dir)) { auto error = fs::create_directories(cache_dir); if(error) { - LOGGING_WARN("Fail to create directory for PCH building: {}", cache_dir); + LOG_WARN("Fail to create directory for PCH building: {}", cache_dir); co_return false; } } @@ -189,6 +191,7 @@ async::Task build_pch_task(LookupInfo& info, CompilationParams params; params.kind = CompilationUnit::Preamble; params.output_file = path::join(cache_dir, path::filename(path) + ".pch"); + params.arguments_from_database = true; params.arguments = std::move(info.arguments); params.diagnostics = diagnostics; params.add_remapped_file(path, content, bound); @@ -199,7 +202,7 @@ async::Task build_pch_task(LookupInfo& info, command += argument; } - LOGGING_INFO("Start building PCH for {}, command: [{}]", path, command); + LOG_INFO("Start building PCH for {}, command: [{}]", path, command); command.clear(); PCHInfo pch; @@ -220,14 +223,14 @@ async::Task build_pch_task(LookupInfo& info, }); if(!success) { - LOGGING_WARN("Building PCH fails for {}, Because: {}", path, message); + LOG_WARN("Building PCH fails for {}, Because: {}", path, message); for(auto& diagnostic: *diagnostics) { - LOGGING_WARN("{}", diagnostic.message); + LOG_WARN("{}", diagnostic.message); } co_return false; } - LOGGING_INFO("Building PCH successfully for {}", path); + LOG_INFO("Building PCH successfully for {}", path); /// Update the built PCH info. open_file->pch = std::move(pch); @@ -245,7 +248,7 @@ async::Task build_pch_task(LookupInfo& info, async::Task Server::build_pch(std::string file, std::string content) { CommandOptions options; options.resource_dir = true; - options.query_driver = true; + options.query_toolchain = true; auto info = database.lookup(file, options); auto bound = compute_preamble_bound(content); @@ -254,7 +257,7 @@ async::Task Server::build_pch(std::string file, std::string content) { /// Check update ... if(open_file->pch && !check_pch_update(content, bound, info, *open_file->pch)) { /// If not need update, return directly. - LOGGING_INFO("PCH is already up-to-date for {}", file); + LOG_INFO("PCH is already up-to-date for {}", file); co_return true; } @@ -263,12 +266,12 @@ async::Task Server::build_pch(std::string file, std::string content) { if(!task.empty()) { if(task.finished()) { task.release().destroy(); - LOGGING_INFO("Release old pch task!"); + LOG_INFO("Release old pch task!"); } else { task.cancel(); task.dispose(); } - LOGGING_INFO("Cancel old PCH building task!"); + LOG_INFO("Cancel old PCH building task!"); } /// Schedule the new building task. @@ -304,15 +307,16 @@ async::Task<> Server::build_ast(std::string path, std::string content) { auto pch = file->pch; if(!pch) { - LOGGING_FATAL("Expected PCH built at this point"); + LOG_FATAL("Expected PCH built at this point"); } CommandOptions options; options.resource_dir = true; - options.query_driver = true; + options.query_toolchain = true; CompilationParams params; params.kind = CompilationUnit::Content; + params.arguments_from_database = true; params.arguments = database.lookup(path, options).arguments; params.add_remapped_file(path, content); params.pch = {pch->path, pch->preamble.size()}; @@ -324,9 +328,9 @@ async::Task<> Server::build_ast(std::string path, std::string content) { auto ast = co_await async::submit([&] { return compile(params); }); if(!ast) { /// FIXME: Fails needs cancel waiting tasks. - LOGGING_WARN("Building AST fails for {}, Beacuse: {}", path, ast.error()); + LOG_ERROR("Building AST fails for {}, Beacuse: {}", path, ast.error()); for(auto& diagnostic: *file->diagnostics) { - LOGGING_WARN("{}", diagnostic.message); + LOG_ERROR("{}", diagnostic.message); } co_return; } @@ -349,7 +353,7 @@ async::Task<> Server::build_ast(std::string path, std::string content) { /// Dispose the task so that it will destroyed when task complete. file->ast_build_task.dispose(); - LOGGING_INFO("Building AST successfully for {}", path); + LOG_INFO("Building AST successfully for {}", path); } async::Task> Server::add_document(std::string path, std::string content) { @@ -363,12 +367,12 @@ async::Task> Server::add_document(std::string path, st if(!task.empty()) { if(task.finished()) { task.release().destroy(); - LOGGING_INFO("Release old AST building Task!"); + LOG_INFO("Release old AST building Task!"); } else { task.cancel(); task.dispose(); } - LOGGING_INFO("Cancel old AST building Task!"); + LOG_INFO("Cancel old AST building Task!"); } /// Create and schedule a new task. diff --git a/src/Server/Feature.cpp b/src/Server/Feature.cpp index 092826ad..0e20e65a 100644 --- a/src/Server/Feature.cpp +++ b/src/Server/Feature.cpp @@ -31,6 +31,7 @@ auto Server::on_completion(proto::CompletionParams params) -> Result { /// Set compilation params ... . CompilationParams params; params.kind = CompilationUnit::Completion; + params.arguments_from_database = true; params.arguments = database.lookup(path).arguments; params.add_remapped_file(path, content); params.pch = {pch->path, pch->preamble.size()}; @@ -88,6 +89,7 @@ async::Task Server::on_signature_help(proto::SignatureHelpParams pa /// Set compilation params ... . CompilationParams params; params.kind = CompilationUnit::Completion; + params.arguments_from_database = true; params.arguments = database.lookup(path, options).arguments; params.add_remapped_file(path, content); params.pch = {pch->path, pch->preamble.size()}; diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index 4ba15961..0b9e5dd1 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -10,12 +10,13 @@ namespace clice { async::Task<> Indexer::index(llvm::StringRef path) { CompilationParams params; params.kind = CompilationUnit::Indexing; + params.arguments_from_database = true; params.arguments = database.lookup(path).arguments; auto path_id = project_index.path_pool.path_id(path); auto& merged_index = get_index(path_id); if(!merged_index.need_update(project_index.path_pool.paths)) { - LOGGING_INFO("Check update for {}, not need to update", path); + LOG_INFO("Check update for {}, not need to update", path); co_return; } @@ -25,7 +26,7 @@ async::Task<> Indexer::index(llvm::StringRef path) { auto tu_index = co_await async::submit([&]() -> std::optional { auto unit = compile(params); if(!unit) { - LOGGING_INFO("Fail to index for {}, because: {}", path, unit.error()); + LOG_INFO("Fail to index for {}, because: {}", path, unit.error()); return std::nullopt; } @@ -55,7 +56,7 @@ async::Task<> Indexer::index(llvm::StringRef path) { std::move(tu_index->graph.locations), tu_index->main_file_index); - LOGGING_INFO("Successfully index {}", path); + LOG_INFO("Successfully index {}", path); } async::Task<> Indexer::schedule_next() { @@ -107,9 +108,9 @@ void Indexer::load_from_disk() { if(auto content = fs::read(output_path); content && !content->empty()) { /// FIXME: from should return a expected ... project_index = index::ProjectIndex::from(content->data()); - LOGGING_INFO("Load project index form {} successfully", output_path); + LOG_INFO("Load project index form {} successfully", output_path); } else { - LOGGING_INFO("Fail to load project index form {}", output_path); + LOG_INFO("Fail to load project index form {}", output_path); } /// FIXME: check indices update .... @@ -117,9 +118,7 @@ void Indexer::load_from_disk() { void Indexer::save_to_disk() { if(auto err = fs::create_directories(config.project.index_dir)) { - LOGGING_WARN("Fail to create index output dir: {}, because: {}", - config.project.index_dir, - err); + LOG_WARN("Fail to create index output dir: {}, because: {}", config.project.index_dir, err); return; } @@ -139,7 +138,7 @@ void Indexer::save_to_disk() { std::error_code err; llvm::raw_fd_ostream os(output_path, err, fs::CreationDisposition::CD_CreateAlways); if(err) { - LOGGING_INFO("Fail to create output index file: {}, because: {}", output_path, err); + LOG_INFO("Fail to create output index file: {}, because: {}", output_path, err); continue; } @@ -147,7 +146,7 @@ void Indexer::save_to_disk() { auto opath_id = project_index.path_pool.path_id(output_path); project_index.indices.try_emplace(path_id, opath_id); - LOGGING_INFO("Successfully save index for {} to {}", path, output_path); + LOG_INFO("Successfully save index for {} to {}", path, output_path); } } @@ -156,12 +155,12 @@ void Indexer::save_to_disk() { std::error_code err; llvm::raw_fd_ostream os(output_path, err, fs::CreationDisposition::CD_CreateAlways); if(err) { - LOGGING_INFO("Fail to create output index file: {}, because: {}", output_path, err); + LOG_INFO("Fail to create output index file: {}, because: {}", output_path, err); return; } project_index.serialize(os); - LOGGING_INFO("Successfully save project index to {}", output_path); + LOG_INFO("Successfully save project index to {}", output_path); } auto Indexer::lookup(llvm::StringRef path, std::uint32_t offset, RelationKind kind) -> Result { diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index 395fad04..ed841d07 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -3,9 +3,9 @@ namespace clice { async::Task Server::on_initialize(proto::InitializeParams params) { - LOGGING_INFO("Initialize from client: {}, version: {}", - params.clientInfo.name, - params.clientInfo.version); + LOG_INFO("Initialize from client: {}, version: {}", + params.clientInfo.name, + params.clientInfo.version); /// FIXME: adjust position encoding. kind = PositionEncodingKind::UTF16; @@ -17,15 +17,15 @@ async::Task Server::on_initialize(proto::InitializeParams params) { return *params.rootUri; } - LOGGING_FATAL("The client should provide one workspace folder or rootUri at least!"); + LOG_FATAL("The client should provide one workspace folder or rootUri at least!"); })()); /// Initialize configuration. if(auto result = config.parse(workspace)) { - LOGGING_INFO("Config initialized successfully: {0:4}", json::serialize(config)); + LOG_INFO("Config initialized successfully: {0:4}", json::serialize(config)); } else { - LOGGING_WARN("Fail to load config, because: {0}", result.error()); - LOGGING_INFO("Use default config: {0:4}", json::serialize(config)); + LOG_WARN("Fail to load config, because: {0}", result.error()); + LOG_INFO("Use default config: {0:4}", json::serialize(config)); } if(!config.project.logging_dir.empty()) { @@ -36,7 +36,9 @@ async::Task Server::on_initialize(proto::InitializeParams params) { opening_files.set_capability(config.project.max_active_file); /// Load compile commands.json - database.load_compile_database(config.project.compile_commands_dirs, workspace); + for(auto& dir: config.project.compile_commands_dirs) { + database.load_compile_database(path::join(dir, "compile_commands.json")); + } /// Load cache info. load_cache_info(); diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index ab6ed3c8..60268d10 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -124,7 +124,7 @@ Server::Server() : indexer(database, config, kind) { async::Task<> Server::on_receive(json::Value value) { auto object = value.getAsObject(); if(!object) [[unlikely]] { - LOGGING_FATAL("Invalid LSP message, not an object: {}", value); + LOG_FATAL("Invalid LSP message, not an object: {}", value); } /// If the json object has an `id`, it's a request, @@ -135,7 +135,7 @@ async::Task<> Server::on_receive(json::Value value) { if(auto result = object->getString("method")) { method = *result; } else [[unlikely]] { - LOGGING_WARN("Invalid LSP message, method not found: {}", value); + LOG_WARN("Invalid LSP message, method not found: {}", value); if(id) { co_await response(std::move(*id), proto::ErrorCodes::InvalidRequest, @@ -152,7 +152,7 @@ async::Task<> Server::on_receive(json::Value value) { /// Handle request and notification separately. auto it = callbacks.find(method); if(it == callbacks.end()) { - LOGGING_INFO("Ignore unhandled method: {}", method); + LOG_INFO("Ignore unhandled method: {}", method); co_return; } @@ -160,24 +160,24 @@ async::Task<> Server::on_receive(json::Value value) { auto current_id = client_request_id++; auto start_time = std::chrono::steady_clock::now(); - LOGGING_INFO("<-- Handling request: {}({})", method, current_id); + LOG_INFO("<-- Handling request: {}({})", method, current_id); auto result = co_await it->second(*this, std::move(params)); co_await response(std::move(*id), std::move(result)); auto end_time = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end_time - start_time); - LOGGING_INFO("--> Handled request: {}({}) {}ms", method, current_id, duration.count()); + LOG_INFO("--> Handled request: {}({}) {}ms", method, current_id, duration.count()); } else { auto start_time = std::chrono::steady_clock::now(); - LOGGING_INFO("<-- Handling notification: {}", method); + LOG_INFO("<-- Handling notification: {}", method); auto result = co_await it->second(*this, std::move(params)); auto end_time = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end_time - start_time); - LOGGING_INFO("--> Handled notification: {} {}ms", method, duration.count()); + LOG_INFO("--> Handled notification: {} {}ms", method, duration.count()); } co_return; diff --git a/tests/unit/AST/Selection.cpp b/tests/unit/AST/Selection.cpp index 3679eda5..8385696b 100644 --- a/tests/unit/AST/Selection.cpp +++ b/tests/unit/AST/Selection.cpp @@ -213,14 +213,16 @@ std::optional toHalfOpenFileRange(const SourceManager& SM, } // namespace suite<"SelectionTree"> selection = [] { - auto select_right = [](llvm::StringRef code, auto&& callback) { + auto select_right = [](llvm::StringRef code, + auto&& callback, + std::source_location location = std::source_location::current()) { Tester tester; tester.add_main("main.cpp", code); - expect(that % tester.compile()); + fatal / expect(that % tester.compile(), location); /// expect(that % tester.unit->diagnostics().empty()); auto points = tester.nameless_points(); - expect(that % points.size() >= 1); + expect(that % points.size() >= 1, location); LocalSourceRange selected_range; selected_range.begin = points[0]; @@ -229,28 +231,33 @@ suite<"SelectionTree"> selection = [] { callback(tester, tree); }; - auto expect_select = [&](llvm::StringRef code, const char* kind) { - select_right(code, [&](Tester& tester, SelectionTree& tree) { - auto node = tree.common_ancestor(); - if(!kind) { - expect(that % !node); - } else { - expect(that % node); - auto range2 = toHalfOpenFileRange(tester.unit->context().getSourceManager(), - tester.unit->lang_options(), - node->source_range()); - LocalSourceRange range = { - tester.unit->file_offset(range2->getBegin()), - tester.unit->file_offset(range2->getEnd()), - }; + auto expect_select = [&](llvm::StringRef code, + const char* kind, + std::source_location location = std::source_location::current()) { + select_right( + code, + [&](Tester& tester, SelectionTree& tree) { + auto node = tree.common_ancestor(); + if(!kind) { + expect(that % !node, location); + } else { + expect(that % node, location); + auto range2 = toHalfOpenFileRange(tester.unit->context().getSourceManager(), + tester.unit->lang_options(), + node->source_range()); + LocalSourceRange range = { + tester.unit->file_offset(range2->getBegin()), + tester.unit->file_offset(range2->getEnd()), + }; - /// llvm::outs() << tree << "\n"; - /// tree.print(llvm::outs(), *node, 2); + /// llvm::outs() << tree << "\n"; + /// tree.print(llvm::outs(), *node, 2); - expect(that % node->kind() == llvm::StringRef(kind)); - expect(that % range == tester.range()); - } - }); + expect(that % node->kind() == llvm::StringRef(kind), location); + expect(that % range == tester.range(), location); + } + }, + location); }; test("Expressions") = [&] { diff --git a/tests/unit/Compiler/Command.cpp b/tests/unit/Compiler/Command.cpp index a6949c86..9912a8ed 100644 --- a/tests/unit/Compiler/Command.cpp +++ b/tests/unit/Compiler/Command.cpp @@ -93,7 +93,7 @@ suite<"Command"> command = [] { auto expect_strip = [](llvm::StringRef argv, llvm::StringRef result) { CompilationDatabase database; llvm::StringRef file = "main.cpp"; - database.update_command("fake/", file, argv); + database.add_command("fake/", file, argv); CommandOptions options; options.suppress_logging = true; @@ -101,10 +101,10 @@ suite<"Command"> command = [] { }; /// Filter -c, -o and input file. - expect_strip("g++ main.cc", "g++ main.cpp"); + expect_strip("g++ main.cpp", "g++ main.cpp"); expect_strip("clang++ -c main.cpp", "clang++ main.cpp"); expect_strip("clang++ -o main.o main.cpp", "clang++ main.cpp"); - expect_strip("clang++ -c -o main.o main.cc", "clang++ main.cpp"); + expect_strip("clang++ -c -o main.o main.cpp", "clang++ main.cpp"); expect_strip("cl.exe /c /Fomain.cpp.o main.cpp", "cl.exe main.cpp"); /// Filter PCH related. @@ -126,8 +126,8 @@ suite<"Command"> command = [] { using namespace std::literals; CompilationDatabase database; - database.update_command("fake", "test.cpp", "clang++ -std=c++23 test.cpp"sv); - database.update_command("fake", "test2.cpp", "clang++ -std=c++23 test2.cpp"sv); + database.add_command("fake", "test.cpp", "clang++ -std=c++23 test.cpp"sv); + database.add_command("fake", "test2.cpp", "clang++ -std=c++23 test2.cpp"sv); CommandOptions options; options.suppress_logging = true; @@ -157,7 +157,7 @@ suite<"Command"> command = [] { }; CompilationDatabase database; - database.update_command("/fake", "main.cpp", args); + database.add_command("/fake", "main.cpp", args); CommandOptions options; @@ -198,56 +198,24 @@ suite<"Command"> command = [] { skip / test("Module") = [] { /// TODO: CompilationDatabase database; - database.update_command("/fake", - "main.cpp", - llvm::StringRef("clang++ @test.txt -std= main.cpp")); - auto info = database.lookup("main.cpp", {.query_driver = false}); - }; - - skip_unless(CIEnvironment) / test("QueryDriver") = [] { - CompilationDatabase database; - auto info = database.query_driver("clang++"); - - fatal / expect(info); - expect(!info->target.empty()); - expect(!info->system_includes.empty()); - - CompilationParams params; - params.kind = CompilationUnit::Indexing; - params.arguments = { - "clang++", - "-nostdlibinc", - "--target", - info->target.data(), - }; - for(auto& include: info->system_includes) { - params.arguments.push_back("-I"); - params.arguments.push_back(include); - } - params.arguments.push_back("main.cpp"); - - llvm::StringRef hello_world = R"( - #include - int main() { - std::cout << "Hello world!" << std::endl; - return 0; - } - )"; - params.add_remapped_file("main.cpp", hello_world); - expect(compile(params)); + database.add_command("/fake", + "main.cpp", + llvm::StringRef("clang++ @test.txt -std= main.cpp")); + auto info = database.lookup("main.cpp", {.query_toolchain = false}); }; test("ResourceDir") = [] { CompilationDatabase database; using namespace std::literals; - database.update_command("/fake", "main.cpp", "clang++ -std=c++23 test.cpp"sv); + database.add_command("/fake", "main.cpp", "clang++ -std=c++23 test.cpp"sv); auto arguments = database.lookup("main.cpp", {.resource_dir = true}).arguments; - fatal / expect(eq(arguments.size(), 4)); + fatal / expect(eq(arguments.size(), 5)); expect(eq(arguments[0], "clang++"sv)); expect(eq(arguments[1], "-std=c++23"sv)); - expect(eq(arguments[2], std::format("-resource-dir={}", fs::resource_dir))); - expect(eq(arguments[3], "main.cpp"sv)); + expect(eq(arguments[2], "-resource-dir"sv)); + expect(eq(arguments[3], fs::resource_dir)); + expect(eq(arguments[4], "main.cpp"sv)); }; auto expect_load = [](llvm::StringRef content, @@ -273,75 +241,87 @@ suite<"Command"> command = [] { }; /// TODO: add windows path testcase - skip_unless(Linux || MacOS) / test("LoadAbsoluteUnixStyle") = [expect_load] { - constexpr const char* cmake = R"([ - { - "directory": "/home/developer/clice/build", - "command": "/usr/bin/c++ -I/home/developer/clice/include -I/home/developer/clice/build/_deps/libuv-src/include -isystem /home/developer/clice/build/_deps/tomlplusplus-src/include -std=gnu++23 -fno-rtti -fno-exceptions -Wno-deprecated-declarations -Wno-undefined-inline -O3 -o CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o -c /home/developer/clice/src/Driver/clice.cpp", - "file": "/home/developer/clice/src/Driver/clice.cpp", - "output": "CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o" - } - ])"; + // skip_unless(Linux || MacOS) / test("LoadAbsoluteUnixStyle") = [expect_load] { + // constexpr const char* cmake = R"([ + // { + // "directory": "/home/developer/clice/build", + // "command": "/usr/bin/c++ -I/home/developer/clice/include + // -I/home/developer/clice/build/_deps/libuv-src/include -isystem + // /home/developer/clice/build/_deps/tomlplusplus-src/include -std=gnu++23 -fno-rtti + // -fno-exceptions -Wno-deprecated-declarations -Wno-undefined-inline -O3 -o + // CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o -c + // /home/developer/clice/src/Driver/clice.cpp", "file": + // "/home/developer/clice/src/Driver/clice.cpp", "output": + // "CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o" + // } + // ])"; + // + // expect_load(cmake, + // "/home/developer/clice", + // "/home/developer/clice/src/Driver/clice.cpp", + // "/home/developer/clice/build", + // { + // "/usr/bin/c++", + // "-I", + // "/home/developer/clice/include", + // "-I", + // "/home/developer/clice/build/_deps/libuv-src/include", + // "-isystem", + // "/home/developer/clice/build/_deps/tomlplusplus-src/include", + // "-std=gnu++23", + // "-fno-rtti", + // "-fno-exceptions", + // "-Wno-deprecated-declarations", + // "-Wno-undefined-inline", + // "-O3", + // "/home/developer/clice/src/Driver/clice.cpp", + // }); + // }; - expect_load(cmake, - "/home/developer/clice", - "/home/developer/clice/src/Driver/clice.cpp", - "/home/developer/clice/build", - { - "/usr/bin/c++", - "-I", - "/home/developer/clice/include", - "-I", - "/home/developer/clice/build/_deps/libuv-src/include", - "-isystem", - "/home/developer/clice/build/_deps/tomlplusplus-src/include", - "-std=gnu++23", - "-fno-rtti", - "-fno-exceptions", - "-Wno-deprecated-declarations", - "-Wno-undefined-inline", - "-O3", - "/home/developer/clice/src/Driver/clice.cpp", - }); - }; - - skip_unless(Linux || MacOS) / test("LoadRelativeUnixStyle") = [expect_load] { - constexpr const char* xmake = R"([ - { - "directory": "/home/developer/clice", - "arguments": ["/usr/bin/clang", "-c", "-Qunused-arguments", "-m64", "-g", "-O0", "-std=c++23", "-Iinclude", "-I/home/developer/clice/include", "-fno-exceptions", "-fno-cxx-exceptions", "-isystem", "/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include", "-isystem", "/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include", "-fsanitize=address", "-fno-rtti", "-o", "build/.objs/clice/linux/x86_64/debug/src/Driver/clice.cc.o", "src/Driver/clice.cc"], - "file": "src/Driver/clice.cc" - } - ])"; - - expect_load( - xmake, - "/home/developer/clice", - "/home/developer/clice/src/Driver/clice.cc", - "/home/developer/clice", - { - "/usr/bin/clang", - "-Qunused-arguments", - "-m64", - "-g", - "-O0", - "-std=c++23", - // parameter "-Iinclude" in CDB, should be convert to absolute path - "-I", - "/home/developer/clice/include", - "-I", - "/home/developer/clice/include", - "-fno-exceptions", - "-fno-cxx-exceptions", - "-isystem", - "/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include", - "-isystem", - "/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include", - "-fsanitize=address", - "-fno-rtti", - "/home/developer/clice/src/Driver/clice.cc", - }); - }; + // skip_unless(Linux || MacOS) / test("LoadRelativeUnixStyle") = [expect_load] { + // constexpr const char* xmake = R"([ + // { + // "directory": "/home/developer/clice", + // "arguments": ["/usr/bin/clang", "-c", "-Qunused-arguments", "-m64", "-g", "-O0", + // "-std=c++23", "-Iinclude", "-I/home/developer/clice/include", "-fno-exceptions", + // "-fno-cxx-exceptions", "-isystem", + // "/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include", + // "-isystem", + // "/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include", + // "-fsanitize=address", "-fno-rtti", "-o", + // "build/.objs/clice/linux/x86_64/debug/src/Driver/clice.cc.o", "src/Driver/clice.cc"], + // "file": "src/Driver/clice.cc" + // } + // ])"; + // + // expect_load( + // xmake, + // "/home/developer/clice", + // "/home/developer/clice/src/Driver/clice.cc", + // "/home/developer/clice", + // { + // "/usr/bin/clang", + // "-Qunused-arguments", + // "-m64", + // "-g", + // "-O0", + // "-std=c++23", + // // parameter "-Iinclude" in CDB, should be convert to absolute path + // "-I", + // "/home/developer/clice/include", + // "-I", + // "/home/developer/clice/include", + // "-fno-exceptions", + // "-fno-cxx-exceptions", + // "-isystem", + // "/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include", + // "-isystem", + // "/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include", + // "-fsanitize=address", + // "-fno-rtti", + // "/home/developer/clice/src/Driver/clice.cc", + // }); + //}; }; } // namespace diff --git a/tests/unit/Compiler/Toolchain.cpp b/tests/unit/Compiler/Toolchain.cpp new file mode 100644 index 00000000..67d93474 --- /dev/null +++ b/tests/unit/Compiler/Toolchain.cpp @@ -0,0 +1,126 @@ +#include "Compiler/Compilation.h" +#include "Test/Test.h" +#include "Compiler/Toolchain.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/StringSaver.h" +#include "clang/Driver/Driver.h" +#include "Support/Logging.h" + +namespace clice::testing { + +namespace { + +suite<"Toolchain"> suite = [] { + auto expect_family = [](llvm::StringRef name, + toolchain::CompilerFamily family, + std::source_location location = std::source_location::current()) { + expect(eq(refl::enum_name(toolchain::driver_family(name)), refl::enum_name((family))), + location); + }; + + test("Family") = [&] { + using enum toolchain::CompilerFamily; + + expect_family("gcc", GCC); + expect_family("g++", GCC); + expect_family("x86_64-linux-gnu-g++-14", GCC); + expect_family("arm-none-eabi-gcc", GCC); + + expect_family("clang", Clang); + expect_family("clang++", Clang); + expect_family("clang.exe", Clang); + expect_family("clang++.exe", Clang); + expect_family("clang-20", Clang); + expect_family("clang-20.exe", Clang); + expect_family("clang-cl", ClangCL); + expect_family("clang-cl-20", ClangCL); + expect_family("clang-cl-20.exe", ClangCL); + + expect_family("cl.exe", MSVC); + + expect_family("zig", Zig); + expect_family("zig.exe", Zig); + }; + + using namespace std::string_view_literals; + + skip_unless(CIEnvironment && (Windows || Linux)) / test("GCC") = [] { + auto file = fs::createTemporaryFile("clice", "cpp"); + if(!file) { + LOG_ERROR_RET(void(), "{}", file.error()); + } + + llvm::BumpPtrAllocator a; + llvm::StringSaver s(a); + auto arguments = toolchain::query_toolchain({ + .arguments = {"g++", + "-std=c++23", "-resource-dir", + fs::resource_dir.c_str(), + "-xc++", file->c_str()}, + .callback = [&](const char* str) { return s.save(str).data(); } + }); + + expect(arguments.size() > 2); + expect(eq(arguments[1], "-cc1"sv)); + + CompilationParams params; + params.arguments_from_database = true; + params.arguments = arguments; + params.add_remapped_file(file->c_str(), R"( + #include + int main() { + std::println("Hello world!"); + return 0; + } + )"); + + auto unit = compile(params); + expect(unit && unit->diagnostics().empty()); + }; + + skip_unless(CIEnvironment) / test("MSVC") = [] { + + }; + + skip_unless(CIEnvironment) / test("Clang") = [] { + auto file = fs::createTemporaryFile("clice", "cpp"); + if(!file) { + LOG_ERROR_RET(void(), "{}", file.error()); + } + + llvm::BumpPtrAllocator a; + llvm::StringSaver s(a); + auto arguments = toolchain::query_toolchain({ + .arguments = {"clang++", + "-std=c++23", "-resource-dir", + fs::resource_dir.c_str(), + "-xc++", file->c_str()}, + .callback = [&](const char* str) { return s.save(str).data(); } + }); + + expect(arguments.size() > 2); + expect(eq(arguments[1], "-cc1"sv)); + + CompilationParams params; + params.arguments_from_database = true; + params.arguments = arguments; + params.add_remapped_file(file->c_str(), R"( + #include + int main() { + std::println("Hello world!"); + return 0; + } + )"); + + auto unit = compile(params); + expect(unit && unit->diagnostics().empty()); + }; + + skip_unless(CIEnvironment) / test("Zig") = [] { + + }; +}; + +} // namespace + +} // namespace clice::testing diff --git a/tests/unit/Index/USR.cpp b/tests/unit/Index/USR.cpp index 348cb0c5..69ce9a2a 100644 --- a/tests/unit/Index/USR.cpp +++ b/tests/unit/Index/USR.cpp @@ -36,7 +36,7 @@ struct GetUSRVisitor : public clang::RecursiveASTVisitor { if(offset.has_value() && USR.has_value()) { USRs[*offset] = USRInfo{*USR, *offset, decl}; - // LOGGING_INFO("USR: {} at {}:{}", USR->str(), pos->line, pos->character); + // LOG_INFO("USR: {} at {}:{}", USR->str(), pos->line, pos->character); } return true; @@ -61,7 +61,7 @@ struct USRTester : public Tester { llvm::StringRef lookup(llvm::StringRef key) { auto iter = USRs.find((*this)["main.cpp", key]); if(iter == USRs.end()) { - LOGGING_FATAL("USR not found for key: {}", key); + LOG_FATAL("USR not found for key: {}", key); } return iter->second.USR; } diff --git a/tests/unit/Test/Tester.cpp b/tests/unit/Test/Tester.cpp new file mode 100644 index 00000000..b819cd6d --- /dev/null +++ b/tests/unit/Test/Tester.cpp @@ -0,0 +1,165 @@ +#include "Test/Tester.h" + +namespace clice::testing { + +void Tester::prepare(llvm::StringRef standard) { + auto command = std::format("clang++ {} {} -fms-extensions", standard, src_path); + + database.add_command("fake", src_path, command); + params.kind = CompilationUnit::Content; + + CommandOptions options; + options.resource_dir = true; + options.query_toolchain = true; + options.suppress_logging = true; + + params.arguments_from_database = true; + params.arguments = database.lookup(src_path, options).arguments; + + for(auto& [file, source]: sources.all_files) { + if(file == src_path) { + params.add_remapped_file(file, source.content); + } else { + /// FIXME: This is a workaround. + std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); + params.add_remapped_file(path, source.content); + } + } +} + +bool Tester::compile(llvm::StringRef standard) { + prepare(standard); + + auto unit = clice::compile(params); + if(!unit) { + LOG_ERROR("{}", unit.error()); + for(auto& diag: *params.diagnostics) { + LOG_ERROR("{}", diag.message); + } + return false; + } + + this->unit.emplace(std::move(*unit)); + return true; +} + +bool Tester::compile_with_pch(llvm::StringRef standard) { + params.diagnostics = std::make_shared>(); + auto command = std::format("clang++ {} {} -fms-extensions", standard, src_path); + + database.add_command("fake", src_path, command); + params.kind = CompilationUnit::Preamble; + + CommandOptions options; + options.resource_dir = true; + options.query_toolchain = true; + options.suppress_logging = true; + + params.arguments_from_database = true; + params.arguments = database.lookup(src_path, options).arguments; + + auto path = fs::createTemporaryFile("clice", "pch"); + if(!path) { + llvm::outs() << path.error().message() << "\n"; + } + + /// Build PCH + params.output_file = *path; + + for(auto& [file, source]: sources.all_files) { + if(file == src_path) { + auto bound = compute_preamble_bound(source.content); + params.add_remapped_file(file, source.content, bound); + } else { + /// FIXME: This is a workaround. + std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); + params.add_remapped_file(path, source.content); + } + } + + PCHInfo info; + { + auto unit = clice::compile(params, info); + if(!unit) { + LOG_ERROR("{}", unit.error()); + for(auto& diag: *params.diagnostics) { + LOG_ERROR("{}", diag.message); + } + return false; + } + } + + /// Build AST + params.output_file.clear(); + params.kind = CompilationUnit::Content; + params.pch = {info.path, info.preamble.size()}; + for(auto& [file, source]: sources.all_files) { + if(file == src_path) { + params.add_remapped_file(file, source.content); + } else { + /// FIXME: This is a workaround. + std::string path = path::is_absolute(file) ? file.str() : path::join(".", file); + params.add_remapped_file(path, source.content); + } + } + + auto unit = clice::compile(params); + if(!unit) { + LOG_ERROR("{}", unit.error()); + for(auto& diag: *params.diagnostics) { + LOG_ERROR("{}", diag.message); + } + return false; + } + + this->unit.emplace(std::move(*unit)); + return true; +} + +std::uint32_t Tester::point(llvm::StringRef name, llvm::StringRef file) { + if(file.empty()) { + file = src_path; + } + + auto& offsets = sources.all_files[file].offsets; + if(name.empty()) { + assert(offsets.size() == 1); + return offsets.begin()->second; + } else { + assert(offsets.contains(name)); + return offsets.lookup(name); + } +} + +llvm::ArrayRef Tester::nameless_points(llvm::StringRef file) { + if(file.empty()) { + file = src_path; + } + + return sources.all_files[file].nameless_offsets; +} + +LocalSourceRange Tester::range(llvm::StringRef name, llvm::StringRef file) { + if(file.empty()) { + file = src_path; + } + + auto& ranges = sources.all_files[file].ranges; + if(name.empty()) { + assert(ranges.size() == 1); + return ranges.begin()->second; + } else { + assert(ranges.contains(name)); + return ranges.lookup(name); + } +} + +void Tester::clear() { + params = CompilationParams(); + database = CompilationDatabase(); + unit.reset(); + sources.all_files.clear(); + src_path.clear(); +} + +} // namespace clice::testing diff --git a/xmake.lua b/xmake.lua index 8047750e..5dd12e3b 100644 --- a/xmake.lua +++ b/xmake.lua @@ -226,7 +226,11 @@ rule("clice_build_config") end if has_config("ci") then - target:add("cxxflags", "-DCLICE_CI_ENVIRONMENT") + target:add("cxxflags", "-DCLICE_CI_ENVIRONMENT=1") + end + + if has_config("enable_test") then + target:add("cxxflags", "-DCLICE_ENABLE_TEST=1") end end)