From 741f70a12ca3a08abef95235d704d7715337b330 Mon Sep 17 00:00:00 2001 From: ykiko Date: Fri, 18 Jul 2025 19:50:53 +0800 Subject: [PATCH] Improve command handle (#149) --- CMakeLists.txt | 3 +- include/Compiler/Command.h | 147 +++++--- include/Compiler/Compilation.h | 9 +- include/Test/CTest.h | 6 +- src/Compiler/Command.cpp | 530 +++++++++++++++++++-------- src/Compiler/Compilation.cpp | 4 + src/Feature/SemanticToken.cpp | 5 + src/Server/Indexer.cpp | 2 +- src/Server/Scheduler.cpp | 8 +- src/Server/Server.cpp | 5 +- unittests/Compiler/Command.cpp | 193 ++++++++-- unittests/Compiler/Diagnostic.cpp | 13 +- unittests/Compiler/Module.cpp | 15 +- unittests/Compiler/Preamble.cpp | 15 +- unittests/Feature/CodeCompletion.cpp | 4 +- unittests/Feature/SignatureHelp.cpp | 4 +- xmake.lua | 1 + 17 files changed, 698 insertions(+), 266 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50127c78..44b38ef7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,7 @@ if(NOT WIN32) target_link_libraries(clice-core PUBLIC LLVMSupport LLVMFrontendOpenMP + LLVMOption clangAST clangASTMatchers clangBasic @@ -135,7 +136,7 @@ if(CLICE_ENABLE_TEST) ) FetchContent_MakeAvailable(googletest) - target_link_libraries(unit_tests PRIVATE gtest_main clice-core) + target_link_libraries(unit_tests PRIVATE gtest_main clice-core) # integration_tests add_executable(integration_tests "${CMAKE_SOURCE_DIR}/src/Driver/integration_tests.cc") diff --git a/include/Compiler/Command.h b/include/Compiler/Command.h index 3f26731f..c2db740d 100644 --- a/include/Compiler/Command.h +++ b/include/Compiler/Command.h @@ -4,67 +4,128 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Allocator.h" +#include "llvm/ADT/ArrayRef.h" +#include namespace clice { -/// `CompilationDatabase` is responsible for managing the compile commands. -/// -/// FIXME: currently we assume that a file only occurs once in the CDB. -/// This is not always correct, but it is enough for now. class CompilationDatabase { public: using Self = CompilationDatabase; - /// Update the compile commands with the given file. - void update_commands(this Self& self, llvm::StringRef file); - - /// Update the module map with the given file and module name. - void update_module(llvm::StringRef file, llvm::StringRef name); - - /// Lookup the module interface unit file path of the given module name. - llvm::StringRef get_module_file(llvm::StringRef name); - - auto size() const { - return commands.size(); - } - - enum class Style { - GNU = 0, - MSVC, + enum class UpdateKind : std::uint8_t { + Unchange, + Create, + Update, + Delete, }; - void add_command(this Self& self, - llvm::StringRef path, - llvm::StringRef command, - Style style = Style::GNU); + struct CommandInfo { + /// TODO: add sysroot or no stdinc command info. + llvm::StringRef dictionary; - llvm::ArrayRef get_command(this Self& self, llvm::StringRef path); + /// The canonical command list. + llvm::ArrayRef arguments; + }; + + struct DriverInfo { + /// The target of this driver. + llvm::StringRef target; + + /// The default system includes of this driver. + llvm::ArrayRef system_includes; + }; + + struct UpdateInfo { + /// The kind of update. + UpdateKind kind; + + llvm::StringRef file; + }; + + struct LookupInfo { + llvm::StringRef dictionary; + + std::vector arguments; + }; + + CompilationDatabase(); + + auto save_string(this Self& self, llvm::StringRef string) -> llvm::StringRef; + + auto save_cstring_list(this Self& self, llvm::ArrayRef arguments) + -> llvm::ArrayRef; + + /// Get an the option for specific argument. + static std::optional get_option_id(llvm::StringRef argument); + + /// Query the compiler driver and return its driver info. + auto query_driver(this Self& self, llvm::StringRef driver) + -> std::expected; + + /// Update with arguments. + auto update_command(this Self& self, + llvm::StringRef dictionary, + llvm::StringRef file, + llvm::ArrayRef arguments) -> UpdateInfo; + + /// Update with full command. + auto update_command(this Self& self, + llvm::StringRef dictionary, + llvm::StringRef file, + llvm::StringRef command) -> UpdateInfo; + + /// Update commands from json file and return all updated file. + auto load_commands(this Self& self, llvm::StringRef json_content) + -> std::expected, std::string>; + + auto get_command(this Self& self, llvm::StringRef file, bool resource_dir = false) -> LookupInfo; private: - /// Save a string into memory pool. Make sure end with `\0`. - llvm::StringRef save_string(this Self& self, llvm::StringRef string); + /// The memory pool to hold all cstring and command list. + llvm::BumpPtrAllocator allocator; - std::vector save_args(this Self& self, llvm::ArrayRef args); + /// A cache between input string and its cache cstring + /// in the allocator, make sure end with `\0`. + llvm::DenseSet string_cache; -private: - /// For C++20 module, we only can got dependent module name - /// in source context. But we need dependent module file path - /// to build PCM. So we will scan(preprocess) all project files - /// to build a module map between module name and module file path. - /// **Note that** this only includes module interface unit, for module - /// implementation unit, the scan could be delayed until compiling it. - llvm::StringMap moduleMap; + /// A cache between input command and its cache array + /// in the allocator. + llvm::DenseSet> arguments_cache; - /// Memory pool for command arguments. - llvm::BumpPtrAllocator memory_pool; + /// The clang options we want to filter in all cases, like -c and -o. + llvm::DenseSet filtered_options; - /// For lookup whether we already have the key. - llvm::DenseSet unique; + /// A map between file path and its canonical command list. + llvm::DenseMap command_infos; - // A map between file path and compile commands. - /// TODO: Path cannot represent unique file, we should use better, like inode ... - llvm::DenseMap>> commands; + /// A map between driver path and its query driver info. + llvm::DenseMap driver_infos; }; } // namespace clice +namespace llvm { + +template <> +struct DenseMapInfo> { + using T = llvm::ArrayRef; + + inline static T getEmptyKey() { + return T(reinterpret_cast(~0), T::size_type(0)); + } + + inline static T getTombstoneKey() { + return T(reinterpret_cast(~1), T::size_type(0)); + } + + static unsigned getHashValue(const T& value) { + return llvm::hash_combine_range(value.begin(), value.end()); + } + + static bool isEqual(const T& lhs, const T& rhs) { + return lhs == rhs; + } +}; + +} // namespace llvm diff --git a/include/Compiler/Compilation.h b/include/Compiler/Compilation.h index b0f69b45..708478a4 100644 --- a/include/Compiler/Compilation.h +++ b/include/Compiler/Compilation.h @@ -15,8 +15,10 @@ struct CompilationParams { /// Output file path. llvm::SmallString<128> outPath; + std::string directory; + /// Responsible for storing the arguments. - llvm::ArrayRef arguments; + std::vector arguments; llvm::IntrusiveRefCntPtr vfs = new ThreadSafeFS(); @@ -32,12 +34,11 @@ struct CompilationParams { /// The memory buffers for all remapped file. llvm::StringMap> buffers; - void add_remapped_file(llvm::StringRef path, llvm::StringRef content, std::uint32_t bound = -1) { - if(bound != -1) { - assert(bound < content.size()); + if(bound != 0 && bound != -1) { + assert(bound <= content.size()); content = content.substr(0, bound); } buffers.try_emplace(path, llvm::MemoryBuffer::getMemBufferCopy(content)); diff --git a/include/Test/CTest.h b/include/Test/CTest.h index d87414e2..b3311fa0 100644 --- a/include/Test/CTest.h +++ b/include/Test/CTest.h @@ -80,9 +80,9 @@ public: Tester& compile(llvm::StringRef standard = "-std=c++20") { auto command = std::format("clang++ {} {} -fms-extensions", standard, src_path); - database.add_command(src_path, command); - params.arguments = database.get_command(src_path); - + database.update_command("fake", src_path, command); + params.arguments = database.get_command(src_path).arguments; + auto info = clice::compile(params); ASSERT_TRUE(info); this->unit.emplace(std::move(*info)); diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index d94729e7..51792b8e 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -1,195 +1,421 @@ -#include "Support/Logger.h" #include "Compiler/Command.h" #include "Compiler/Compilation.h" #include "Support/FileSystem.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Program.h" +#include "clang/Driver/Driver.h" namespace clice { -/// Update the compile commands with the given file. -void CompilationDatabase::update_commands(this Self& self, llvm::StringRef filename) { - auto path = path::real_path(filename); - filename = path; +CompilationDatabase::CompilationDatabase() { + using opions = clang::driver::options::ID; - /// Read the compile commands from the file. - json::Value json = nullptr; + /// Remove the input file, we will add input file ourselves. + filtered_options.insert(opions::OPT_INPUT); - if(auto buffer = llvm::MemoryBuffer::getFile(filename)) { - if(auto result = json::parse(buffer->get()->getBuffer())) { - /// llvm::json::Value will hold on string buffer. - /// Do not worry about the lifetime of the buffer. - /// Release buffer to save memory. - json = std::move(result.get()); - } else { - log::warn("Failed to parse json file at {0}, because {1}", - filename, - result.takeError()); - return; - } - } else { - log::warn("Failed to read file {0}", filename); - return; - } + /// -c and -o are meaningless for frontend. + filtered_options.insert(opions::OPT_c); + filtered_options.insert(opions::OPT_o); + filtered_options.insert(opions::OPT_dxc_Fc); + filtered_options.insert(opions::OPT_dxc_Fo); - assert(json.kind() != json::Value::Null && "json is nullptr"); + /// Remove all options related to PCH building. + filtered_options.insert(opions::OPT_emit_pch); + filtered_options.insert(opions::OPT_include_pch); + filtered_options.insert(opions::OPT__SLASH_Yu); + filtered_options.insert(opions::OPT__SLASH_Fp); - if(json.kind() != json::Value::Array) { - log::warn( - "Compilation CompilationDatabase requires a array of object, but get {0}, input file: {1}", - refl::enum_name(json.kind()), - filename); - return; - } - - auto elements = json.getAsArray(); - assert(elements && "json is not an array"); - - for(auto& element: *elements) { - auto object = element.getAsObject(); - if(!object) { - log::warn( - "Compilation CompilationDatabase requires an array of object, but get a array of {0}, input file: {1}", - refl::enum_name(element.kind()), - filename); - continue; - } - - /// FIXME: currently we assume all path here is absolute. - /// Add `directory` field in the future. - - llvm::SmallString<128> path; - - if(auto file = object->getString("file")) { - if(auto error = fs::real_path(*file, path)) { - log::warn("Failed to get real path of {0}, because {1}", *file, error.message()); - continue; - } - } else { - log::warn("The element does not have a file field, input file: {0}", filename); - continue; - } - - auto command = object->getString("command"); - if(!command) { - log::warn("The key:{0} does not have a command field, input file: {1}", path, filename); - continue; - } - - self.add_command(path, *command); - } - - log::info("Successfully loaded compile commands from {0}, total {1} commands", - filename, - self.commands.size()); - - /// Scan all files to build module map. - // CompilationParams params; - // for(auto& [path, command]: commands) { - // params.srcPath = path; - // params.command = command; - // - // auto name = scanModuleName(params); - // if(!name.empty()) { - // moduleMap[name] = path; - // } - //} - - log::info("Successfully built module map, total {0} modules", self.moduleMap.size()); + /// Remove all options related to C++ module, we will + /// build module and set deps ourselves. + filtered_options.insert(opions::OPT_fmodule_file); + filtered_options.insert(opions::OPT_fmodule_output); + filtered_options.insert(opions::OPT_fprebuilt_module_path); } -/// Update the module map with the given file and module name. -void CompilationDatabase::update_module(llvm::StringRef file, llvm::StringRef name) { - moduleMap[path::real_path(file)] = file; -} - -/// Lookup the module interface unit file path of the given module name. -llvm::StringRef CompilationDatabase::get_module_file(llvm::StringRef name) { - auto iter = moduleMap.find(name); - if(iter == moduleMap.end()) { - return ""; - } - return iter->second; -} - -llvm::StringRef CompilationDatabase::save_string(this Self& self, llvm::StringRef string) { - auto it = self.unique.find(string); - - /// FIXME: arg may be empty? +auto CompilationDatabase::save_string(this Self& self, llvm::StringRef string) -> llvm::StringRef { + assert(!string.empty() && "expected non empty string"); + auto it = self.string_cache.find(string); /// If we already store the argument, reuse it. - if(it != self.unique.end()) { + if(it != self.string_cache.end()) { return *it; } - /// Allocate new argument. + /// Allocate for new string. const auto size = string.size(); - auto ptr = self.memory_pool.Allocate(size + 1); + auto ptr = self.allocator.Allocate(size + 1); std::memcpy(ptr, string.data(), size); ptr[size] = '\0'; - /// Insert new argument. + /// Insert it to cache. auto result = llvm::StringRef(ptr, size); - self.unique.insert(result); + self.string_cache.insert(result); return result; } -std::vector CompilationDatabase::save_args(this Self& self, - llvm::ArrayRef args) { - std::vector result; - result.reserve(args.size()); +auto CompilationDatabase::save_cstring_list(this Self& self, llvm::ArrayRef arguments) + -> llvm::ArrayRef { + auto it = self.arguments_cache.find(arguments); - for(auto i = 0; i < args.size(); i++) { - result.emplace_back(self.save_string(args[i]).data()); + /// If we already store the argument, reuse it. + if(it != self.arguments_cache.end()) { + return *it; } + /// Allocate for new array. + const auto size = arguments.size(); + auto ptr = self.allocator.Allocate(size); + ranges::copy(arguments, ptr); + + /// Insert it to cache. + auto result = llvm::ArrayRef(ptr, size); + self.arguments_cache.insert(result); return result; } -void CompilationDatabase::add_command(this Self& self, - llvm::StringRef path, - llvm::StringRef command, - Style style) { - llvm::SmallVector args; +std::optional CompilationDatabase::get_option_id(llvm::StringRef argument) { + auto& table = clang::driver::getDriverOptTable(); - /// temporary allocator to meet the argument requirements of tokenize. - llvm::BumpPtrAllocator allocator; - llvm::StringSaver saver(allocator); + llvm::SmallString<64> buffer = argument; - /// FIXME: we may want to check the first argument of command to - /// make sure its mode. - if(style == Style::GNU) { - llvm::cl::TokenizeGNUCommandLine(command, saver, args); - } else if(style == Style::MSVC) { - llvm::cl::TokenizeWindowsCommandLineFull(command, saver, args); - } else { - std::abort(); + if(argument.ends_with("=")) { + buffer += "placeholder"; } - auto path_ = self.save_string(path); - auto new_args = self.save_args(args); + unsigned index = 0; + std::array arguments = {buffer.c_str(), "placeholder"}; + llvm::opt::InputArgList arg_list(arguments.data(), arguments.data() + arguments.size()); - /// FIXME: Use a better way to handle resource dir. - /// new_args.push_back(self.save_string(std::format("-resource-dir={}", - /// fs::resource_dir)).data()); - - auto it = self.commands.find(path_.data()); - if(it == self.commands.end()) { - self.commands.try_emplace(path_.data(), - std::make_unique>(std::move(new_args))); - } else { - *it->second = std::move(new_args); - } -} - -llvm::ArrayRef CompilationDatabase::get_command(this Self& self, - llvm::StringRef path) { - auto path_ = self.save_string(path); - auto it = self.commands.find(path_.data()); - if(it != self.commands.end()) { - return *it->second; + if(auto arg = table.ParseOneArg(arg_list, index)) { + return arg->getOption().getID(); } else { return {}; } } +auto CompilationDatabase::query_driver(this Self& self, llvm::StringRef driver) + -> std::expected { + driver = self.save_string(driver); + + auto it = self.driver_infos.find(driver.data()); + if(it != self.driver_infos.end()) { + return it->second; + } + + auto driver_name = path::filename(driver); + + llvm::SmallString<128> output_path; + if(auto error = llvm::sys::fs::createTemporaryFile("system-includes", "clangd", output_path)) { + return std::unexpected(std::format("{}", error)); + } + + auto clean_up = llvm::make_scope_exit([&output_path]() { fs::remove(output_path); }); + + bool is_std_err = true; + + std::optional redirects[] = {{""}, {""}, {""}}; + redirects[is_std_err ? 2 : 1] = output_path.str(); + + llvm::StringRef argv[] = {driver, "-E", "-v", "-xc++", "/dev/null"}; + + std::string message; + if(int RC = llvm::sys::ExecuteAndWait(driver, + argv, + /*Env=*/std::nullopt, + redirects, + /*SecondsToWait=*/0, + /*MemoryLimit=*/0, + &message)) { + return std::unexpected(std::format("{}", message)); + } + + auto file = llvm::MemoryBuffer::getFile(output_path); + if(!file) { + return std::unexpected(std::format("{}", file.getError())); + } + + llvm::StringRef content = file.get()->getBuffer(); + + const char* TS = "Target: "; + const char* SIS = "#include <...> search starts here:"; + const char* SIE = "End of search list."; + + llvm::SmallVector lines; + content.split(lines, '\n', -1, false); + + bool in_includes_block = false; + bool found_start_marker = false; + + llvm::StringRef target; + llvm::SmallVector system_includes; + + for(const auto& line_ref: lines) { + auto line = line_ref.trim(); + + if(line.starts_with(TS)) { + line.consume_front(TS); + target = line; + continue; + } + + if(line == SIS) { + found_start_marker = true; + in_includes_block = true; + continue; + } + + if(line == SIE) { + if(in_includes_block) { + in_includes_block = false; + } + continue; + } + + if(in_includes_block) { + if(line.contains("lib/gcc")) { + continue; + } + + system_includes.push_back(line); + } + } + + if(!found_start_marker) { + return std::unexpected("Start marker not found..."); + } + + if(in_includes_block) { + return std::unexpected("End marker not found..."); + } + + llvm::SmallVector includes; + for(auto include: system_includes) { + includes.emplace_back(self.save_string(include).data()); + } + + DriverInfo info; + info.target = self.save_string(target); + info.system_includes = self.save_cstring_list(includes); + self.driver_infos.try_emplace(driver.data(), info); + return info; +} + +auto CompilationDatabase::update_command(this Self& self, + llvm::StringRef dictionary, + llvm::StringRef file, + llvm::ArrayRef arguments) -> UpdateInfo { + file = self.save_string(file); + dictionary = self.save_string(dictionary); + + llvm::SmallVector filtered_arguments; + + /// Append + auto add_argument = [&](llvm::StringRef argument) { + auto saved = self.save_string(argument); + filtered_arguments.emplace_back(saved.data()); + }; + + /// Append driver sperately. + add_argument(arguments.front()); + + unsigned missing_arg_index = 0; + unsigned missing_arg_count = 0; + auto& table = clang::driver::getDriverOptTable(); + + /// The driver should be discarded. + auto list = table.ParseArgs(arguments.drop_front(), missing_arg_index, missing_arg_count); + + bool remove_pch = false; + + /// Append and filter useless arguments. + for(auto arg: list.getArgs()) { + auto& opt = arg->getOption(); + auto id = opt.getID(); + + /// Filter options we don't need. + if(self.filtered_options.contains(id)) { + continue; + } + + /// A workaround to remove extra PCH when cmake + /// generate PCH flags for clang. + if(id == clang::driver::options::OPT_Xclang) { + if(arg->getNumValues() == 1) { + if(remove_pch) { + remove_pch = false; + continue; + } + + llvm::StringRef value = arg->getValue(0); + if(value == "-include-pch") { + remove_pch = true; + continue; + } + } + } + + /// Rewrite the argument to filter arguments, we basically reimplement + /// the logic of `Arg::render` to use our allocator to allocate memory. + switch(opt.getRenderStyle()) { + case llvm::opt::Option::RenderValuesStyle: { + for(auto value: arg->getValues()) { + add_argument(value); + } + break; + } + + case llvm::opt::Option::RenderSeparateStyle: { + add_argument(arg->getSpelling()); + for(auto value: arg->getValues()) { + add_argument(value); + } + break; + } + + case llvm::opt::Option::RenderJoinedStyle: { + llvm::SmallString<256> first = {arg->getSpelling(), arg->getValue(0)}; + add_argument(first); + for(auto value: llvm::ArrayRef(arg->getValues()).drop_front()) { + add_argument(value); + } + break; + } + + case llvm::opt::Option::RenderCommaJoinedStyle: { + llvm::SmallString<256> buffer = arg->getSpelling(); + for(auto i = 0; i < arg->getNumValues(); i++) { + if(i) { + buffer += ','; + } + buffer += arg->getValue(i); + } + add_argument(buffer); + break; + } + } + } + + /// Save arguments. + arguments = self.save_cstring_list(filtered_arguments); + + UpdateKind kind = UpdateKind::Unchange; + CommandInfo info = {dictionary, arguments}; + auto [it, success] = self.command_infos.try_emplace(file.data(), info); + if(success) { + kind = UpdateKind::Create; + } else { + auto& info = it->second; + if(info.dictionary.data() != dictionary.data() || + info.arguments.data() != arguments.data()) { + kind = UpdateKind::Update; + info.dictionary = dictionary; + info.arguments = arguments; + } + } + + return UpdateInfo{kind, file}; +} + +auto CompilationDatabase::update_command(this Self& self, + llvm::StringRef dictionary, + llvm::StringRef file, + llvm::StringRef command) -> UpdateInfo { + llvm::BumpPtrAllocator local; + llvm::StringSaver saver(local); + + llvm::SmallVector arguments; + auto [driver, _] = command.split(' '); + driver = path::filename(driver); + + /// FIXME: Use a better to handle this. + if(driver.starts_with("cl") || driver.starts_with("clang-cl")) { + llvm::cl::TokenizeWindowsCommandLineFull(command, saver, arguments); + } else { + llvm::cl::TokenizeGNUCommandLine(command, saver, arguments); + } + + return self.update_command(dictionary, file, arguments); +} + +auto CompilationDatabase::load_commands(this Self& self, llvm::StringRef json_content) + -> std::expected, std::string> { + std::vector infos; + + auto json = json::parse(json_content); + if(!json) { + return std::unexpected(std::format("Fail to parse json: {}", json.takeError())); + } + + if(json->kind() != json::Value::Array) { + return std::unexpected("Compilation Database must be an array of object"); + } + + /// FIXME: warn illegal item. + for(auto& item: *json->getAsArray()) { + /// Ignore non-object item. + if(item.kind() != json::Value::Object) { + continue; + } + + auto& object = *item.getAsObject(); + + auto file = object.getString("file"); + auto directory = object.getString("directory"); + if(!file || !directory) { + continue; + } + + if(auto arguments = object.getArray("arguments")) { + /// Construct cstring array. + llvm::BumpPtrAllocator local; + llvm::StringSaver saver(local); + llvm::SmallVector carguments; + + for(auto& argument: *arguments) { + if(argument.kind() == json::Value::String) { + carguments.emplace_back(saver.save(*argument.getAsString()).data()); + } + } + + auto info = self.update_command(*directory, *file, carguments); + if(info.kind != UpdateKind::Unchange) { + infos.emplace_back(info); + } + } else if(auto command = object.getString("command")) { + auto info = self.update_command(*directory, *file, *command); + if(info.kind != UpdateKind::Unchange) { + infos.emplace_back(info); + } + } + } + + return infos; +} + +auto CompilationDatabase::get_command(this Self& self, llvm::StringRef file, bool resource_dir) + -> LookupInfo { + LookupInfo info; + + file = self.save_string(file); + auto it = self.command_infos.find(file.data()); + if(it != self.command_infos.end()) { + info.dictionary = it->second.dictionary; + info.arguments = it->second.arguments; + } else { + /// FIXME: Use a better way to handle fallback command. + info.dictionary = {}; + info.arguments = {"clang++", "-std=c++20"}; + } + + if(resource_dir) { + info.arguments.emplace_back( + self.save_string(std::format("-resource-dir={}", fs::resource_dir)).data()); + } + + info.arguments.emplace_back(file.data()); + + return info; +} + } // namespace clice diff --git a/src/Compiler/Compilation.cpp b/src/Compiler/Compilation.cpp index fdc5baca..c4dd04de 100644 --- a/src/Compiler/Compilation.cpp +++ b/src/Compiler/Compilation.cpp @@ -46,6 +46,10 @@ auto create_invocation(CompilationParams& params, clang::CreateInvocationOptions options = { .Diags = diagnostic_engine, .VFS = params.vfs, + + /// Avoid replacing -include with -include-pch, also + /// see https://github.com/clangd/clangd/issues/856. + .ProbePrecompiled = false, }; auto invocation = clang::createInvocation(params.arguments, options); diff --git a/src/Feature/SemanticToken.cpp b/src/Feature/SemanticToken.cpp index 4147816f..b42e10cd 100644 --- a/src/Feature/SemanticToken.cpp +++ b/src/Feature/SemanticToken.cpp @@ -108,6 +108,11 @@ public: } auto [fid, range] = unit.decompose_range(location); + if(fid != unit.interested_file()){ + /// FIXME: Use a better way to handle this. + return; + } + auto& tokens = interestedOnly ? result : sharedResult[fid]; tokens.emplace_back(range, kind, modifiers); } diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index 8303d242..699df77f 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -47,7 +47,7 @@ async::Task<> Indexer::index(CompilationUnit& unit) { async::Task<> Indexer::index(llvm::StringRef file) { CompilationParams params; - params.arguments = database.get_command(file); + params.arguments = database.get_command(file).arguments; auto AST = co_await async::submit([&] { return compile(params); }); diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index 367ad6e1..1efbe47d 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -57,7 +57,7 @@ async::Task Scheduler::completion(std::string path, std::uint32_t o /// Set compilation params ... . CompilationParams params; - params.arguments = database.get_command(path); + params.arguments = database.get_command(path, true).arguments; params.add_remapped_file(path, openFile->content); params.pch = {PCH->path, PCH->preamble.size()}; params.completion = {path, offset}; @@ -77,7 +77,7 @@ async::Task Scheduler::isPCHOutdated(llvm::StringRef path, llvm::StringRef } /// Check command and preamble matchs. - auto command = database.get_command(path); + auto command = database.get_command(path, true).arguments; /// FIXME: check command. openFile->PCH->command != command if(openFile->PCH->preamble != preamble) { co_return true; @@ -108,7 +108,7 @@ async::Task<> Scheduler::buildPCH(std::string path, std::string content) { std::uint32_t bound, std::string content) -> async::Task<> { CompilationParams params; - params.arguments = scheduler.database.get_command(path); + params.arguments = scheduler.database.get_command(path, true).arguments; params.outPath = path::join(config::cache.dir, path::filename(path) + ".pch"); params.add_remapped_file(path, content, bound); @@ -177,7 +177,7 @@ async::Task<> Scheduler::buildAST(std::string path, std::string content) { } CompilationParams params; - params.arguments = database.get_command(path); + params.arguments = database.get_command(path, true).arguments; params.add_remapped_file(path, content); params.pch = {PCH->path, PCH->preamble.size()}; diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 168001d0..4b3bac90 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -118,7 +118,10 @@ async::Task Server::onInitialize(json::Value value) { config::init(converter.workspace()); for(auto& dir: config::server.compile_commands_dirs) { - database.update_commands(dir + "/compile_commands.json"); + auto content = fs::read(dir + "/compile_commands.json"); + if(content) { + auto updated = database.load_commands(*content); + } } co_return result; diff --git a/unittests/Compiler/Command.cpp b/unittests/Compiler/Command.cpp index e72cb6c9..fcdcd810 100644 --- a/unittests/Compiler/Command.cpp +++ b/unittests/Compiler/Command.cpp @@ -1,33 +1,150 @@ #include "Test/Test.h" #include "Compiler/Command.h" +#include "clang/Driver/Driver.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Program.h" +#include "llvm/ADT/ScopeExit.h" namespace clice::testing { namespace { -TEST(Command, SimpleAddGet) { - CompilationDatabase database; - database.add_command("test.cpp", "clang++ -std=c++23 test.cpp"); - auto command = database.get_command("test.cpp"); - ASSERT_EQ(command.size(), 3); +std::string printArgv(llvm::ArrayRef args) { + std::string buf; + llvm::raw_string_ostream os(buf); + bool Sep = false; + for(llvm::StringRef arg: args) { + if(Sep) + os << ' '; + Sep = true; + if(llvm::all_of(arg, llvm::isPrint) && + arg.find_first_of(" \t\n\"\\") == llvm::StringRef::npos) { + os << arg; + continue; + } + os << '"'; + os.write_escaped(arg, /*UseHexEscapes=*/true); + os << '"'; + } + return std::move(os.str()); +} - using namespace std::literals; - EXPECT_EQ(command[0], "clang++"sv); - EXPECT_EQ(command[1], "-std=c++23"sv); - EXPECT_EQ(command[2], "test.cpp"sv); +void parse_and_dump(llvm::StringRef command) { + llvm::BumpPtrAllocator local; + llvm::StringSaver saver(local); + + llvm::SmallVector arguments; + auto [driver, _] = command.split(' '); + driver = path::filename(driver); + + /// FIXME: Use a better to handle this. + if(driver.starts_with("cl") || driver.starts_with("clang-cl")) { + llvm::cl::TokenizeWindowsCommandLineFull(command, saver, arguments); + } else { + llvm::cl::TokenizeGNUCommandLine(command, saver, arguments); + } + + auto& table = clang::driver::getDriverOptTable(); + std::uint32_t count = 0; + std::uint32_t index = 0; + auto list = table.ParseArgs(arguments, count, index); + + for(auto arg: list.getArgs()) { + arg->dump(); + } +} + +TEST(Command, GetOptionID) { + auto GET_OPTION_ID = CompilationDatabase::get_option_id; + namespace option = clang::driver::options; + + /// GroupClass + EXPECT_EQ(GET_OPTION_ID("-g"), option::OPT_g_Flag); + + /// InputClass + EXPECT_EQ(GET_OPTION_ID("main.cpp"), option::OPT_INPUT); + + /// UnknownClass + EXPECT_EQ(GET_OPTION_ID("--clice"), option::OPT_UNKNOWN); + + /// FlagClass + EXPECT_EQ(GET_OPTION_ID("-v"), option::OPT_v); + EXPECT_EQ(GET_OPTION_ID("-c"), option::OPT_c); + EXPECT_EQ(GET_OPTION_ID("-pedantic"), option::OPT_pedantic); + EXPECT_EQ(GET_OPTION_ID("--pedantic"), option::OPT_pedantic); + + /// JoinedClass + EXPECT_EQ(GET_OPTION_ID("-Wno-unused-variable"), option::OPT_W_Joined); + EXPECT_EQ(GET_OPTION_ID("-W*"), option::OPT_W_Joined); + EXPECT_EQ(GET_OPTION_ID("-W"), option::OPT_W_Joined); + + /// ValuesClass + + /// SeparateClass + EXPECT_EQ(GET_OPTION_ID("-Xclang"), option::OPT_Xclang); + /// EXPECT_EQ(GET_ID("-Xclang -ast-dump"), option::OPT_Xclang); + + /// RemainingArgsClass + + /// RemainingArgsJoinedClass + + /// CommaJoinedClass + EXPECT_EQ(GET_OPTION_ID("-Wl,"), option::OPT_Wl_COMMA); + + /// MultiArgClass + + /// JoinedOrSeparateClass + EXPECT_EQ(GET_OPTION_ID("-o"), option::OPT_o); + EXPECT_EQ(GET_OPTION_ID("-I"), option::OPT_I); + EXPECT_EQ(GET_OPTION_ID("--include-directory="), option::OPT_I); + EXPECT_EQ(GET_OPTION_ID("-x"), option::OPT_x); + EXPECT_EQ(GET_OPTION_ID("--language="), option::OPT_x); + + /// JoinedAndSeparateClass +} + +void EXPECT_STRIP(llvm::StringRef argv, + llvm::StringRef result, + LocationChain chain = LocationChain()) { + CompilationDatabase database; + database.update_command("fake/", "main.cpp", argv); + EXPECT_EQ(printArgv(database.get_command("main.cpp").arguments), result, chain); +}; + +TEST(Command, DefaultFilters) { + /// Filter -c, -o and input file. + EXPECT_STRIP("g++ main.cc", "g++ main.cpp"); + EXPECT_STRIP("clang++ -c main.cpp", "clang++ main.cpp"); + EXPECT_STRIP("clang++ -o main.o main.cpp", "clang++ main.cpp"); + EXPECT_STRIP("clang++ -c -o main.o main.cc", "clang++ main.cpp"); + EXPECT_STRIP("cl.exe /c /Fomain.cpp.o main.cpp", "cl.exe main.cpp"); + + /// Filter PCH related. + + /// CMake + EXPECT_STRIP("g++ -std=gnu++20 -Winvalid-pch -include cmake_pch.hxx -o main.cpp.o -c main.cpp", + "g++ -std=gnu++20 -Winvalid-pch -include cmake_pch.hxx main.cpp"); + EXPECT_STRIP( + "clang++ -Winvalid-pch -Xclang -include-pch -Xclang cmake_pch.hxx.pch -Xclang -include -Xclang cmake_pch.hxx -o main.cpp.o -c main.cpp", + "clang++ -Winvalid-pch -Xclang -include -Xclang cmake_pch.hxx main.cpp"); + EXPECT_STRIP("cl.exe /Yufoo.h /FIfoo.h /Fpfoo.h_v143.pch /c /Fomain.cpp.o main.cpp", + "cl.exe -include foo.h main.cpp"); + + /// TODO: Test more commands from other build system. } TEST(Command, Reuse) { - CompilationDatabase database; - database.add_command("test.cpp", "clang++ -std=c++23 test.cpp"); - database.add_command("test2.cpp", "clang++ -std=c++23 test2.cpp"); + using namespace std::literals; - auto command1 = database.get_command("test.cpp"); - auto command2 = database.get_command("test2.cpp"); + CompilationDatabase database; + database.update_command("fake", "test.cpp", "clang++ -std=c++23 test.cpp"sv); + database.update_command("fake", "test2.cpp", "clang++ -std=c++23 test2.cpp"sv); + + auto command1 = database.get_command("test.cpp").arguments; + auto command2 = database.get_command("test2.cpp").arguments; ASSERT_EQ(command1.size(), 3); ASSERT_EQ(command2.size(), 3); - using namespace std::literals; EXPECT_EQ(command1[0], "clang++"sv); EXPECT_EQ(command1[1], "-std=c++23"sv); EXPECT_EQ(command1[2], "test.cpp"sv); @@ -37,14 +154,48 @@ TEST(Command, Reuse) { EXPECT_EQ(command2[2], "test2.cpp"sv); } -TEST(Command, Merge) {} - -TEST(Command, Filter) {} - -TEST(Command, QueryDriver) {} - TEST(Command, Module) {} +TEST(Command, QueryDriver) { +#if __linux__ + using namespace std::literals; + + CompilationDatabase database; + auto info = database.query_driver("/usr/bin/g++"); + ASSERT_TRUE(info); + + EXPECT_EQ(info->target, "x86_64-linux-gnu"); + EXPECT_EQ(info->system_includes.size(), 6); + + if(_GLIBCXX_RELEASE == 13) { + EXPECT_EQ(info->system_includes[0], "/usr/include/c++/13"sv); + EXPECT_EQ(info->system_includes[1], "/usr/include/x86_64-linux-gnu/c++/13"sv); + EXPECT_EQ(info->system_includes[2], "/usr/include/c++/13/backward"sv); + } else if(_GLIBCXX_RELEASE == 14) { + EXPECT_EQ(info->system_includes[0], "/usr/include/c++/14"sv); + EXPECT_EQ(info->system_includes[1], "/usr/include/x86_64-linux-gnu/c++/14"sv); + EXPECT_EQ(info->system_includes[2], "/usr/include/c++/14/backward"sv); + } + + EXPECT_EQ(info->system_includes[3], "/usr/local/include"sv); + EXPECT_EQ(info->system_includes[4], "/usr/include/x86_64-linux-gnu"sv); + EXPECT_EQ(info->system_includes[5], "/usr/include"sv); +#endif +} + +TEST(Command, ResourceDir) { + /// CompilationDatabase database; + /// database.update_command("test.cpp", "clang++ -std=c++23 test.cpp"); + /// auto command = database.get_command("test.cpp", false, true); + /// ASSERT_EQ(command.size(), 4); + /// + /// using namespace std::literals; + /// EXPECT_EQ(command[0], "clang++"sv); + /// EXPECT_EQ(command[1], "-std=c++23"sv); + /// EXPECT_EQ(command[2], "test.cpp"sv); + /// EXPECT_EQ(command[3], std::format("-resource-dir={}", fs::resource_dir)); +} + } // namespace } // namespace clice::testing diff --git a/unittests/Compiler/Diagnostic.cpp b/unittests/Compiler/Diagnostic.cpp index 788de2b3..9a506ea2 100644 --- a/unittests/Compiler/Diagnostic.cpp +++ b/unittests/Compiler/Diagnostic.cpp @@ -9,26 +9,17 @@ namespace { using namespace clice; TEST(Diagnostic, CommandError) { - std::vector arguments = { - "clang++", - }; - CompilationParams params; /// miss input file. - params.arguments = arguments; + params.arguments = {"clang++"}; params.add_remapped_file("main.cpp", "int main() { return 0; }"); auto unit = compile(params); ASSERT_FALSE(unit); } TEST(Diagnostic, Error) { - std::vector arguments = { - "clang++", - "main.cpp", - }; - CompilationParams params; - params.arguments = arguments; + params.arguments = {"clang++", "main.cpp"}; params.add_remapped_file("main.cpp", "int main() { return 0 }"); auto unit = compile(params); ASSERT_TRUE(unit); diff --git a/unittests/Compiler/Module.cpp b/unittests/Compiler/Module.cpp index 74f8539c..28aa69f6 100644 --- a/unittests/Compiler/Module.cpp +++ b/unittests/Compiler/Module.cpp @@ -11,16 +11,15 @@ PCMInfo buildPCM(llvm::StringRef file, llvm::StringRef code) { fs::createUniquePath(llvm::Twine(file) + "%%%%%%.pcm", outPath, true); std::string path = file.str(); - std::vector arguments = { + + CompilationParams params; + params.outPath = outPath; + params.arguments = { "clang++", "-std=c++20", "-xc++", path.c_str(), }; - - CompilationParams params; - params.outPath = outPath; - params.arguments = arguments; params.add_remapped_file(file, code); params.add_remapped_file("./test.h", "export int foo2();"); @@ -34,15 +33,13 @@ PCMInfo buildPCM(llvm::StringRef file, llvm::StringRef code) { } ModuleInfo scan(llvm::StringRef content) { - std::vector arguments = { + CompilationParams params; + params.arguments = { "clang++", "-std=c++20", "-xc++", "main.ixx", }; - - CompilationParams params; - params.arguments = arguments; params.add_remapped_file("main.ixx", content); params.add_remapped_file("./test.h", "export module A"); auto info = scanModule(params); diff --git a/unittests/Compiler/Preamble.cpp b/unittests/Compiler/Preamble.cpp index e7c1b26c..81bf3e46 100644 --- a/unittests/Compiler/Preamble.cpp +++ b/unittests/Compiler/Preamble.cpp @@ -79,19 +79,18 @@ void EXPECT_BUILD_PCH(llvm::StringRef main_file, auto bound = computePreambleBound(content); params.add_remapped_file(main_file, content, bound); - std::vector arguments = { + params.arguments = { "clang++", "-xc++", "-std=c++20", }; if(!preamble.empty()) { - arguments.emplace_back("--include=preamble.h"); + params.arguments.emplace_back("--include=preamble.h"); } std::string buffer = main_file.str(); - arguments.emplace_back(buffer.c_str()); - params.arguments = arguments; + params.arguments.emplace_back(buffer.c_str()); for(auto& [path, content]: files) { params.add_remapped_file(path::join(".", path), content); @@ -210,9 +209,7 @@ int foo(); CompilationParams params; params.add_remapped_file("main.cpp", content); - - std::vector arguments = {"clang++", "-std=c++20", "main.cpp"}; - params.arguments = arguments; + params.arguments = {"clang++", "-std=c++20", "main.cpp"}; for(auto& [path, file]: files) { params.add_remapped_file(path::join(".", path), file); @@ -263,9 +260,7 @@ int y = foo(); auto bounds = computePreambleBounds(content); CompilationParams params; - - std::vector arguments = {"clang++", "-std=c++20", "main.cpp"}; - params.arguments = arguments; + params.arguments = {"clang++", "-std=c++20", "main.cpp"}; PCHInfo info; std::uint32_t last_bound = 0; diff --git a/unittests/Feature/CodeCompletion.cpp b/unittests/Feature/CodeCompletion.cpp index ce6b4ace..f5810e4b 100644 --- a/unittests/Feature/CodeCompletion.cpp +++ b/unittests/Feature/CodeCompletion.cpp @@ -9,9 +9,7 @@ struct CodeCompletion : TestFixture { auto code_complete(llvm::StringRef code) { Annotation annotation = {code}; CompilationParams params; - std::vector arguments = {"clang++", "-std=c++20", "main.cpp"}; - params.arguments = arguments; - + params.arguments = {"clang++", "-std=c++20", "main.cpp"}; params.completion = {"main.cpp", annotation.offset("pos")}; params.add_remapped_file("main.cpp", annotation.source()); diff --git a/unittests/Feature/SignatureHelp.cpp b/unittests/Feature/SignatureHelp.cpp index 9dfcc3f3..0a1a59c6 100644 --- a/unittests/Feature/SignatureHelp.cpp +++ b/unittests/Feature/SignatureHelp.cpp @@ -19,9 +19,7 @@ int main() { )cpp"; CompilationParams params; - std::vector arguments = {"clang++", "-std=c++20", "main.cpp"}; - params.arguments = arguments; - + params.arguments = {"clang++", "-std=c++20", "main.cpp"}; params.add_remapped_file("main.cpp", code); /// params.completion = {"main.cpp", 9, 10}; diff --git a/xmake.lua b/xmake.lua index 96eb08d5..8c8ae6c5 100644 --- a/xmake.lua +++ b/xmake.lua @@ -62,6 +62,7 @@ target("clice-core") links = { "LLVMSupport", "LLVMFrontendOpenMP", + "LLVMOption", "clangAST", "clangASTMatchers", "clangBasic",