caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: better load and fetch compile command (#201)

Browse Source 
Co-authored-by: ykiko <ykikoykikoykiko@gmail.com>
This commit is contained in:
Shiyu
2025-09-02 22:12:05 +08:00
committed by GitHub
parent 8337475b91
commit 155773cf66
6 changed files with 258 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
include/Compiler/Command.h
View File

@@ -35,7 +35,7 @@ public:
struct CommandInfo {
/// TODO: add sysroot or no stdinc command info.
llvm::StringRef dictionary;
llvm::StringRef directory;
/// The canonical command list.
llvm::ArrayRef<const char*> arguments;
@@ -57,7 +57,7 @@ public:
};
struct LookupInfo {
llvm::StringRef dictionary;
llvm::StringRef directory;
std::vector<const char*> arguments;
};
@@ -106,12 +106,23 @@ public:
llvm::StringRef command) -> UpdateInfo;
/// Update commands from json file and return all updated file.
auto load_commands(this Self& self, llvm::StringRef json_content)
auto load_commands(this Self& self, llvm::StringRef json_content, llvm::StringRef workspace)
-> std::expected<std::vector<UpdateInfo>, std::string>;
/// Get compile command from database. `file` should has relative path of workspace.
auto get_command(this Self& self, llvm::StringRef file, CommandOptions options = {})
-> LookupInfo;
/// Load compile commands from given directories. If no valid commands are found,
/// search recursively from the workspace directory.
auto load_compile_commands(this Self& self,
llvm::ArrayRef<std::string> compile_commands_dirs,
llvm::StringRef workspace) -> void;
private:
/// If file not found in CDB file, try to guess commands or use the default case.
auto guess_or_fallback(this Self& self, llvm::StringRef file) -> LookupInfo;
private:
/// The memory pool to hold all cstring and command list.
llvm::BumpPtrAllocator allocator;
@@ -128,10 +139,10 @@ private:
llvm::DenseSet<std::uint32_t> filtered_options;
/// A map between file path and its canonical command list.
llvm::DenseMap<const void*, CommandInfo> command_infos;
llvm::DenseMap<const char*, CommandInfo> command_infos;
/// A map between driver path and its query driver info.
llvm::DenseMap<const void*, DriverInfo> driver_infos;
llvm::DenseMap<const char*, DriverInfo> driver_infos;
};
} // namespace clice

2
include/Protocol/Lifecycle.h
View File

@@ -14,7 +14,7 @@ struct LSPInfo {
std::string name;
/// The version of server or client.
std::string verion;
std::string version;
};
struct WindowCapacities {};

166
src/Compiler/Command.cpp
View File

@@ -148,6 +148,7 @@ auto CompilationDatabase::query_driver(this Self& self, llvm::StringRef driver)
bool keep_output_file = true;
auto clean_up = llvm::make_scope_exit([&output_path, &keep_output_file]() {
if(keep_output_file) {
log::warn("Query driver failed, output file:{}", output_path);
return;
}
@@ -256,11 +257,11 @@ auto CompilationDatabase::query_driver(this Self& self, llvm::StringRef driver)
}
auto CompilationDatabase::update_command(this Self& self,
llvm::StringRef dictionary,
llvm::StringRef directory,
llvm::StringRef file,
llvm::ArrayRef<const char*> arguments) -> UpdateInfo {
file = self.save_string(file);
dictionary = self.save_string(dictionary);
directory = self.save_string(directory);
llvm::SmallVector<const char*, 16> filtered_arguments;
@@ -292,6 +293,21 @@ auto CompilationDatabase::update_command(this Self& self,
continue;
}
/// For arguments -I<dir>, convert directory to absolute path.
/// i.e xmake will generate commands in this style.
if(id == clang::driver::options::OPT_I) {
if(arg->getNumValues() == 1) {
add_argument("-I");
llvm::StringRef value = arg->getValue(0);
if(!value.empty() && !path::is_absolute(value)) {
add_argument(path::join(directory, value));
} else {
add_argument(value);
}
}
continue;
}
/// A workaround to remove extra PCH when cmake
/// generate PCH flags for clang.
if(id == clang::driver::options::OPT_Xclang) {
@@ -354,16 +370,15 @@ auto CompilationDatabase::update_command(this Self& self,
arguments = self.save_cstring_list(filtered_arguments);
UpdateKind kind = UpdateKind::Unchange;
CommandInfo info = {dictionary, arguments};
CommandInfo info = {directory, arguments};
auto [it, success] = self.command_infos.try_emplace(file.data(), info);
if(success) {
kind = UpdateKind::Create;
} else {
auto& info = it->second;
if(info.dictionary.data() != dictionary.data() ||
info.arguments.data() != arguments.data()) {
if(info.directory.data() != directory.data() || info.arguments.data() != arguments.data()) {
kind = UpdateKind::Update;
info.dictionary = dictionary;
info.directory = directory;
info.arguments = arguments;
}
}
@@ -372,7 +387,7 @@ auto CompilationDatabase::update_command(this Self& self,
}
auto CompilationDatabase::update_command(this Self& self,
llvm::StringRef dictionary,
llvm::StringRef directory,
llvm::StringRef file,
llvm::StringRef command) -> UpdateInfo {
llvm::BumpPtrAllocator local;
@@ -389,20 +404,22 @@ auto CompilationDatabase::update_command(this Self& self,
llvm::cl::TokenizeGNUCommandLine(command, saver, arguments);
}
return self.update_command(dictionary, file, arguments);
return self.update_command(directory, file, arguments);
}
auto CompilationDatabase::load_commands(this Self& self, llvm::StringRef json_content)
auto CompilationDatabase::load_commands(this Self& self,
llvm::StringRef json_content,
llvm::StringRef workspace)
-> std::expected<std::vector<UpdateInfo>, std::string> {
std::vector<UpdateInfo> infos;
auto json = json::parse(json_content);
if(!json) {
return std::unexpected(std::format("Fail to parse json: {}", json.takeError()));
return std::unexpected(std::format("parse json failed: {}", json.takeError()));
}
if(json->kind() != json::Value::Array) {
return std::unexpected("Compilation Database must be an array of object");
return std::unexpected("compile_commands.json must be an array of object");
}
/// FIXME: warn illegal item.
@@ -414,9 +431,22 @@ auto CompilationDatabase::load_commands(this Self& self, llvm::StringRef json_co
auto& object = *item.getAsObject();
auto file = object.getString("file");
auto directory = object.getString("directory");
if(!file || !directory) {
if(!directory) {
continue;
}
/// Always store relative path of source file.
std::string source;
if(auto file = object.getString("file")) {
if(path::is_absolute(*file)) {
llvm::SmallString<256> buffer = *file;
path::replace_path_prefix(buffer, workspace, "");
source = path::relative_path(buffer).str();
} else {
source = file->str();
}
} else {
continue;
}
@@ -432,12 +462,12 @@ auto CompilationDatabase::load_commands(this Self& self, llvm::StringRef json_co
}
}
auto info = self.update_command(*directory, *file, carguments);
auto info = self.update_command(*directory, source, carguments);
if(info.kind != UpdateKind::Unchange) {
infos.emplace_back(info);
}
} else if(auto command = object.getString("command")) {
auto info = self.update_command(*directory, *file, *command);
auto info = self.update_command(*directory, source, *command);
if(info.kind != UpdateKind::Unchange) {
infos.emplace_back(info);
}
@@ -454,32 +484,27 @@ auto CompilationDatabase::get_command(this Self& self, llvm::StringRef file, Com
file = self.save_string(file);
auto it = self.command_infos.find(file.data());
if(it != self.command_infos.end()) {
info.dictionary = it->second.dictionary;
info.directory = it->second.directory;
info.arguments = it->second.arguments;
} else {
/// FIXME: Use a better way to handle fallback command.
info.dictionary = {};
info.arguments = {
self.save_string("clang++").data(),
self.save_string("-std=c++20").data(),
};
info = self.guess_or_fallback(file);
}
auto append_argument = [&](llvm::StringRef argument) {
auto record = [&info, &self](llvm::StringRef argument) {
info.arguments.emplace_back(self.save_string(argument).data());
};
if(options.query_driver) {
llvm::StringRef driver = info.arguments[0];
if(auto driver_info = self.query_driver(driver)) {
append_argument("-nostdlibinc");
record("-nostdlibinc");
/// FIXME: Use target information here, this is useful for cross compilation.
/// FIXME: Cache -I so that we can append directly, avoid duplicate lookup.
for(auto& system_header: driver_info->system_includes) {
append_argument("-I");
append_argument(system_header);
record("-I");
record(system_header);
}
} else if(!options.suppress_log) {
log::warn("Failed to query driver:{}, error:{}", driver, driver_info.error());
@@ -487,11 +512,98 @@ auto CompilationDatabase::get_command(this Self& self, llvm::StringRef file, Com
}
if(options.resource_dir) {
append_argument(std::format("-resource-dir={}", fs::resource_dir));
record(std::format("-resource-dir={}", fs::resource_dir));
}
info.arguments.emplace_back(file.data());
/// TODO: apply rules in clice.toml.
return info;
}
auto CompilationDatabase::guess_or_fallback(this Self& self, llvm::StringRef file) -> LookupInfo {
// Try to guess command from other file in same directory or parent directory
llvm::StringRef dir = path::parent_path(file);
// Search up to 3 levels of parent directories
int up_level = 0;
while(!dir.empty() && up_level < 3) {
// If any file in the directory has a command, use that command
for(const auto& [other_file, info]: self.command_infos) {
llvm::StringRef other = other_file;
// Filter case that dir is /path/to/foo and there's another directory /path/to/foobar
if(other.starts_with(dir) &&
(other.size() == dir.size() || path::is_separator(other[dir.size()]))) {
log::info("Guess command for:{}, from existed file: {}", file, other_file);
return LookupInfo{info.directory, info.arguments};
}
}
dir = path::parent_path(dir);
up_level += 1;
}
/// FIXME: use a better default case.
// Fallback to default case.
LookupInfo info;
constexpr const char* fallback[] = {"clang++", "-std=c++20"};
for(const char* arg: fallback) {
info.arguments.emplace_back(self.save_string(arg).data());
}
return info;
}
auto CompilationDatabase::load_compile_commands(this Self& self,
llvm::ArrayRef<std::string> compile_commands_dirs,
llvm::StringRef workspace) -> void {
auto try_load = [&self, workspace](llvm::StringRef dir) {
std::string filepath = path::join(dir, "compile_commands.json");
auto content = fs::read(filepath);
if(!content) {
log::warn("Failed to read CDB file: {}, {}", filepath, content.error());
return false;
}
auto load = self.load_commands(*content, workspace);
if(!load) {
log::warn("Failed to load CDB file: {}. {}", filepath, load.error());
return false;
}
log::info("Load CDB file: {} successfully, {} items loaded", filepath, load->size());
return true;
};
if(std::ranges::any_of(compile_commands_dirs, try_load)) {
return;
}
log::info(
"Can not found any valid CDB file from given directories, search recursively from workspace: {} ...",
workspace);
std::error_code ec;
for(fs::recursive_directory_iterator it(workspace, ec), end; it != end && !ec;
it.increment(ec)) {
auto status = it->status();
if(!status) {
continue;
}
// Skip hidden directories.
llvm::StringRef filename = path::filename(it->path());
if(fs::is_directory(*status) && filename.starts_with('.')) {
it.no_push();
continue;
}
if(fs::is_regular_file(*status) && filename == "compile_commands.json") {
if(try_load(path::parent_path(it->path()))) {
return;
}
}
}
/// TODO: Add a default command in clice.toml. Or load commands from .clangd ?
log::warn("Can not found any valid CDB file in current workspace, fallback to default mode.");
}
} // namespace clice

9
src/Server/Document.cpp
View File

@@ -194,8 +194,6 @@ async::Task<bool> build_pch_task(CompilationDatabase::LookupInfo& info,
params.diagnostics = diagnostics;
params.add_remapped_file(path, content, bound);
PCHInfo pch;
std::string command;
for(auto argument: params.arguments) {
command += " ";
@@ -203,13 +201,14 @@ async::Task<bool> build_pch_task(CompilationDatabase::LookupInfo& info,
}
log::info("Start building PCH for {}, command: [{}]", path, command);
command.clear();
std::string message;
PCHInfo pch;
std::string message = std::move(command); // reuse buffer
std::vector<feature::DocumentLink> links;
bool success = co_await async::submit([&params, &pch, &message, &links] -> bool {
/// PCH file is written until destructing, Add a single block
/// for it.
/// PCH file is written until destructing, Add a single block for it.
auto unit = compile(params, pch);
if(!unit) {
message = std::move(unit.error());

11
src/Server/Lifecycle.cpp
View File

@@ -5,7 +5,7 @@ namespace clice {
async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
log::info("Initialize from client: {}, version: {}",
params.clientInfo.name,
params.clientInfo.verion);
params.clientInfo.version);
/// FIXME: adjust position encoding.
kind = PositionEncodingKind::UTF16;
@@ -27,12 +27,7 @@ async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
opening_files.set_capability(config::server.max_active_file);
/// Load compile commands.json
for(auto& dir: config::server.compile_commands_dirs) {
auto content = fs::read(dir + "/compile_commands.json");
if(content) {
auto updated = database.load_commands(*content);
}
}
database.load_compile_commands(config::server.compile_commands_dirs, workspace);
/// Load cache info.
load_cache_info();
@@ -40,7 +35,7 @@ async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
proto::InitializeResult result;
auto& [info, capabilities] = result;
info.name = "clice";
info.verion = "0.0.1";
info.version = "0.0.1";
capabilities.positionEncoding = "utf-16";

95
tests/unit/Compiler/Command.cpp
View File

@@ -216,6 +216,101 @@ suite<"Command"> command = [] {
/// expect(that % command[2] == "test.cpp"sv);
/// expect(that % command[3] == std::format("-resource-dir={}", fs::resource_dir));
};
auto expect_load = [](llvm::StringRef content,
llvm::StringRef workspace,
llvm::StringRef file,
llvm::StringRef directory,
llvm::ArrayRef<const char*> arguments) {
CompilationDatabase database;
auto loaded = database.load_commands(content, workspace);
expect(that % loaded.has_value());
CommandOptions options;
options.suppress_log = true;
auto info = database.get_command(file, options);
expect(that % info.directory == directory);
expect(that % info.arguments.size() == arguments.size());
for(size_t i = 0; i < arguments.size(); i++) {
llvm::StringRef arg = info.arguments[i];
llvm::StringRef expect_arg = arguments[i];
expect(that % arg == expect_arg);
}
};
#if defined(__unix__) || defined(__APPLE__)
/// TODO: add windows path testcase
test("LoadAbsoluteUnixStyle") = [expect_load] {
constexpr const char* cmake = R"([
{
"directory": "/home/developer/clice/build",
"command": "/usr/bin/c++ -I/home/developer/clice/include -I/home/developer/clice/build/_deps/libuv-src/include -isystem /home/developer/clice/build/_deps/tomlplusplus-src/include -std=gnu++23 -fno-rtti -fno-exceptions -Wno-deprecated-declarations -Wno-undefined-inline -O3 -o CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o -c /home/developer/clice/src/Driver/clice.cpp",
"file": "/home/developer/clice/src/Driver/clice.cpp",
"output": "CMakeFiles/clice-core.dir/src/Driver/clice.cpp.o"
}
])";
expect_load(cmake,
"/home/developer/clice",
"src/Driver/clice.cpp",
"/home/developer/clice/build",
{
"/usr/bin/c++",
"-I",
"/home/developer/clice/include",
"-I",
"/home/developer/clice/build/_deps/libuv-src/include",
"-isystem",
"/home/developer/clice/build/_deps/tomlplusplus-src/include",
"-std=gnu++23",
"-fno-rtti",
"-fno-exceptions",
"-Wno-deprecated-declarations",
"-Wno-undefined-inline",
"-O3",
"src/Driver/clice.cpp",
});
};
test("LoadRelativeUnixStyle") = [expect_load] {
constexpr const char* xmake = R"([
{
"directory": "/home/developer/clice",
"arguments": ["/usr/bin/clang", "-c", "-Qunused-arguments", "-m64", "-g", "-O0", "-std=c++23", "-Iinclude", "-I/home/developer/clice/include", "-fno-exceptions", "-fno-cxx-exceptions", "-isystem", "/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include", "-isystem", "/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include", "-fsanitize=address", "-fno-rtti", "-o", "build/.objs/clice/linux/x86_64/debug/src/Driver/clice.cc.o", "src/Driver/clice.cc"],
"file": "src/Driver/clice.cc"
}
])";
expect_load(
xmake,
"/home/developer/clice",
"src/Driver/clice.cc",
"/home/developer/clice",
{
"/usr/bin/clang",
"-Qunused-arguments",
"-m64",
"-g",
"-O0",
"-std=c++23",
// parameter "-Iinclude" in CDB, should be convert to absolute path
"-I",
"/home/developer/clice/include",
"-I",
"/home/developer/clice/include",
"-fno-exceptions",
"-fno-cxx-exceptions",
"-isystem",
"/home/developer/.xmake/packages/l/libuv/v1.51.0/3ca1562e6c5d485f9ccafec8e0c50b6f/include",
"-isystem",
"/home/developer/.xmake/packages/t/toml++/v3.4.0/bde7344d843e41928b1d325fe55450e0/include",
"-fsanitize=address",
"-fno-rtti",
"src/Driver/clice.cc",
});
};
#endif
};
} // namespace