caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: incremental update for compilation database and introduce query toolchain (#311)

Browse Source
This commit is contained in:
ykiko
2025-11-23 18:43:36 +08:00
committed by GitHub
parent f16867902c
commit 8aff090a08
31 changed files with 1871 additions and 1168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/AST/Selection.cpp
View File

@@ -948,10 +948,10 @@ private:
}
if(!checker.may_hit(S)) {
LOGGING_DEBUG("{2}skip: {0} {1}",
print_node_to_string(N, print_policy),
S.printToString(SM),
indent());
LOG_DEBUG("{2}skip: {0} {1}",
print_node_to_string(N, print_policy),
S.printToString(SM),
indent());
return true;
}
@@ -971,10 +971,10 @@ private:
// Performs early hit detection for some nodes (on the earlySourceRange).
void push(clang::DynTypedNode node) {
clang::SourceRange Early = early_source_range(node);
LOGGING_DEBUG("{2}push: {0} {1}",
print_node_to_string(node, print_policy),
node.getSourceRange().printToString(SM),
indent());
LOG_DEBUG("{2}push: {0} {1}",
print_node_to_string(node, print_policy),
node.getSourceRange().printToString(SM),
indent());
nodes.emplace_back();
nodes.back().data = std::move(node);
nodes.back().parent = stack.top();
@@ -987,7 +987,7 @@ private:
// Performs primary hit detection.
void pop() {
Node& N = *stack.top();
LOGGING_DEBUG("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1));
LOG_DEBUG("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1));
claim_tokens_for(N.data, N.selected);
if(N.selected == no_tokens) {
N.selected = SelectionTree::Unselected;
@@ -1118,7 +1118,7 @@ private:
}
if(result && result != no_tokens) {
LOGGING_DEBUG("{1}hit selection: {0}", S.printToString(SM), indent());
LOG_DEBUG("{1}hit selection: {0}", S.printToString(SM), indent());
}
}
@@ -1243,9 +1243,9 @@ SelectionTree::SelectionTree(CompilationUnit& unit, LocalSourceRange range) :
print_policy.IncludeNewlines = false;
auto [begin, end] = range;
LOGGING_DEBUG("Computing selection for {0}",
clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end))
.printToString(SM));
LOG_DEBUG("Computing selection for {0}",
clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end))
.printToString(SM));
nodes = SelectionVisitor::collect(unit, print_policy, range, fid);
m_root = nodes.empty() ? nullptr : &nodes.front();

2
src/Async/FileSystem.cpp
View File

@@ -122,7 +122,7 @@ handle::~handle() {
uv_fs_t request;
int error = uv_fs_close(async::loop, &request, file, nullptr);
if(error < 0) {
LOGGING_WARN("Failed to close file: {}", uv_strerror(error));
LOG_WARN("Failed to close file: {}", uv_strerror(error));
}
uv_fs_req_cleanup(&request);
}

6
src/Async/Network.cpp
View File

@@ -30,7 +30,7 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) {
/// If an error occurred while reading, we can't continue.
if(nread < 0) [[unlikely]] {
LOGGING_FATAL("An error occurred while reading: {0}", uv_strerror(nread));
LOG_FATAL("An error occurred while reading: {0}", uv_strerror(nread));
}
/// We have at most one connection and use default event loop. So there is no data race
@@ -57,7 +57,7 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) {
task.dispose();
} else {
/// If the message is invalid, we can't continue.
LOGGING_FATAL("Unexpected JSON input: {0}", result);
LOG_FATAL("Unexpected JSON input: {0}", result);
}
/// Remove the processed message from the buffer.
@@ -130,7 +130,7 @@ struct write {
uv_write(&req, writer, buf, 2, [](uv_write_t* req, int status) {
if(status < 0) {
LOGGING_FATAL("An error occurred while writing: {0}", uv_strerror(status));
LOG_FATAL("An error occurred while writing: {0}", uv_strerror(status));
}
auto& awaiter = uv_cast<struct write>(req);

7
src/Async/libuv.cpp
View File

@@ -29,11 +29,8 @@ const std::error_category& category() {
/// Use source_location to log the file, line, and function name where the error occurred.
void uv_check_result(const int result, const std::source_location location) {
if(result < 0) {
LOGGING_WARN("libuv error: {}", uv_strerror(result));
LOGGING_WARN("At {}:{}:{}",
location.file_name(),
location.line(),
location.function_name());
LOG_WARN("libuv error: {}", uv_strerror(result));
LOG_WARN("At {}:{}:{}", location.file_name(), location.line(), location.function_name());
}
}

1085
src/Compiler/Command.cpp
View File

File diff suppressed because it is too large Load Diff

48
src/Compiler/Compilation.cpp
View File

@@ -10,6 +10,7 @@
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/MultiplexConsumer.h"
#include "Support/Logging.h"
namespace clice {
@@ -99,20 +100,40 @@ private:
auto create_invocation(CompilationParams& params,
llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine>& diagnostic_engine)
-> std::unique_ptr<clang::CompilerInvocation> {
if(params.arguments.empty()) {
LOG_ERROR_RET(nullptr, "Fail to create invocation: empty argument list from database");
}
/// Create clang invocation.
clang::CreateInvocationOptions options = {
.Diags = diagnostic_engine,
.VFS = params.vfs,
std::unique_ptr<clang::CompilerInvocation> invocation;
/// Avoid replacing -include with -include-pch, also
/// see https://github.com/clangd/clangd/issues/856.
.ProbePrecompiled = false,
};
/// Arguments from compilation database are already cc1
if(params.arguments_from_database) {
invocation = std::make_unique<clang::CompilerInvocation>();
if(!clang::CompilerInvocation::CreateFromArgs(*invocation,
llvm::ArrayRef(params.arguments).drop_front(),
*diagnostic_engine,
params.arguments[0])) {
LOG_ERROR_RET(nullptr,
" Fail to create invocation, arguments list is: {}",
print_argv(params.arguments));
}
} else {
/// Create clang invocation.
clang::CreateInvocationOptions options = {
.Diags = diagnostic_engine,
.VFS = params.vfs,
auto invocation = clang::createInvocation(params.arguments, options);
if(!invocation) {
return nullptr;
/// Avoid replacing -include with -include-pch, also
/// see https://github.com/clangd/clangd/issues/856.
.ProbePrecompiled = false,
};
invocation = clang::createInvocation(params.arguments, options);
if(!invocation) {
LOG_ERROR_RET(nullptr,
" Fail to create invocation, arguments list is: {}",
print_argv(params.arguments));
}
}
auto& pp_opts = invocation->getPreprocessorOpts();
@@ -336,7 +357,10 @@ CompilationResult preprocess(CompilationParams& params) {
}
CompilationResult compile(CompilationParams& params) {
return run_clang<clang::SyntaxOnlyAction>(params);
return run_clang<clang::SyntaxOnlyAction>(params, [](clang::CompilerInstance& instance) {
/// Make sure the output file is empty.
instance.getFrontendOpts().OutputFile.clear();
});
}
CompilationResult compile(CompilationParams& params, PCHInfo& out) {

6
src/Compiler/Tidy.cpp
View File

@@ -334,11 +334,11 @@ std::unique_ptr<ClangTidyChecker> configure(clang::CompilerInstance& instance,
return nullptr;
}
auto file_name = input.getFile();
LOGGING_INFO("Tidy configure file: {}", file_name);
LOG_INFO("Tidy configure file: {}", file_name);
tidy::ClangTidyOptions opts = create_options();
if(opts.Checks) {
LOGGING_INFO("Tidy configure checks: {}", *opts.Checks);
LOG_INFO("Tidy configure checks: {}", *opts.Checks);
}
{
@@ -391,7 +391,7 @@ std::unique_ptr<ClangTidyChecker> configure(clang::CompilerInstance& instance,
checker->context.setCurrentFile(file_name);
checker->context.setSelfContainedDiags(true);
checker->checks = factories.createChecksForLanguage(&checker->context);
LOGGING_INFO("Tidy configure checks: {}", checker->checks.size());
LOG_INFO("Tidy configure checks: {}", checker->checks.size());
clang::Preprocessor* pp = &instance.getPreprocessor();
for(const auto& check: checker->checks) {
check->registerPPCallbacks(instance.getSourceManager(), pp, pp);

576
src/Compiler/Toolchain.cpp
View File

@@ -1,4 +1,5 @@
#include "Compiler/Toolchain.h"
#include "Compiler/Command.h"
#include "Support/FileSystem.h"
#include "Support/Logging.h"
#include "llvm/ADT/ScopeExit.h"
@@ -6,96 +7,184 @@
#include "llvm/Support/Program.h"
#include "llvm/Support/FileSystem.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Tool.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/TargetParser/Host.h"
#ifndef _WIN32
#include <unistd.h>
extern char** environ;
static llvm::ArrayRef<llvm::StringRef> envs() {
static std::vector<std::string> storage;
static auto refs = [] {
std::vector<llvm::StringRef> refs;
if(environ) {
for(char** env = environ; *env != nullptr; ++env) {
llvm::StringRef s(*env);
if(!s.starts_with("LANG=")) {
storage.emplace_back(*env);
}
}
storage.emplace_back("LANG=C");
}
/// Note that store the reference os strings in the vector
/// is not safe when vector grows capacity. But we store it
/// after all insertion are completed. It's safe here.
for(const auto& s: storage) {
refs.emplace_back(s);
}
return refs;
}();
return refs;
}
#endif
#ifdef _WIN32
llvm::StringRef null_dev = "NUL";
#else
llvm::StringRef null_dev = "/dev/null";
#endif
namespace clice::toolchain {
namespace opt = llvm::opt;
namespace driver = clang::driver;
namespace {
/// Checks if dash-dash (`--`) parsing is enabled. If enabled, all arguments
/// after a standalone `--` are treated as positional arguments (e.g., input files).
bool enable_dash_dash_parsing(const opt::OptTable& table);
std::optional<std::string> execute_command(llvm::ArrayRef<const char*> arguments,
bool capture_stdout = false) {
LOG_INFO("Execute command: {}", print_argv(arguments));
/// Checks if grouped short options are enabled. If enabled, a short option group
/// like `-ab` is parsed as separate options `-a` and `-b`.
bool enable_grouped_short_options(const opt::OptTable& table);
/// Get the specific toolchain of given target, we mainly use it to get msvc toolchain.
const driver::ToolChain& get_toolchain(driver::Driver& driver,
const opt::ArgList& Args,
const llvm::Triple& Target);
template <auto MP1, auto MP2, auto MP3>
struct Thief {
friend bool enable_dash_dash_parsing(const opt::OptTable& table) {
return table.*MP1;
llvm::SmallString<64> path;
if(auto e = fs::createTemporaryFile("query-toolchain", "clice", path)) {
LOG_ERROR_RET(std::nullopt, "Fail to create temporary file: {}", e);
}
friend bool enable_grouped_short_options(const opt::OptTable& table) {
return table.*MP2;
auto _ = llvm::make_scope_exit([&path]() {
if(auto e = fs::remove(path)) {
LOG_ERROR("Fail to remove temporary file: {}", e);
}
});
#ifdef _WIN32
/// If the env is `std::nullopt`, `ExecuteAndWait` will inherit env from parent process,
/// which is very important for msvc and clang on windows. Thay depend on the environment
/// variables to find correct standard library path.
constexpr auto env = std::nullopt;
#else
/// For linux, we should append or modify the "LANG=C" to the env, this is important
/// for gcc with locality. Otherwise, it will output non-ASCII char. We also want
/// to inherit the environment variables like windows.
auto env = envs();
#endif
std::optional<llvm::StringRef> redirects[3] = {
{null_dev}, // stdin
{capture_stdout ? path.str() : null_dev}, // stdout
{capture_stdout ? null_dev : path.str()}, // stderr
};
llvm::SmallVector<llvm::StringRef> argv(arguments.begin(), arguments.end());
std::string message;
if(int rc = llvm::sys::ExecuteAndWait(arguments[0],
argv,
env,
redirects,
/*SecondsToWait=*/0,
/*MemoryLimit=*/0,
&message)) {
/// FIXME: handle error when rc is positive.
LOG_ERROR_RET(std::nullopt,
"Fail to execute {}, return code is {}, because: {}",
arguments[0],
rc,
message);
}
friend const driver::ToolChain& get_toolchain(driver::Driver& driver,
const opt::ArgList& args,
const llvm::Triple& target) {
return (driver.*MP3)(args, target);
auto file = llvm::MemoryBuffer::getFile(path);
if(!file) {
LOG_ERROR_RET(std::nullopt, "Fail to read redirect file: {}", file.getError());
}
};
template struct Thief<&opt::OptTable::DashDashParsing,
&opt::OptTable::GroupedShortOptions,
&driver::Driver::getToolChain>;
Toolchain query_toolchain(llvm::ArrayRef<const char*> arguments) {
llvm::StringRef driver = arguments[0];
/// judge tool chain kind ...
return {};
return file->get()->getBuffer().str();
}
using ErrorKind = toolchain::QueryDriverError::ErrorKind;
bool query_driver(
llvm::ArrayRef<const char*> arguments,
llvm::function_ref<void(const char* driver, llvm::ArrayRef<const char*> cc1_args)> callback) {
/// FIXME: collect diagnostic here ...
clang::DiagnosticOptions options;
clang::DiagnosticsEngine engine(new clang::DiagnosticIDs(),
options,
new clang::IgnoringDiagConsumer());
auto unexpected(ErrorKind kind, std::string message) {
return std::unexpected<toolchain::QueryDriverError>({kind, std::move(message)});
};
llvm::SmallVector<const char*, 256> list;
list.emplace_back(arguments.consume_front());
list.emplace_back("-fsyntax-only");
list.append(arguments.begin(), arguments.end());
arguments = list;
enum class CompilerFamily {
Unknown,
GCC, // Covers gcc, g++, cc, c++, and versioned/arch variants
Clang, // Covers clang, clang++, and versioned variants (excluding clang-cl)
MSVC, // Covers cl
ClangCL, // Covers clang-cl explicitly
NVCC, // Covers nvcc
Intel, // Covers icc, icpc, icx, dpcpp
Zig, // Covers zig cc / zig c++ (assumed GCC/Clang compatible for query)
};
/// Note that clang use the `ClangExecutable` to determine the driver mode when
/// --driver-mode is not found in the arguments. and `TargetTriple` is used when
/// non --target argument is found in the arguments list. See
/// `clang::driver::BuildCompilation`. We use default arguments because we will
/// inject related commands before querying.
clang::driver::Driver driver(/*ClangExecutable=*/arguments[0],
/*TargetTriple=*/llvm::sys::getDefaultTargetTriple(),
/*Diags=*/engine);
driver.setCheckInputsExist(false);
driver.setProbePrecompiled(false);
CompilerFamily driver_family(llvm::StringRef driver) {
auto driver_name = llvm::sys::path::filename(driver);
driver_name.consume_back(".exe");
if(driver_name == "cl") {
return CompilerFamily::MSVC;
} else if(driver_name == "nvcc") {
return CompilerFamily::NVCC;
} else if(driver_name.contains("clang-cl")) {
return CompilerFamily::ClangCL;
} else if(driver_name.contains("clang")) {
return CompilerFamily::Clang;
} else if(driver_name == "cc" || driver_name == "c++" || driver_name.contains("gcc") ||
driver_name.contains("g++")) {
return CompilerFamily::GCC;
} else if(driver_name.contains("icpc") || driver_name.contains("icc") ||
driver_name.contains("dpcpp") || driver_name.contains("icx")) {
return CompilerFamily::Intel;
} else if(driver_name.contains("zig")) {
return CompilerFamily::Zig;
std::unique_ptr<clang::driver::Compilation> compilation(driver.BuildCompilation(arguments));
if(!compilation) {
LOG_ERROR_RET(false, "Fail to query driver");
}
return CompilerFamily::Unknown;
// We expect to get back exactly one command job, if we didn't something
// failed. Offload compilation is an exception as it creates multiple jobs. If
// that's the case, we proceed with the first job. If caller needs a
// particular job, it should be controlled via options (e.g.
// --cuda-{host|device}-only for CUDA) passed to the driver.
const clang::driver::JobList& jobs = compilation->getJobs();
bool offload_compilation = false;
if(jobs.size() > 1) {
for(auto& action: compilation->getActions()) {
// On MacOSX real actions may end up being wrapped in BindArchAction
if(llvm::isa<clang::driver::BindArchAction>(action)) {
action = *action->input_begin();
}
if(llvm::isa<clang::driver::OffloadAction>(action)) {
offload_compilation = true;
break;
}
}
}
auto cmd = llvm::find_if(jobs, [](const clang::driver::Command& cmd) {
return cmd.getCreator().getName() == llvm::StringRef("clang");
});
if(cmd == jobs.end()) {
LOG_ERROR_RET(false, "Fail to query driver, clang job was not found!");
}
callback(arguments[0], cmd->getArguments());
return true;
}
auto parse_query_result(llvm::StringRef content, QueryResult& info)
-> std::expected<void, QueryDriverError> {
struct QueryResult {
llvm::StringRef target;
std::vector<llvm::StringRef> includes;
};
/// TODO: use this to print the output of -v.
void parse_version_result(llvm::StringRef content, QueryResult& info) {
const char* TS = "Target: ";
const char* SIS = "#include <...> search starts here:";
const char* SIE = "End of search list.";
@@ -134,18 +223,69 @@ auto parse_query_result(llvm::StringRef content, QueryResult& info)
}
if(!found_start_marker) {
return unexpected(ErrorKind::InvalidOutputFormat, "Start marker not found...");
LOG_ERROR("Failed to parse version output: missing include search start marker");
return;
}
if(in_includes_block) {
return unexpected(ErrorKind::InvalidOutputFormat, "End marker not found...");
LOG_ERROR("Failed to parse version output: unclosed include search block");
return;
}
return std::expected<void, QueryDriverError>();
}
auto query_driver(llvm::StringRef driver) -> std::expected<QueryResult, QueryDriverError> {
llvm::SmallString<128> path;
} // namespace
CompilerFamily driver_family(llvm::StringRef driver) {
auto try_get = [](llvm::StringRef name) {
if(name == "cl") {
return CompilerFamily::MSVC;
} else if(name == "nvcc") {
return CompilerFamily::NVCC;
} else if(name.ends_with("clang") || name.ends_with("clang++")) {
return CompilerFamily::Clang;
} else if(name.ends_with("clang-cl")) {
return CompilerFamily::ClangCL;
} else if(name.ends_with("cc") || name.ends_with("c++") || name.ends_with("gcc") ||
name.ends_with("g++")) {
return CompilerFamily::GCC;
} else if(name.contains("icpc") || name.contains("icc") || name.contains("dpcpp") ||
name.contains("icx")) {
return CompilerFamily::Intel;
} else if(name.ends_with("zig")) {
return CompilerFamily::Zig;
}
return CompilerFamily::Unknown;
};
auto driver_name = llvm::sys::path::filename(driver);
auto family = try_get(driver_name);
if(family != CompilerFamily::Unknown) {
return family;
}
// Stripping the executable suffix: clang++.exe -> clang++
driver_name.consume_back(".exe");
family = try_get(driver_name);
if(family != CompilerFamily::Unknown) {
return family;
}
// Stripping any trailing version number: clang++3.5 -> clang++
driver_name = driver_name.rtrim("0123456789.-");
family = try_get(driver_name);
if(family != CompilerFamily::Unknown) {
return family;
}
/// Stripping trailing -component. clang++-tot -> clang++
driver_name = driver_name.slice(0, driver_name.rfind('-'));
family = try_get(driver_name);
return family;
}
std::vector<const char*> query_toolchain(const QueryParams& params) {
auto arguments = params.arguments;
llvm::StringRef driver = arguments[0];
/// Note: The name used to invoke the compiler driver affects its behavior.
/// For example, `/usr/bin/clang++` is often a symbolic link to
@@ -153,130 +293,198 @@ auto query_driver(llvm::StringRef driver) -> std::expected<QueryResult, QueryDri
/// and links C++ libraries by default, while invoking as `clang` defaults to C mode.
/// Therefore, never use `realpath` on the initial `driver` name, as that
/// would lose the context needed for the driver to behave correctly (and break caching).
if(!llvm::sys::path::is_absolute(driver)) {
llvm::SmallString<128> path;
if(!path::is_absolute(driver)) {
/// If the path is not absolute path like g++, find it in the env vars.
auto program = llvm::sys::findProgramByName(driver);
if(!program) {
return unexpected(ErrorKind::NotFoundInPATH, program.getError().message());
LOG_ERROR_RET({}, "Fail to query driver, cannot find the driver: {}", driver);
}
path = *program;
driver = path;
driver = path.c_str();
}
/// Check whether we can execute the driver.
if(!llvm::sys::fs::exists(driver) || !llvm::sys::fs::can_execute(driver)) {
/// FIXME: Add whitelisting, blacklisting (do not trust workspace executables),
/// and toolchain integrity checks.
return unexpected(ErrorKind::NotFoundInPATH, "");
if(!fs::exists(driver) || !fs::can_execute(driver)) {
LOG_ERROR_RET({}, "Fail to query driver, driver: {} is not existent or executable", driver);
}
auto params_copy = params;
llvm::SmallVector<const char*, 256> modified_arguments;
/// Remove driver
arguments.consume_front();
modified_arguments.emplace_back(driver.data());
/// Remove input file
auto ext = path::extension(params.file);
ext.consume_front(".");
modified_arguments.append(arguments.begin(), arguments.end());
/// Create a file with same suffix of input file, because the input file may
/// not exist in the disk.
llvm::SmallString<64> src_path;
if(auto e = fs::createTemporaryFile("query-toolchain", ext, src_path)) {
LOG_ERROR_RET({}, "Fail to create temporary file: {}", e);
}
auto _ = llvm::make_scope_exit([&src_path]() {
if(auto e = fs::remove(src_path)) {
LOG_ERROR("Fail to remove temporary file: {}", e);
}
});
modified_arguments.emplace_back(src_path.c_str());
arguments = modified_arguments;
params_copy.arguments = arguments;
auto family = driver_family(driver);
/// FIXME: Handle nvcc and intel compiler.
if(family == CompilerFamily::NVCC || family == CompilerFamily::Intel ||
family == CompilerFamily::Zig || family == CompilerFamily::Unknown) [[unlikely]] {
/// FIXME: nvcc and intel compilers need further exploration.
/// zig is easy to handle, just use `zig cc` or `zig c++`, then
/// it will behave like clang.
return unexpected(ErrorKind::NotImplemented, "");
}
/// Query the compiler for includes information.
if(family == CompilerFamily::GCC || family == CompilerFamily::Clang) {
llvm::SmallString<128> output_path;
if(auto error =
llvm::sys::fs::createTemporaryFile("system-includes", "clice", output_path)) {
return unexpected(ErrorKind::FailToCreateTempFile, error.message());
switch(family) {
case CompilerFamily::GCC: {
return query_gcc_toolchain(params_copy);
}
// If we fail to get the driver infomation, keep the output file for user to debug.
bool keep_output_file = true;
auto clean_up = llvm::make_scope_exit([&output_path, &keep_output_file]() {
if(keep_output_file) {
LOGGING_WARN("Query driver failed, output file:{}", output_path);
return;
}
if(auto errc = llvm::sys::fs::remove(output_path)) {
LOGGING_WARN("Fail to remove temporary file: {}", errc.message());
}
});
/// FIXME: Is it possible that the output is not in stderr?
std::optional<llvm::StringRef> redirects[3] = {
{""},
{""},
{output_path.str()},
};
#ifdef _WIN32
/// If the env is `std::nullopt`, `ExecuteAndWait` will inherit env from parent process,
/// which is very important for msvc and clang on windows. Thay depend on the environment
/// variables to find correct standard library path.
constexpr auto env = std::nullopt;
llvm::SmallVector<llvm::StringRef, 6> argv = {driver, "-E", "-v", "-xc++", "NUL"};
#else
/// FIXME: We should find a better way to convert "LANG=C", this is important
/// for gcc with locality. Otherwise, it will output non-ASCII char. We also
/// want to inherit the environment variables like windows.
llvm::SmallVector<llvm::StringRef> env = {"LANG=C"};
llvm::SmallVector<llvm::StringRef> argv = {driver, "-E", "-v", "-xc++", "/dev/null"};
#endif
std::string message;
if(int RC = llvm::sys::ExecuteAndWait(driver,
argv,
env,
redirects,
/*SecondsToWait=*/0,
/*MemoryLimit=*/0,
&message)) {
return unexpected(ErrorKind::InvokeDriverFail, std::move(message));
case CompilerFamily::Clang:
case CompilerFamily::Zig: {
return query_clang_toolchain(params_copy);
}
case CompilerFamily::MSVC:
case CompilerFamily::ClangCL: {
return query_msvc_toolchain(params_copy);
}
auto file = llvm::MemoryBuffer::getFile(output_path);
if(!file) {
return unexpected(ErrorKind::OutputFileNotReadable, file.getError().message());
}
case CompilerFamily::NVCC:
case CompilerFamily::Intel:
case CompilerFamily::Unknown: {
/// TODO: nvcc and intel compilers need further exploration.
LOG_ERROR("Fail to query driver, unknown supported driver kind: {}, driver is {}",
refl::enum_name(family),
driver);
QueryResult info;
if(auto r = parse_query_result(file.get()->getBuffer(), info)) {
keep_output_file = false;
return info;
} else {
return std::unexpected(r.error());
std::vector<const char*> result;
query_driver(params_copy.arguments,
[&](const char* driver, llvm::ArrayRef<const char*> cc1_args) {
result.emplace_back(params.callback(driver));
for(auto arg: cc1_args) {
result.emplace_back(params.callback(arg));
}
});
return result;
}
}
/// For msvc and clang-cl, we don't need to query driver. Just use clang
/// tool chain to find the built includes.
if(family == CompilerFamily::MSVC || family == CompilerFamily::ClangCL) {
/// FIXME: target information? e.g. arm cross compilation.
llvm::StringRef target = "x86_64-pc-windows-msvc";
/// An workaround to use clang's toolchain to find vsinstall information
/// and related includes.
clang::DiagnosticOptions options;
thread_local clang::DiagnosticsEngine engine(new clang::DiagnosticIDs(), options);
clang::driver::Driver driver("", target, engine);
llvm::SmallVector<const char*> args = {"", "-xc++", "NUL"};
llvm::opt::InputArgList list(args.begin(), args.end());
auto& toolchain = get_toolchain(driver, list, llvm::Triple(target));
/// FIXME: specify specific version of vs?
llvm::opt::ArgStringList includes;
toolchain.AddClangSystemIncludeArgs(list, includes);
QueryResult info;
info.target = target;
for(auto& include: includes) {
info.includes.emplace_back(include);
}
return info;
}
std::unreachable();
}
std::vector<const char*> query_gcc_toolchain(const QueryParams& params) {
auto arguments = params.arguments;
llvm::SmallVector<const char*, 256> query_arguments;
llvm::SmallString<64> target;
llvm::SmallString<64> install_path;
query_arguments = {arguments[0], "-dumpmachine"};
if(auto content = execute_command(query_arguments, true)) {
target = llvm::StringRef(*content).trim();
}
query_arguments = {arguments[0], "-print-search-dirs"};
if(auto content = execute_command(query_arguments, true)) {
llvm::SmallVector<llvm::StringRef, 5> lines;
llvm::StringRef(*content).split(lines, '\n', -1, /*KeepEmpty=*/false);
for(auto line: lines) {
line = line.trim();
if(line.consume_front_insensitive("install:")) {
install_path = line.trim();
break;
}
}
}
target = std::format("--target={}", target);
install_path = std::format("--gcc-install-dir={}", install_path);
query_arguments.clear();
query_arguments.emplace_back(arguments.consume_front());
query_arguments.emplace_back(target.c_str());
query_arguments.emplace_back(install_path.c_str());
query_arguments.append(arguments.begin(), arguments.end());
std::vector<const char*> result;
query_driver(query_arguments, [&](const char* driver, llvm::ArrayRef<const char*> cc1_args) {
result.emplace_back(params.callback(driver));
for(auto arg: cc1_args) {
result.emplace_back(params.callback(arg));
}
});
return result;
}
std::vector<const char*> query_clang_toolchain(const QueryParams& params) {
auto arguments = params.arguments;
llvm::SmallVector<const char*, 256> query_arguments;
if(driver_family(arguments[0]) == CompilerFamily::Zig) {
/// zig cc or zig c++ consumes two arguments.
query_arguments.emplace_back(arguments.consume_front());
query_arguments.emplace_back(arguments.consume_front());
} else {
query_arguments.emplace_back(arguments.consume_front());
}
query_arguments.emplace_back("-###");
query_arguments.emplace_back("-fsyntax-only");
query_arguments.append(arguments.begin(), arguments.end());
std::vector<const char*> result;
if(auto content = execute_command(query_arguments, false)) {
llvm::SmallVector<llvm::StringRef> lines;
llvm::StringRef(*content).split(lines, '\n', -1, /*KeepEmpty=*/false);
for(llvm::StringRef line: lines) {
line = line.trim();
if(line.empty() || line.front() != '"') {
continue;
}
llvm::SmallVector<const char*, 256> args;
llvm::BumpPtrAllocator allocator;
llvm::StringSaver saver(allocator);
llvm::cl::TokenizeGNUCommandLine(line, saver, args);
using namespace std::string_view_literals;
if(args.size() < 2 || args[1] != "-cc1"sv) {
continue;
}
for(auto arg: args) {
if(arg == "-###"sv) {
continue;
}
result.emplace_back(params.callback(arg));
}
}
}
return result;
}
std::vector<const char*> query_msvc_toolchain(const QueryParams& params) {
auto arguments = params.arguments;
llvm::SmallVector<const char*, 256> query_arguments;
query_arguments.emplace_back(arguments.consume_front());
/// When clang in cl mode, the target will be set to windows-msvc automatically.
/// We don't need to add extra flag.
query_arguments.emplace_back("--driver-mode=cl");
query_arguments.append(arguments.begin(), arguments.end());
std::vector<const char*> result;
query_driver(query_arguments, [&](const char* driver, llvm::ArrayRef<const char*> cc) {
result.emplace_back(params.callback(driver));
for(auto c: cc) {
result.emplace_back(params.callback(c));
}
});
return result;
}
std::vector<const char*> query_nvcc_toolchain(const QueryParams& params);
} // namespace clice::toolchain

6
src/Feature/Diagnostic.cpp
View File

@@ -32,9 +32,9 @@ json::Value diagnostics(PositionEncodingKind kind, PathMapping mapping, Compilat
if(level == DiagnosticLevel::Note || level == DiagnosticLevel::Remark) {
/// FIXME: figure out why it may be invalid.
if(fid.isInvalid()) {
LOGGING_INFO("code: {}, message: {}",
raw_diagnostic.id.diagnostic_code(),
raw_diagnostic.message);
LOG_INFO("code: {}, message: {}",
raw_diagnostic.id.diagnostic_code(),
raw_diagnostic.message);
continue;
}

2
src/Feature/Formatting.cpp
View File

@@ -41,7 +41,7 @@ std::vector<proto::TextEdit> document_format(llvm::StringRef file,
range ? tooling::Range(range->begin, range->length()) : tooling::Range(0, content.size());
auto replacements = format(file, content, selection);
if(!replacements) {
LOGGING_INFO("Fail to format for {}\n{}", file, replacements.error());
LOG_INFO("Fail to format for {}\n{}", file, replacements.error());
return edits;
}

62
src/Server/Document.cpp
View File

@@ -12,14 +12,14 @@ void Server::load_cache_info() {
auto path = path::join(config.project.cache_dir, "cache.json");
auto file = llvm::MemoryBuffer::getFile(path);
if(!file) {
LOGGING_WARN("Fail to load cache info, because: {}", file.getError());
LOG_WARN("Fail to load cache info, because: {}", file.getError());
return;
}
llvm::StringRef content = file.get()->getBuffer();
auto json = json::parse(content);
if(!json) {
LOGGING_WARN("Fail to load cache info, invalid json: {}", json.takeError());
LOG_WARN("Fail to load cache info, invalid json: {}", json.takeError());
return;
}
@@ -30,7 +30,7 @@ void Server::load_cache_info() {
auto version = object->getString("version");
if(!version) {
LOGGING_INFO("Fail to load cache info, the cache info is outdated");
LOG_INFO("Fail to load cache info, the cache info is outdated");
return;
}
@@ -75,7 +75,7 @@ void Server::load_cache_info() {
}
}
LOGGING_INFO("Load cache info successfully");
LOG_INFO("Load cache info successfully");
}
void Server::save_cache_info() {
@@ -105,20 +105,20 @@ void Server::save_cache_info() {
llvm::SmallString<128> temp_path;
if(auto error = llvm::sys::fs::createTemporaryFile("cache", "json", temp_path)) {
LOGGING_WARN("Fail to create temporary file for cache info: {}", error.message());
LOG_WARN("Fail to create temporary file for cache info: {}", error.message());
return;
}
auto clean_up = llvm::make_scope_exit([&temp_path]() {
if(auto errc = llvm::sys::fs::remove(temp_path)) {
LOGGING_WARN("Fail to remove temporary file: {}", errc.message());
LOG_WARN("Fail to remove temporary file: {}", errc.message());
}
});
std::error_code EC;
llvm::raw_fd_ostream os(temp_path, EC, llvm::sys::fs::OF_None);
if(EC) {
LOGGING_WARN("Fail to open temporary file for writing: {}", EC.message());
LOG_WARN("Fail to open temporary file for writing: {}", EC.message());
return;
}
@@ -127,24 +127,26 @@ void Server::save_cache_info() {
os.close();
if(os.has_error()) {
LOGGING_WARN("Fail to write cache info to temporary file");
LOG_WARN("Fail to write cache info to temporary file");
return;
}
if(auto error = llvm::sys::fs::rename(temp_path, final_path)) {
LOGGING_WARN("Fail to rename temporary file to final cache file: {}", error.message());
LOG_WARN("Fail to rename temporary file to final cache file: {}", error.message());
return;
}
clean_up.release();
LOGGING_INFO("Save cache info successfully");
LOG_INFO("Save cache info successfully");
}
namespace {
bool
check_pch_update(llvm::StringRef content, std::uint32_t bound, LookupInfo& info, PCHInfo& pch) {
bool check_pch_update(llvm::StringRef content,
std::uint32_t bound,
CompilationContext& info,
PCHInfo& pch) {
if(content.substr(0, bound) != pch.preamble) {
return true;
}
@@ -168,7 +170,7 @@ bool
}
/// The actual PCH build task.
async::Task<bool> build_pch_task(LookupInfo& info,
async::Task<bool> build_pch_task(CompilationContext& info,
std::string cache_dir,
std::shared_ptr<OpenFile> open_file,
std::string path,
@@ -178,7 +180,7 @@ async::Task<bool> build_pch_task(LookupInfo& info,
if(!fs::exists(cache_dir)) {
auto error = fs::create_directories(cache_dir);
if(error) {
LOGGING_WARN("Fail to create directory for PCH building: {}", cache_dir);
LOG_WARN("Fail to create directory for PCH building: {}", cache_dir);
co_return false;
}
}
@@ -189,6 +191,7 @@ async::Task<bool> build_pch_task(LookupInfo& info,
CompilationParams params;
params.kind = CompilationUnit::Preamble;
params.output_file = path::join(cache_dir, path::filename(path) + ".pch");
params.arguments_from_database = true;
params.arguments = std::move(info.arguments);
params.diagnostics = diagnostics;
params.add_remapped_file(path, content, bound);
@@ -199,7 +202,7 @@ async::Task<bool> build_pch_task(LookupInfo& info,
command += argument;
}
LOGGING_INFO("Start building PCH for {}, command: [{}]", path, command);
LOG_INFO("Start building PCH for {}, command: [{}]", path, command);
command.clear();
PCHInfo pch;
@@ -220,14 +223,14 @@ async::Task<bool> build_pch_task(LookupInfo& info,
});
if(!success) {
LOGGING_WARN("Building PCH fails for {}, Because: {}", path, message);
LOG_WARN("Building PCH fails for {}, Because: {}", path, message);
for(auto& diagnostic: *diagnostics) {
LOGGING_WARN("{}", diagnostic.message);
LOG_WARN("{}", diagnostic.message);
}
co_return false;
}
LOGGING_INFO("Building PCH successfully for {}", path);
LOG_INFO("Building PCH successfully for {}", path);
/// Update the built PCH info.
open_file->pch = std::move(pch);
@@ -245,7 +248,7 @@ async::Task<bool> build_pch_task(LookupInfo& info,
async::Task<bool> Server::build_pch(std::string file, std::string content) {
CommandOptions options;
options.resource_dir = true;
options.query_driver = true;
options.query_toolchain = true;
auto info = database.lookup(file, options);
auto bound = compute_preamble_bound(content);
@@ -254,7 +257,7 @@ async::Task<bool> Server::build_pch(std::string file, std::string content) {
/// Check update ...
if(open_file->pch && !check_pch_update(content, bound, info, *open_file->pch)) {
/// If not need update, return directly.
LOGGING_INFO("PCH is already up-to-date for {}", file);
LOG_INFO("PCH is already up-to-date for {}", file);
co_return true;
}
@@ -263,12 +266,12 @@ async::Task<bool> Server::build_pch(std::string file, std::string content) {
if(!task.empty()) {
if(task.finished()) {
task.release().destroy();
LOGGING_INFO("Release old pch task!");
LOG_INFO("Release old pch task!");
} else {
task.cancel();
task.dispose();
}
LOGGING_INFO("Cancel old PCH building task!");
LOG_INFO("Cancel old PCH building task!");
}
/// Schedule the new building task.
@@ -304,15 +307,16 @@ async::Task<> Server::build_ast(std::string path, std::string content) {
auto pch = file->pch;
if(!pch) {
LOGGING_FATAL("Expected PCH built at this point");
LOG_FATAL("Expected PCH built at this point");
}
CommandOptions options;
options.resource_dir = true;
options.query_driver = true;
options.query_toolchain = true;
CompilationParams params;
params.kind = CompilationUnit::Content;
params.arguments_from_database = true;
params.arguments = database.lookup(path, options).arguments;
params.add_remapped_file(path, content);
params.pch = {pch->path, pch->preamble.size()};
@@ -324,9 +328,9 @@ async::Task<> Server::build_ast(std::string path, std::string content) {
auto ast = co_await async::submit([&] { return compile(params); });
if(!ast) {
/// FIXME: Fails needs cancel waiting tasks.
LOGGING_WARN("Building AST fails for {}, Beacuse: {}", path, ast.error());
LOG_ERROR("Building AST fails for {}, Beacuse: {}", path, ast.error());
for(auto& diagnostic: *file->diagnostics) {
LOGGING_WARN("{}", diagnostic.message);
LOG_ERROR("{}", diagnostic.message);
}
co_return;
}
@@ -349,7 +353,7 @@ async::Task<> Server::build_ast(std::string path, std::string content) {
/// Dispose the task so that it will destroyed when task complete.
file->ast_build_task.dispose();
LOGGING_INFO("Building AST successfully for {}", path);
LOG_INFO("Building AST successfully for {}", path);
}
async::Task<std::shared_ptr<OpenFile>> Server::add_document(std::string path, std::string content) {
@@ -363,12 +367,12 @@ async::Task<std::shared_ptr<OpenFile>> Server::add_document(std::string path, st
if(!task.empty()) {
if(task.finished()) {
task.release().destroy();
LOGGING_INFO("Release old AST building Task!");
LOG_INFO("Release old AST building Task!");
} else {
task.cancel();
task.dispose();
}
LOGGING_INFO("Cancel old AST building Task!");
LOG_INFO("Cancel old AST building Task!");
}
/// Create and schedule a new task.

2
src/Server/Feature.cpp
View File

@@ -31,6 +31,7 @@ auto Server::on_completion(proto::CompletionParams params) -> Result {
/// Set compilation params ... .
CompilationParams params;
params.kind = CompilationUnit::Completion;
params.arguments_from_database = true;
params.arguments = database.lookup(path).arguments;
params.add_remapped_file(path, content);
params.pch = {pch->path, pch->preamble.size()};
@@ -88,6 +89,7 @@ async::Task<json::Value> Server::on_signature_help(proto::SignatureHelpParams pa
/// Set compilation params ... .
CompilationParams params;
params.kind = CompilationUnit::Completion;
params.arguments_from_database = true;
params.arguments = database.lookup(path, options).arguments;
params.add_remapped_file(path, content);
params.pch = {pch->path, pch->preamble.size()};

23
src/Server/Indexer.cpp
View File

@@ -10,12 +10,13 @@ namespace clice {
async::Task<> Indexer::index(llvm::StringRef path) {
CompilationParams params;
params.kind = CompilationUnit::Indexing;
params.arguments_from_database = true;
params.arguments = database.lookup(path).arguments;
auto path_id = project_index.path_pool.path_id(path);
auto& merged_index = get_index(path_id);
if(!merged_index.need_update(project_index.path_pool.paths)) {
LOGGING_INFO("Check update for {}, not need to update", path);
LOG_INFO("Check update for {}, not need to update", path);
co_return;
}
@@ -25,7 +26,7 @@ async::Task<> Indexer::index(llvm::StringRef path) {
auto tu_index = co_await async::submit([&]() -> std::optional<index::TUIndex> {
auto unit = compile(params);
if(!unit) {
LOGGING_INFO("Fail to index for {}, because: {}", path, unit.error());
LOG_INFO("Fail to index for {}, because: {}", path, unit.error());
return std::nullopt;
}
@@ -55,7 +56,7 @@ async::Task<> Indexer::index(llvm::StringRef path) {
std::move(tu_index->graph.locations),
tu_index->main_file_index);
LOGGING_INFO("Successfully index {}", path);
LOG_INFO("Successfully index {}", path);
}
async::Task<> Indexer::schedule_next() {
@@ -107,9 +108,9 @@ void Indexer::load_from_disk() {
if(auto content = fs::read(output_path); content && !content->empty()) {
/// FIXME: from should return a expected ...
project_index = index::ProjectIndex::from(content->data());
LOGGING_INFO("Load project index form {} successfully", output_path);
LOG_INFO("Load project index form {} successfully", output_path);
} else {
LOGGING_INFO("Fail to load project index form {}", output_path);
LOG_INFO("Fail to load project index form {}", output_path);
}
/// FIXME: check indices update ....
@@ -117,9 +118,7 @@ void Indexer::load_from_disk() {
void Indexer::save_to_disk() {
if(auto err = fs::create_directories(config.project.index_dir)) {
LOGGING_WARN("Fail to create index output dir: {}, because: {}",
config.project.index_dir,
err);
LOG_WARN("Fail to create index output dir: {}, because: {}", config.project.index_dir, err);
return;
}
@@ -139,7 +138,7 @@ void Indexer::save_to_disk() {
std::error_code err;
llvm::raw_fd_ostream os(output_path, err, fs::CreationDisposition::CD_CreateAlways);
if(err) {
LOGGING_INFO("Fail to create output index file: {}, because: {}", output_path, err);
LOG_INFO("Fail to create output index file: {}, because: {}", output_path, err);
continue;
}
@@ -147,7 +146,7 @@ void Indexer::save_to_disk() {
auto opath_id = project_index.path_pool.path_id(output_path);
project_index.indices.try_emplace(path_id, opath_id);
LOGGING_INFO("Successfully save index for {} to {}", path, output_path);
LOG_INFO("Successfully save index for {} to {}", path, output_path);
}
}
@@ -156,12 +155,12 @@ void Indexer::save_to_disk() {
std::error_code err;
llvm::raw_fd_ostream os(output_path, err, fs::CreationDisposition::CD_CreateAlways);
if(err) {
LOGGING_INFO("Fail to create output index file: {}, because: {}", output_path, err);
LOG_INFO("Fail to create output index file: {}, because: {}", output_path, err);
return;
}
project_index.serialize(os);
LOGGING_INFO("Successfully save project index to {}", output_path);
LOG_INFO("Successfully save project index to {}", output_path);
}
auto Indexer::lookup(llvm::StringRef path, std::uint32_t offset, RelationKind kind) -> Result {

18
src/Server/Lifecycle.cpp
View File

@@ -3,9 +3,9 @@
namespace clice {
async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
LOGGING_INFO("Initialize from client: {}, version: {}",
params.clientInfo.name,
params.clientInfo.version);
LOG_INFO("Initialize from client: {}, version: {}",
params.clientInfo.name,
params.clientInfo.version);
/// FIXME: adjust position encoding.
kind = PositionEncodingKind::UTF16;
@@ -17,15 +17,15 @@ async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
return *params.rootUri;
}
LOGGING_FATAL("The client should provide one workspace folder or rootUri at least!");
LOG_FATAL("The client should provide one workspace folder or rootUri at least!");
})());
/// Initialize configuration.
if(auto result = config.parse(workspace)) {
LOGGING_INFO("Config initialized successfully: {0:4}", json::serialize(config));
LOG_INFO("Config initialized successfully: {0:4}", json::serialize(config));
} else {
LOGGING_WARN("Fail to load config, because: {0}", result.error());
LOGGING_INFO("Use default config: {0:4}", json::serialize(config));
LOG_WARN("Fail to load config, because: {0}", result.error());
LOG_INFO("Use default config: {0:4}", json::serialize(config));
}
if(!config.project.logging_dir.empty()) {
@@ -36,7 +36,9 @@ async::Task<json::Value> Server::on_initialize(proto::InitializeParams params) {
opening_files.set_capability(config.project.max_active_file);
/// Load compile commands.json
database.load_compile_database(config.project.compile_commands_dirs, workspace);
for(auto& dir: config.project.compile_commands_dirs) {
database.load_compile_database(path::join(dir, "compile_commands.json"));
}
/// Load cache info.
load_cache_info();

14
src/Server/Server.cpp
View File

@@ -124,7 +124,7 @@ Server::Server() : indexer(database, config, kind) {
async::Task<> Server::on_receive(json::Value value) {
auto object = value.getAsObject();
if(!object) [[unlikely]] {
LOGGING_FATAL("Invalid LSP message, not an object: {}", value);
LOG_FATAL("Invalid LSP message, not an object: {}", value);
}
/// If the json object has an `id`, it's a request,
@@ -135,7 +135,7 @@ async::Task<> Server::on_receive(json::Value value) {
if(auto result = object->getString("method")) {
method = *result;
} else [[unlikely]] {
LOGGING_WARN("Invalid LSP message, method not found: {}", value);
LOG_WARN("Invalid LSP message, method not found: {}", value);
if(id) {
co_await response(std::move(*id),
proto::ErrorCodes::InvalidRequest,
@@ -152,7 +152,7 @@ async::Task<> Server::on_receive(json::Value value) {
/// Handle request and notification separately.
auto it = callbacks.find(method);
if(it == callbacks.end()) {
LOGGING_INFO("Ignore unhandled method: {}", method);
LOG_INFO("Ignore unhandled method: {}", method);
co_return;
}
@@ -160,24 +160,24 @@ async::Task<> Server::on_receive(json::Value value) {
auto current_id = client_request_id++;
auto start_time = std::chrono::steady_clock::now();
LOGGING_INFO("<-- Handling request: {}({})", method, current_id);
LOG_INFO("<-- Handling request: {}({})", method, current_id);
auto result = co_await it->second(*this, std::move(params));
co_await response(std::move(*id), std::move(result));
auto end_time = std::chrono::steady_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
LOGGING_INFO("--> Handled request: {}({}) {}ms", method, current_id, duration.count());
LOG_INFO("--> Handled request: {}({}) {}ms", method, current_id, duration.count());
} else {
auto start_time = std::chrono::steady_clock::now();
LOGGING_INFO("<-- Handling notification: {}", method);
LOG_INFO("<-- Handling notification: {}", method);
auto result = co_await it->second(*this, std::move(params));
auto end_time = std::chrono::steady_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
LOGGING_INFO("--> Handled notification: {} {}ms", method, duration.count());
LOG_INFO("--> Handled notification: {} {}ms", method, duration.count());
}
co_return;