caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Basic support C++20 named module. (#12)

Browse Source
This commit is contained in:
ykiko
2024-12-14 13:40:13 +08:00
committed by GitHub
parent 7d42b4e1ec
commit 5326480cd6
30 changed files with 1082 additions and 593 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
include/Basic/URI.h
View File

@@ -1,7 +1,8 @@
#pragma once
#include <string>
#include <Support/Error.h>
#include "Support/Support.h"
namespace clice {
@@ -14,19 +15,34 @@ public:
bool operator== (const URI&) const = default;
/// Returns decoded scheme e.g. "https"
llvm::StringRef scheme() const { return m_scheme; }
/// Returns decoded authority e.g. "reviews.lvm.org"
llvm::StringRef authority() const { return m_authority; }
/// Returns decoded body e.g. "/D41946"
llvm::StringRef body() const { return m_body; }
/// Construct a URI object from the given file path.
static URI from(llvm::StringRef file);
/// Parse the given URI string to create a URI object.
static llvm::Expected<URI> parse(llvm::StringRef content);
/// Same as `parse`, but will crash if failed.
static std::string resolve(llvm::StringRef content);
/// Returns decoded scheme e.g. "https"
llvm::StringRef scheme() const {
return m_scheme;
}
/// Returns decoded authority e.g. "reviews.llvm.org"
llvm::StringRef authority() const {
return m_authority;
}
/// Returns decoded body e.g. "/D41946"
llvm::StringRef body() const {
return m_body;
}
std::string toString() const {
return std::format("{}://{}{}", m_scheme, m_authority, m_body);
}
private:
std::string m_scheme;
std::string m_authority;

116
include/Compiler/Compiler.h
View File

@@ -5,21 +5,26 @@
#include <Compiler/Resolver.h>
#include <Support/Error.h>
#include "Basic/Location.h"
#include "llvm/ADT/StringSet.h"
namespace clice {
struct CompilationParams;
/// All information about AST.
class ASTInfo {
public:
ASTInfo() = default;
ASTInfo(std::unique_ptr<clang::ASTFrontendAction> action,
ASTInfo(std::unique_ptr<clang::FrontendAction> action,
std::unique_ptr<clang::CompilerInstance> instance,
std::unique_ptr<clang::syntax::TokenBuffer> tokBuf,
llvm::DenseMap<clang::FileID, Directive>&& directives) :
action(std::move(action)), instance(std::move(instance)), m_TokBuf(std::move(tokBuf)),
m_Directives(std::move(directives)) {
m_Resolver = std::make_unique<TemplateResolver>(this->instance->getSema());
llvm::DenseMap<clang::FileID, Directive>&& directives,
std::vector<std::string> deps) :
action(std::move(action)), m_Instance(std::move(instance)), m_TokBuf(std::move(tokBuf)),
m_Directives(std::move(directives)), m_Deps(std::move(deps)) {
m_Resolver = std::make_unique<TemplateResolver>(this->m_Instance->getSema());
}
ASTInfo(const ASTInfo&) = delete;
@@ -33,32 +38,32 @@ public:
}
}
clang::Sema& sema() {
return instance->getSema();
auto& sema() {
return m_Instance->getSema();
}
clang::ASTContext& context() {
return instance->getASTContext();
auto& context() {
return m_Instance->getASTContext();
}
clang::SourceManager& srcMgr() {
return instance->getSourceManager();
auto& srcMgr() {
return m_Instance->getSourceManager();
}
clang::Preprocessor& pp() {
return instance->getPreprocessor();
auto& pp() {
return m_Instance->getPreprocessor();
}
clang::TranslationUnitDecl* tu() {
return instance->getASTContext().getTranslationUnitDecl();
return m_Instance->getASTContext().getTranslationUnitDecl();
}
clang::syntax::TokenBuffer& tokBuf() {
auto& tokBuf() {
assert(m_TokBuf && "Token buffer is not available");
return *m_TokBuf;
}
TemplateResolver& resolver() {
auto& resolver() {
return *m_Resolver;
}
@@ -66,13 +71,21 @@ public:
return m_Directives;
}
Directive& directive(clang::FileID id) {
auto& directive(clang::FileID id) {
return m_Directives[id];
}
auto& deps() {
return m_Deps;
}
auto& instance() {
return *m_Instance;
}
/// Get the length of the token at the given location.
auto getTokenLength(clang::SourceLocation loc) {
return clang::Lexer::MeasureTokenLength(loc, srcMgr(), instance->getLangOpts());
return clang::Lexer::MeasureTokenLength(loc, srcMgr(), m_Instance->getLangOpts());
}
/// Get the spelling of the token at the given location.
@@ -85,13 +98,22 @@ public:
}
private:
std::unique_ptr<clang::ASTFrontendAction> action;
std::unique_ptr<clang::CompilerInstance> instance;
std::unique_ptr<clang::FrontendAction> action;
std::unique_ptr<clang::CompilerInstance> m_Instance;
std::unique_ptr<clang::syntax::TokenBuffer> m_TokBuf;
std::unique_ptr<TemplateResolver> m_Resolver;
llvm::DenseMap<clang::FileID, Directive> m_Directives;
std::vector<std::string> m_Deps;
};
/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler.
/// Note this function will not check whether we need to update the PCH or PCM, caller should check
/// their reusability and update in time.
llvm::Expected<ASTInfo> compile(CompilationParams& params);
/// Run code completion at the given location.
llvm::Expected<ASTInfo> compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer);
struct PCHInfo {
/// PCM file path.
std::string path;
@@ -115,20 +137,44 @@ struct PCHInfo {
bool needUpdate(llvm::StringRef content);
};
struct PCMInfo {
/// Build PCH from given file path and content.
llvm::Expected<ASTInfo> compile(CompilationParams& params, PCHInfo& out);
struct ModuleInfo {
/// Whether this module is an interface unit.
/// i.e. has export module declaration.
bool isInterfaceUnit = false;
/// Module name.
std::string name;
/// Dependent modules of this module.
std::vector<std::string> mods;
};
/// Run the preprocessor to scan the given module unit to
/// collect its module name and dependencies.
llvm::Expected<ModuleInfo> scanModule(CompilationParams& params);
inherited_struct(PCMInfo, ModuleInfo) {
/// PCM file path.
std::string path;
/// Source file path.
std::string srcPath;
/// Module name.
std::string name;
/// Files involved in building this PCM(not include module).
std::vector<std::string> deps;
bool needUpdate();
bool needUpdate() {
return true;
}
};
struct CompliationParams {
/// Build PCM from given file path and content.
llvm::Expected<ASTInfo> compile(CompilationParams& params, PCMInfo& out);
struct CompilationParams {
/// Source file content.
llvm::StringRef content;
@@ -164,7 +210,7 @@ struct CompliationParams {
clang::PreambleBounds pchBounds = {0, false};
/// Information about reuse PCM(name, path).
llvm::SmallVector<std::pair<std::string, std::string>> pcms;
llvm::StringMap<std::string> pcms;
/// Code completion file:line:column.
llvm::StringRef file = "";
@@ -177,22 +223,10 @@ struct CompliationParams {
}
void addPCM(const PCMInfo& info) {
pcms.emplace_back(info.name, info.path);
assert((!pcms.contains(info.name) || pcms[info.name] == info.path) &&
"Add a different PCM with the same name");
pcms[info.name] = info.path;
}
};
/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler.
/// Note this function will not check whether we need to update the PCH or PCM, caller should check
/// their reusability and update in time.
llvm::Expected<ASTInfo> compile(CompliationParams& params);
/// Build PCH from given file path and content.
llvm::Expected<ASTInfo> compile(CompliationParams& params, PCHInfo& out);
/// Build PCM from given file path and content.
llvm::Expected<ASTInfo> compile(CompliationParams& params, PCMInfo& out);
/// Run code completion at the given location.
llvm::Expected<ASTInfo> compile(CompliationParams& params, clang::CodeCompleteConsumer* consumer);
} // namespace clice

4
include/Feature/CodeCompletion.h
View File

@@ -4,7 +4,7 @@
#include <Support/JSON.h>
namespace clice {
struct CompliationParams;
struct CompilationParams;
}
namespace clice::proto {
@@ -117,7 +117,7 @@ json::Value capability(json::Value clientCapabilities);
/// Run code completion in given file and location. `compiler` should be
/// set properly if any PCH or PCM is needed. Each completion requires a
/// new compiler instance.
proto::CompletionResult codeCompletion(CompliationParams& compliation,
proto::CompletionResult codeCompletion(CompilationParams& compliation,
uint32_t line,
uint32_t column,
llvm::StringRef file,

14
include/Server/Async.h
View File

@@ -35,7 +35,7 @@ extern uv_loop_t* loop;
template <typename T, typename U>
T& uv_cast(U* u) {
assert(u && u->data && "uv_cast: invalid uv handle");
return *static_cast<T*>(u->data);
return *static_cast<std::remove_cvref_t<T>*>(u->data);
}
using Callback = llvm::unique_function<promise<void>(json::Value)>;
@@ -44,7 +44,17 @@ void start_server(Callback callback);
void start_server(Callback callback, const char* ip, unsigned int port);
void write(json::Value id, json::Value result);
/// Send a request to the client.
void request(llvm::StringRef method, json::Value params);
/// Send a notification to the client.
void notify(llvm::StringRef method, json::Value params);
/// Send a response to the client.
void response(json::Value id, json::Value result);
/// Send an register capability to the client.
void registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions);
template <typename Value>
struct result {

24
include/Server/Command.h
View File

@@ -1,24 +0,0 @@
#pragma once
#include "Support/Support.h"
#include "clang/Tooling/CompilationDatabase.h"
namespace clice {
class CommandManager {
public:
void update(llvm::StringRef dir);
/// Return the commands of first meet file.
llvm::StringRef lookupFirst(llvm::StringRef file);
llvm::ArrayRef<std::string> lookup(llvm::StringRef file);
private:
/// CDB file -> file -> [commands]
using Commands = std::vector<std::string>;
using CDB = llvm::StringMap<Commands>;
llvm::StringMap<CDB> CDBs;
};
} // namespace clice

59
include/Server/Protocol.h
View File

@@ -33,7 +33,64 @@ struct ClientInfo {
string version;
};
struct ClientCapabilities {};
struct DidChangeWatchedFilesClientCapabilities {
/// Did change watched files notification supports dynamic registration.
/// Please note that the current protocol doesn't support static
/// configuration for file changes from the server side.
bool dynamicRegistration = false;
};
struct ClientCapabilities {
/// Workspace specific client capabilities.
struct {
/// Capabilities specific to the `workspace/didChangeWatchedFiles`
/// notification.
DidChangeWatchedFilesClientCapabilities didChangeWatchedFiles;
} workspace;
};
struct FileSystemWatcher {
/// The glob pattern to watch.
std::string globPattern;
enum WatchKind {
Create = 1,
Change = 2,
Delete = 4,
};
/// The kind of events of interest.
int kind = WatchKind::Create | WatchKind::Change | WatchKind::Delete;
};
struct DidChangeWatchedFilesRegistrationOptions {
/// The watchers to register.
std::vector<FileSystemWatcher> watchers;
};
enum class FileChangeType {
/// The file got created.
Created = 1,
/// The file got changed.
Changed = 2,
/// The file got deleted.
Deleted = 3,
};
struct FileEvent {
/// The file's URI.
string uri;
/// The change type.
FileChangeType type;
};
struct DidChangeWatchedFilesParams {
/// The actual file events.
std::vector<FileEvent> changes;
};
struct Workplace {
/// The associated URI for this workspace folder.

91
include/Server/Scheduler.h
View File

@@ -3,7 +3,6 @@
#include <deque>
#include "Server/Async.h"
#include "Server/Command.h"
#include "Server/Protocol.h"
#include "Server/Trace.h"
#include "Compiler/Compiler.h"
@@ -45,88 +44,38 @@ struct llvm::DenseMapInfo<clice::SourceContext> {
namespace clice {
struct File2 {
bool isIdle = true;
llvm::DenseMap<SourceContext, ASTInfo> contexts;
};
struct File;
struct Task {
/// Whether this task is a build task.
bool isBuild = false;
/// The coroutine handle of this task.
std::coroutine_handle<> waiting;
};
struct File {
bool isIdle = true;
std::string content;
/// The compiler instance of this file.
ASTInfo compiler;
std::deque<Task> waitings;
};
/// Responsible for manage the files and schedule the tasks.
class Scheduler {
private:
async::promise<void> updatePCH(llvm::StringRef path,
llvm::StringRef content,
llvm::StringRef command);
async::promise<> updatePCH(CompilationParams& params, class Synchronizer& sync);
async::promise<void> updatePCM() {
co_return;
}
/// Clang requires all direct and indirect dependent modules to be added during module building.
/// This function adds the dependencies of the given module to the compilation parameters.
/// Note: It is assumed that all dependent modules have already been built.
llvm::Error addModuleDeps(CompilationParams& params, const ModuleInfo& moduleInfo) const;
async::promise<void> buildAST(llvm::StringRef path, llvm::StringRef content);
async::promise<> updatePCM(llvm::StringRef name, class Synchronizer& sync);
public:
async::promise<void> add(llvm::StringRef path, llvm::StringRef content);
async::promise<> update(llvm::StringRef filename,
llvm::StringRef content,
class Synchronizer& sync);
async::promise<void> update(llvm::StringRef path, llvm::StringRef content);
/// Load all Information about PCHs and PCMs from disk.
void loadFromDisk();
async::promise<void> save(llvm::StringRef path);
/// Save all Information about PCHs and PCMs to disk.
/// So that we can reuse them next time.
void saveToDisk() const;
async::promise<void> close(llvm::StringRef path);
async::promise<proto::CompletionResult> codeComplete(llvm::StringRef path,
unsigned int line,
unsigned int column);
/// Schedule a task for a file. If the file is building, the task will be
/// appended to the task list of the file and wait for the building to finish.
/// Otherwise, the task will be executed immediately.
template <typename Task>
auto schedule(llvm::StringRef path, Task&& task)
-> async::promise<decltype(task(std::declval<ASTInfo&>()))> {
auto& file = files[path];
if(!file.isIdle) {
co_await async::suspend([&](auto handle) {
file.waitings.push_back({.isBuild = false, .waiting = handle});
});
}
file.isIdle = false;
auto& compiler = file.compiler;
auto result = co_await async::schedule_task([&task, &compiler] { return task(compiler); });
if(!file.waitings.empty()) {
auto task = std::move(file.waitings.front());
async::schedule(task.waiting);
file.waitings.pop_front();
}
file.isIdle = true;
co_return result;
}
struct File {};
private:
/// [file name] -> [PCHInfo]
llvm::StringMap<PCHInfo> pchs;
llvm::StringMap<File> files;
CommandManager cmdMgr;
/// [module name] -> [PCMInfo]
llvm::StringMap<PCMInfo> pcms;
};
} // namespace clice

8
include/Server/Server.h
View File

@@ -5,6 +5,7 @@
#include "Server/Config.h"
#include "Server/Logger.h"
#include "Server/Scheduler.h"
#include "Server/Synchronizer.h"
#include "Server/Protocol.h"
#include "Support/Support.h"
@@ -127,6 +128,12 @@ private:
async::promise<void> onRangeFormatting(json::Value id,
const proto::DocumentRangeFormattingParams& params);
/// ============================================================================
/// Workspace Features
/// ============================================================================
async::promise<void> onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params);
/// ============================================================================
/// Extension
/// ============================================================================
@@ -139,6 +146,7 @@ private:
private:
Scheduler scheduler;
Synchronizer synchronizer;
llvm::StringMap<onRequest> requests;
llvm::StringMap<onNotification> notifications;
};

43
include/Server/Synchronizer.h Normal file
View File

@@ -0,0 +1,43 @@
#pragma once
#include "Support/Support.h"
namespace clice {
/// Responsible for synchronizing changes to the CDB file,
/// including updating compile commands and the module map.
class Synchronizer {
public:
/// FIXME: The CDB file can be very large, reaching the size of
/// several gigabytes (GB). Therefore, it's better for this function
/// to be an asynchronous function.
/// Update the compile commands.
void sync(llvm::StringRef file);
/// Update the module map for active file.
void sync(llvm::StringRef name, llvm::StringRef path);
/// Lookup the compile commands of the given file.
llvm::StringRef lookup(llvm::StringRef file) const;
/// Lookup the module interface unit file path of the given module name.
llvm::StringRef map(llvm::StringRef name) const;
private:
/// FIXME: currently we assume that a file only occurs once in the CDB.
/// This is not always correct, but it is enough for now.
/// A map between file path and compile commands.
llvm::StringMap<std::string> commands;
/// For C++20 module, we only can got dependent module name
/// in source context. But we need dependent module file path
/// to build PCM. So we will scan(preprocess) all project files
/// to build a module map between module name and module file path.
/// **Note that** this only includes module interface unit, for module
/// implementation unit, the scan could be delayed until compiling it.
llvm::StringMap<std::string> moduleMap;
};
} // namespace clice

8
include/Support/Struct.h
View File

@@ -199,7 +199,7 @@ struct Inheritance : Ts... {};
/// Use to define a reflectable struct with inheritance.
#define inherited_struct(name, ...) \
struct name##Body; \
using name = clice::refl::Inheritance<__VA_ARGS__, name##Body>; \
using name = clice::refl::Inheritance<__VA_ARGS__, name##Body>; \
struct name##Body
template <typename... Ts>
@@ -216,7 +216,11 @@ struct Struct<Inheritance<Ts...>> {
template <typename Object>
constexpr static auto collcet_members(Object&& object) {
return std::tuple_cat(Struct<Ts>::collcet_members(static_cast<Ts&>(object))...);
if constexpr(std::is_const_v<std::remove_reference_t<Object>>) {
return std::tuple_cat(Struct<Ts>::collcet_members(static_cast<const Ts&>(object))...);
} else {
return std::tuple_cat(Struct<Ts>::collcet_members(static_cast<Ts&>(object))...);
}
}
};

26
src/Basic/URI.cpp
View File

@@ -5,6 +5,8 @@
namespace clice {
namespace {
/// returns true if the scheme is valid according to RFC 3986.
bool isValidScheme(llvm::StringRef scheme) {
if(scheme.empty()) {
@@ -39,6 +41,30 @@ static std::string decodePercent(llvm::StringRef content) {
return result;
}
} // namespace
URI URI::from(llvm::StringRef file) {
if(!path::is_absolute(file)) {
std::terminate();
}
llvm::SmallString<128> path;
for(auto c: file) {
if(c == '\\') {
path.push_back('/');
} else if(std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '/') {
path.push_back(c);
} else {
path.push_back('%');
path.push_back(llvm::hexdigit(c >> 4));
path.push_back(llvm::hexdigit(c & 0xF));
}
}
return URI("file", "", path);
}
llvm::Expected<URI> URI::parse(llvm::StringRef content) {
URI result("", "", "");
llvm::StringRef uri = content;

4
src/Compiler/Command.cpp
View File

@@ -58,6 +58,10 @@ llvm::Error mangleCommand(llvm::StringRef command,
continue;
}
if(arg.starts_with("@CMakeFiles")) {
continue;
}
/// TODO: remove PCH.
out.push_back(arg.data());

229
src/Compiler/Compiler.cpp
View File

@@ -19,7 +19,7 @@ bool PCHInfo::needUpdate(llvm::StringRef content) {
namespace {
auto createInvocation(CompliationParams& params) {
auto createInvocation(CompilationParams& params) {
llvm::SmallString<1024> buffer;
llvm::SmallVector<const char*, 16> args;
@@ -45,7 +45,7 @@ auto createInvocation(CompliationParams& params) {
return invocation;
}
auto createInstance(CompliationParams& params) {
auto createInstance(CompilationParams& params) {
auto instance = std::make_unique<clang::CompilerInstance>();
instance->setInvocation(createInvocation(params));
@@ -64,10 +64,12 @@ auto createInstance(CompliationParams& params) {
assert(!instance->getPreprocessorOpts().RetainRemappedFileBuffers &&
"RetainRemappedFileBuffers should be false");
instance->getPreprocessorOpts().addRemappedFile(
params.srcPath,
llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath)
.release());
if(!params.content.empty()) {
instance->getPreprocessorOpts().addRemappedFile(
params.srcPath,
llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath)
.release());
}
for(auto& [file, content]: params.remappedFiles) {
instance->getPreprocessorOpts().addRemappedFile(
@@ -75,14 +77,19 @@ auto createInstance(CompliationParams& params) {
llvm::MemoryBuffer::getMemBufferCopy(content, file).release());
}
if(!instance->createTarget()) {
/// FIXME: add error handle here.
std::terminate();
}
return instance;
}
void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) {
void applyPreamble(clang::CompilerInstance& instance, CompilationParams& params) {
auto& PPOpts = instance.getPreprocessorOpts();
auto& pch = params.pch;
auto& bounds = params.pchBounds;
auto& pcms = params.pcms;
if(bounds.Size != 0) {
PPOpts.UsePredefines = false;
PPOpts.ImplicitPCHInclude = std::move(pch);
@@ -91,30 +98,34 @@ void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params)
PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH;
}
auto& pcms = params.pcms;
for(auto& [name, path]: pcms) {
auto& HSOpts = instance.getHeaderSearchOpts();
HSOpts.PrebuiltModuleFiles.try_emplace(std::move(name), std::move(path));
HSOpts.PrebuiltModuleFiles.try_emplace(name.str(), std::move(path));
}
}
llvm::Expected<ASTInfo> ExecuteAction(std::unique_ptr<clang::CompilerInstance> instance,
clang::frontend::ActionKind kind) {
std::unique_ptr<clang::ASTFrontendAction> action;
if(kind == clang::frontend::ActionKind::ParseSyntaxOnly) {
action = std::make_unique<clang::SyntaxOnlyAction>();
} else if(kind == clang::frontend::ActionKind::GeneratePCH) {
action = std::make_unique<clang::GeneratePCHAction>();
} else if(kind == clang::frontend::ActionKind::GenerateReducedModuleInterface) {
action = std::make_unique<clang::GenerateReducedModuleInterfaceAction>();
} else {
llvm::errs() << "Unsupported action kind\n";
std::terminate();
/// Execute given action with the on the given instance. `callback` is called after
/// `BeginSourceFile`. Beacuse `BeginSourceFile` may create new preprocessor.
llvm::Error ExecuteAction(clang::CompilerInstance& instance,
clang::FrontendAction& action,
auto&& callback) {
if(!action.BeginSourceFile(instance, instance.getFrontendOpts().Inputs[0])) {
return error("Failed to begin source file");
}
if(!instance->createTarget()) {
return error("Failed to create target");
callback();
if(auto error = action.Execute()) {
return error;
}
return llvm::Error::success();
}
llvm::Expected<ASTInfo> ExecuteAction(std::unique_ptr<clang::CompilerInstance> instance,
std::unique_ptr<clang::FrontendAction> action) {
if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) {
return error("Failed to begin source file");
}
@@ -129,12 +140,13 @@ llvm::Expected<ASTInfo> ExecuteAction(std::unique_ptr<clang::CompilerInstance> i
llvm::DenseMap<clang::FileID, Directive> directives;
Directive::attach(pp, directives);
std::optional<clang::syntax::TokenCollector> collector;
/// Collect tokens.
std::optional<clang::syntax::TokenCollector> tokCollector;
/// It is not necessary to collect tokens if we are running code completion.
/// And in fact will cause assertion failure.
if(!instance->hasCodeCompletionConsumer()) {
collector.emplace(pp);
tokCollector.emplace(pp);
}
if(auto error = action->Execute()) {
@@ -142,19 +154,23 @@ llvm::Expected<ASTInfo> ExecuteAction(std::unique_ptr<clang::CompilerInstance> i
}
std::unique_ptr<clang::syntax::TokenBuffer> tokBuf;
if(collector) {
tokBuf = std::make_unique<clang::syntax::TokenBuffer>(std::move(*collector).consume());
if(tokCollector) {
tokBuf = std::make_unique<clang::syntax::TokenBuffer>(std::move(*tokCollector).consume());
}
/// FIXME: getDependencies currently return ArrayRef<std::string>, which actually results in
/// extra copy. It would be great to avoid this copy.
return ASTInfo(std::move(action),
std::move(instance),
std::move(tokBuf),
std::move(directives));
std::move(directives),
{});
}
} // namespace
void CompliationParams::computeBounds(llvm::StringRef header) {
void CompilationParams::computeBounds(llvm::StringRef header) {
assert(!bounds.has_value() && "Bounds is already computed");
assert(!content.empty() && "Source content is required to compute bounds");
@@ -248,15 +264,101 @@ void CompliationParams::computeBounds(llvm::StringRef header) {
}
}
llvm::Expected<ASTInfo> compile(CompliationParams& params) {
/// Scan the module name. This will not run the preprocessor.
std::string scanModuleName(llvm::StringRef content) {
clang::LangOptions langOpts;
langOpts.Modules = true;
langOpts.CPlusPlus20 = true;
clang::Lexer lexer(clang::SourceLocation(),
langOpts,
content.begin(),
content.begin(),
content.end());
clang::Token token;
lexer.Lex(token);
while(!token.is(clang::tok::eof)) {
llvm::outs() << token.getName() << "\n";
if(token.is(clang::tok::kw_module)) {
lexer.Lex(token);
if(token.is(clang::tok::coloncolon)) {
lexer.Lex(token);
}
if(token.is(clang::tok::identifier)) {
return token.getIdentifierInfo()->getName().str();
}
}
lexer.Lex(token);
}
return "";
}
llvm::Expected<ModuleInfo> scanModule(CompilationParams& params) {
struct ModuleCollector : public clang::PPCallbacks {
ModuleInfo& info;
ModuleCollector(ModuleInfo& info) : info(info) {}
void moduleImport(clang::SourceLocation importLoc,
clang::ModuleIdPath path,
const clang::Module* imported) override {
assert(path.size() == 1);
info.mods.emplace_back(path[0].first->getName());
}
};
ModuleInfo info;
clang::PreprocessOnlyAction action;
auto instance = createInstance(params);
if(!action.BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) {
return error("Failed to begin source file");
}
auto& pp = instance->getPreprocessor();
pp.addPPCallbacks(std::make_unique<ModuleCollector>(info));
if(auto error = action.Execute()) {
return error;
}
if(pp.isInNamedModule()) {
info.isInterfaceUnit = pp.isInNamedInterfaceUnit();
info.name = pp.getNamedModuleName();
}
return info;
}
llvm::Expected<ASTInfo> compile(CompilationParams& params) {
auto instance = createInstance(params);
applyPreamble(*instance, params);
return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly);
return ExecuteAction(std::move(instance), std::make_unique<clang::SyntaxOnlyAction>());
}
llvm::Expected<ASTInfo> compile(CompliationParams& params, PCHInfo& out) {
llvm::Expected<ASTInfo> compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer) {
auto instance = createInstance(params);
/// Set options to run code completion.
instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str();
instance->getFrontendOpts().CodeCompletionAt.Line = params.line;
instance->getFrontendOpts().CodeCompletionAt.Column = params.column;
instance->setCodeCompletionConsumer(consumer);
applyPreamble(*instance, params);
return ExecuteAction(std::move(instance), std::make_unique<clang::SyntaxOnlyAction>());
}
llvm::Expected<ASTInfo> compile(CompilationParams& params, PCHInfo& out) {
assert(params.bounds.has_value() && "Preamble bounds is required to build PCH");
auto instance = createInstance(params);
@@ -268,25 +370,27 @@ llvm::Expected<ASTInfo> compile(CompliationParams& params, PCHInfo& out) {
instance->getPreprocessorOpts().GeneratePreamble = true;
instance->getLangOpts().CompilingPCH = true;
if(auto info = ExecuteAction(std::move(instance), clang::frontend::ActionKind::GeneratePCH)) {
out.path = params.outPath.str();
out.srcPath = params.srcPath.str();
auto& bounds = *params.bounds;
out.preamble = params.content.substr(0, bounds.Size).str();
if(bounds.PreambleEndsAtStartOfLine) {
out.preamble.append("@");
}
/// TODO: collect files involved in building this PCH.
return std::move(*info);
} else {
auto info = ExecuteAction(std::move(instance), std::make_unique<clang::GeneratePCHAction>());
if(!info) {
return info.takeError();
}
out.path = params.outPath.str();
out.srcPath = params.srcPath.str();
auto& bounds = *params.bounds;
out.preamble = params.content.substr(0, bounds.Size).str();
out.deps = info->deps();
if(bounds.PreambleEndsAtStartOfLine) {
out.preamble.append("@");
}
/// TODO: collect files involved in building this PCH.
return std::move(*info);
}
llvm::Expected<ASTInfo> compile(CompliationParams& params, PCMInfo& out) {
llvm::Expected<ASTInfo> compile(CompilationParams& params, PCMInfo& out) {
auto instance = createInstance(params);
/// Set options to generate PCM.
@@ -295,29 +399,26 @@ llvm::Expected<ASTInfo> compile(CompliationParams& params, PCMInfo& out) {
applyPreamble(*instance, params);
if(auto info = ExecuteAction(std::move(instance),
clang::frontend::ActionKind::GenerateReducedModuleInterface)) {
out.path = params.outPath.str();
out.name = info->context().getCurrentNamedModule()->Name;
return std::move(*info);
} else {
auto info = ExecuteAction(std::move(instance),
std::make_unique<clang::GenerateReducedModuleInterfaceAction>());
if(!info) {
return info.takeError();
}
}
llvm::Expected<ASTInfo> compile(CompliationParams& params, clang::CodeCompleteConsumer* consumer) {
auto instance = createInstance(params);
assert(info->pp().isInNamedInterfaceUnit() &&
"Only module interface unit could be built as PCM");
/// Set options to run code completion.
instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str();
instance->getFrontendOpts().CodeCompletionAt.Line = params.line;
instance->getFrontendOpts().CodeCompletionAt.Column = params.column;
instance->setCodeCompletionConsumer(consumer);
out.isInterfaceUnit = true;
out.name = info->pp().getNamedModuleName();
for(auto& [name, path]: params.pcms) {
out.mods.emplace_back(name);
}
applyPreamble(*instance, params);
out.path = params.outPath.str();
out.srcPath = params.srcPath.str();
out.deps = info->deps();
return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly);
return std::move(*info);
}
} // namespace clice

42
src/Compiler/Dependency.cpp
View File

@@ -1,42 +0,0 @@
#include <clang/Tooling/CompilationDatabase.h>
#include <clang/Tooling/DependencyScanning/DependencyScanningTool.h>
namespace clice::dependencies {
namespace {
/// module name -> file path.
llvm::StringMap<llvm::StringRef> moduleMap;
llvm::BumpPtrAllocator allocator;
void scan() {
using namespace clang::tooling::dependencies;
DependencyScanningService service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::P1689);
// TODO: figure out vfs
DependencyScanningTool tool(service);
// TODO: figure out
clang::tooling::CompileCommand command;
auto rule = tool.getP1689ModuleDependencyFile(command, "CWD");
}
} // namespace
void load(llvm::ArrayRef<llvm::StringRef> dirs) {
for(auto dir: dirs) {
std::string message;
auto CDB = clang::tooling::CompilationDatabase::loadFromDirectory(dir, message);
if(!CDB) {}
// TODO:
// remove unused compile commands.
// scan the whole project(if it is module file scan it).
// record include graph?? unsure
// use BumpPtrAllocator to store the commands for
// - reduce memory usage
// - expose `std::vector<const char*>` to `Compiler::Invocation`
}
}
} // namespace clice::dependencies

4
src/Compiler/Semantic.cpp
View File

@@ -35,6 +35,8 @@ SymbolKind SymbolKind::from(const clang::Decl* decl) {
}
}
SymbolKind SymbolKind::from(const clang::tok::TokenKind kind) {}
SymbolKind SymbolKind::from(const clang::tok::TokenKind kind) {
return {};
}
} // namespace clice

2
src/Feature/CodeCompletion.cpp
View File

@@ -200,7 +200,7 @@ json::Value capability(json::Value clientCapabilities) {
};
}
proto::CompletionResult codeCompletion(CompliationParams& params,
proto::CompletionResult codeCompletion(CompilationParams& params,
uint32_t line,
uint32_t column,
llvm::StringRef file,

102
src/Server/Async.cpp
View File

@@ -75,6 +75,39 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) {
}
}
/// Write a JSON value to the client.
void write(json::Value value) {
struct Buffer {
uv_write_t req;
llvm::SmallString<128> header;
llvm::SmallString<4096> message;
};
Buffer* buffer = new Buffer();
buffer->req.data = buffer;
llvm::raw_svector_ostream os(buffer->message);
os << value;
llvm::raw_svector_ostream sos(buffer->header);
sos << "Content-Length: " << buffer->message.size() << "\r\n\r\n";
uv_buf_t bufs[2] = {
uv_buf_init(buffer->header.data(), buffer->header.size()),
uv_buf_init(buffer->message.data(), buffer->message.size()),
};
auto on_write = [](uv_write_t* req, int status) {
if(status < 0) {
log::fatal("An error occurred while writing: {0}", uv_strerror(status));
}
delete static_cast<Buffer*>(req->data);
};
uv_check_call(uv_write, &buffer->req, writer, bufs, 2, on_write);
}
} // namespace
void start_server(Callback callback) {
@@ -126,42 +159,45 @@ void start_server(Callback callback, const char* ip, unsigned int port) {
uv_check_call(uv_run, async::loop, UV_RUN_DEFAULT);
}
void write(json::Value id, json::Value result) {
json::Value response = json::Object{
{"jsonrpc", "2.0" },
{"id", id },
{"result", result},
};
/// Send a request to the client.
void request(llvm::StringRef method, json::Value params) {
static std::uint32_t id = 0;
write(json::Object{
{"jsonrpc", "2.0" },
{"id", id += 1 },
{"method", method },
{"params", std::move(params)},
});
}
struct Buffer {
uv_write_t req;
llvm::SmallString<128> header;
llvm::SmallString<4096> message;
};
/// Send a notification to the client.
void notify(llvm::StringRef method, json::Value params) {
write(json::Object{
{"jsonrpc", "2.0" },
{"method", method },
{"params", std::move(params)},
});
}
Buffer* buffer = new Buffer();
buffer->req.data = buffer;
void response(json::Value id, json::Value result) {
write(json::Object{
{"jsonrpc", "2.0" },
{"id", id },
{"result", std::move(result)},
});
}
llvm::raw_svector_ostream os(buffer->message);
os << response;
llvm::raw_svector_ostream sos(buffer->header);
sos << "Content-Length: " << buffer->message.size() << "\r\n\r\n";
uv_buf_t bufs[2] = {
uv_buf_init(buffer->header.data(), buffer->header.size()),
uv_buf_init(buffer->message.data(), buffer->message.size()),
};
auto on_write = [](uv_write_t* req, int status) {
if(status < 0) {
log::fatal("An error occurred while writing: {0}", uv_strerror(status));
}
delete static_cast<Buffer*>(req->data);
};
uv_check_call(uv_write, &buffer->req, writer, bufs, 2, on_write);
/// Send an register capability to the client.
void registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions) {
request("client/registerCapability",
json::Object{
{"registrations",
json::Array{json::Object{
{"id", id},
{"method", method},
{"registerOptions", std::move(registerOptions)},
}}},
});
}
} // namespace clice::async

118
src/Server/Command.cpp
View File

@@ -1,118 +0,0 @@
#include "Server/Command.h"
#include "Server/Config.h"
#include "Server/Logger.h"
namespace clice {
void CommandManager::update(llvm::StringRef dir) {
llvm::SmallString<128> path;
path::append(path, dir, "compile_commands.json");
auto buffer = llvm::MemoryBuffer::getFile(path);
if(!buffer) {
log::warn("Failed to read compile_commands.json from {}, because {}",
dir,
buffer.getError().message());
return;
}
auto json = json::parse(buffer.get()->getBuffer());
if(!json) {
log::warn("Failed to parse json file at {}, because {}", path, json.takeError());
return;
}
if(!json->getAsArray()) {
log::warn("Invalid JSON format at {}, Compilation Database requires Array, but get {}",
path,
refl::enum_name(json->kind()));
return;
}
auto& CDB = CDBs[dir];
/// Clear old commands.
/// FIXME: it would be better to have cache in some way.
CDB.clear();
for(auto& value: *json->getAsArray()) {
auto element = value.getAsObject();
if(!element) {
log::warn("Invalid JSON format at {}, Compilation Database requires Object, but get {}",
path,
refl::enum_name(value.kind()));
continue;
}
/// Source file path.
llvm::SmallString<128> path;
auto file = element->getString("file");
if(!file) {
log::warn("the element in {} does not have a file field",
path,
refl::enum_name(value.kind()));
continue;
}
if(path::is_relative(file.value())) {
auto working = element->getString("directory");
if(!working) {
log::warn(
"Invalid JSON format at {}, {} is relative path, but directory is not provided",
path,
file.value());
}
path::append(path, working.value(), file.value());
} else {
path::append(path, file.value());
}
/// Command to compile the source file.
llvm::SmallString<1024> rawCommand;
if(auto command = element->getString("command")) {
rawCommand = command.value();
} else if(auto arguments = element->getArray("arguments")) {
/// FIXME:
std::terminate();
} else {
log::warn(
"Invalid JSON format, the element in {} does not have a command or arguments field",
path,
refl::enum_name(value.kind()));
continue;
}
CDB[path].emplace_back(rawCommand.str());
}
log::info("Compilation Database at {} is up-to-date", dir);
}
llvm::StringRef CommandManager::lookupFirst(llvm::StringRef file) {
for(auto& [dir, cdb]: CDBs) {
auto iter = cdb.find(file);
if(iter != cdb.end()) {
return iter->second.front();
}
}
return {};
}
llvm::ArrayRef<std::string> CommandManager::lookup(llvm::StringRef file) {
for(auto& [dir, cdb]: CDBs) {
/// FIXME: currently we directly return the first match.
/// it is better to provide a callback to handle all matches.
auto iter = cdb.find(file);
if(iter != cdb.end()) {
return iter->second;
}
}
return {};
}
} // namespace clice

9
src/Server/Document.cpp
View File

@@ -5,25 +5,24 @@ namespace clice {
async::promise<void> Server::onDidOpen(const proto::DidOpenTextDocumentParams& params) {
auto path = URI::resolve(params.textDocument.uri);
llvm::StringRef content = params.textDocument.text;
co_await scheduler.add(path, content);
co_await scheduler.update(path, content, synchronizer);
}
async::promise<void> Server::onDidChange(const proto::DidChangeTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
llvm::StringRef content = document.contentChanges[0].text;
co_await scheduler.update(path, content);
co_await scheduler.update(path, content, synchronizer);
}
async::promise<void> Server::onDidSave(const proto::DidSaveTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
co_await scheduler.save(path);
/// co_await scheduler.save(path);
co_return;
}
async::promise<void> Server::onDidClose(const proto::DidCloseTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
co_await scheduler.close(path);
/// co_await scheduler.close(path);
co_return;
}

16
src/Server/Feature.cpp
View File

@@ -90,10 +90,10 @@ async::promise<void> Server::onDocumentSymbol(json::Value id,
async::promise<void> Server::onSemanticTokens(json::Value id,
const proto::SemanticTokensParams& params) {
auto path = URI::resolve(params.textDocument.uri);
auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) {
return feature::semanticTokens(compiler, "");
});
async::write(std::move(id), json::serialize(tokens));
// auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) {
// return feature::semanticTokens(compiler, "");
// });
/// async::response(std::move(id), json::serialize(tokens));
co_return;
}
@@ -104,10 +104,10 @@ async::promise<void> Server::onInlayHint(json::Value id, const proto::InlayHintP
async::promise<void> Server::onCodeCompletion(json::Value id,
const proto::CompletionParams& params) {
auto path = URI::resolve(params.textDocument.uri);
auto result = co_await scheduler.codeComplete(path,
params.position.line + 1,
params.position.character + 1);
async::write(std::move(id), json::serialize(result));
// auto result = co_await scheduler.codeComplete(path,
// params.position.line + 1,
// params.position.character + 1);
// async::response(std::move(id), json::serialize(result));
co_return;
}

27
src/Server/Lifestyle.cpp
View File

@@ -6,12 +6,35 @@ async::promise<void> Server::onInitialize(json::Value id, const proto::Initializ
auto workplace = URI::resolve(params.workspaceFolders[0].uri);
config::init(workplace);
if(!params.capabilities.workspace.didChangeWatchedFiles.dynamicRegistration) {
log::fatal(
"clice requires the client to support file event watching to monitor updates to CDB files");
}
proto::InitializeResult result = {};
async::write(std::move(id), json::serialize(result));
async::response(std::move(id), json::serialize(result));
/// Load the compile commands from the workspace.
for(auto dir: config::frontend().compile_commands_directorys) {
synchronizer.sync(dir + "/compile_commands.json");
}
co_return;
}
async::promise<void> Server::onInitialized(const proto::InitializedParams& params) {
proto::DidChangeWatchedFilesRegistrationOptions options;
for(auto& dir: config::frontend().compile_commands_directorys) {
options.watchers.emplace_back(proto::FileSystemWatcher{
dir + "/compile_commands.json",
});
}
async::registerCapacity("watchedFiles",
"workspace/didChangeWatchedFiles",
json::serialize(options));
/// Load all information about PCHs and PCMs from disk.
scheduler.loadFromDisk();
co_return;
}
@@ -20,6 +43,8 @@ async::promise<void> Server::onExit(const proto::None&) {
}
async::promise<void> Server::onShutdown(json::Value id, const proto::None&) {
/// Save all information about PCHs and PCMs to disk.
scheduler.saveToDisk();
co_return;
}

399
src/Server/Scheduler.cpp
View File

@@ -4,185 +4,276 @@
namespace clice {
async::promise<void> Scheduler::updatePCH(llvm::StringRef filepath,
llvm::StringRef content,
llvm::StringRef command) {
auto [iter, success] = pchs.try_emplace(filepath);
if(success || iter->second.needUpdate(content)) {
Tracer tracer;
static std::string getPCHOutPath(llvm::StringRef srcPath) {
llvm::SmallString<128> outPath = srcPath;
path::replace_path_prefix(outPath, config::workplace(), config::frontend().cache_directory);
path::replace_extension(outPath, ".pch");
CompliationParams params;
params.content = content;
params.srcPath = filepath;
params.outPath = filepath;
params.command = command;
path::replace_path_prefix(params.outPath,
config::workplace(),
config::frontend().cache_directory);
path::replace_extension(params.outPath, ".pch");
log::info("Start building PCH for {0} at {1}", params.srcPath, params.outPath);
PCHInfo pch;
co_await async::schedule_task([&] {
auto dir = path::parent_path(params.outPath);
if(!fs::exists(dir)) {
if(auto error = fs::create_directories(dir)) {
log::fatal("Failed to create directory {0}, because {1}, build PCH stopped",
dir,
error.message());
return;
}
}
if(auto info = compile(params, pch); !info) {
log::fatal("Failed to build PCH for {0}, because {1}",
filepath.str(),
info.takeError());
return;
}
});
log::info("PCH for {0} is up-to-date, elapsed {1}ms",
filepath.str(),
tracer.duration().count());
pchs[filepath] = std::move(pch);
} else {
log::info("Reuse PCH for {0} from {1}", filepath.str(), iter->second.path);
}
co_return;
}
async::promise<void> Scheduler::buildAST(llvm::StringRef filepath, llvm::StringRef content) {
llvm::SmallString<128> path = filepath;
auto [iter, success] = files.try_emplace(filepath);
if(!success && !iter->second.isIdle) {
/// If the file is already existed and is building, append the task to the waiting list.
co_await async::suspend([&](auto handle) {
iter->second.waitings.emplace_back(Task{
.isBuild = true,
.waiting = handle,
});
});
}
files[filepath].isIdle = false;
static bool initCmd = false;
if(!initCmd) {
cmdMgr.update(config::frontend().compile_commands_directorys[0]);
initCmd = true;
}
CompliationParams params;
params.srcPath = path;
params.content = content;
params.command = cmdMgr.lookupFirst(filepath);
params.addPCH(pchs.at(filepath));
/// through arguments to judge is it a module.
bool isModule = false;
co_await (isModule ? updatePCM() : updatePCH(filepath, content, params.command));
Tracer tracer;
log::info("Start building AST for {0}, command: [{1}]", filepath, params.command.str());
auto task = [&] {
/// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be
/// Invalid Because other changed the `pchs` map. We also cannot to retrieve the `pch` from
/// the `pchs` map in this task, beacuse it is called in thread pool which will result in
/// data race. So temporarily copy the `pch` here. There must be a better way to solve this
/// problem.
auto info = clice::compile(params);
if(!info) {
log::fatal("Failed to build AST for {0}", filepath);
if(auto dir = path::parent_path(outPath); !fs::exists(dir)) {
if(auto error = fs::create_directories(dir)) {
log::fatal("Failed to create directory {0}, because {1}", dir, error.message());
}
return std::move(*info);
};
auto compiler = co_await async::schedule_task(std::move(task));
auto& file = files[path];
file.content = content;
file.compiler = std::move(compiler);
log::info("Build AST successfully for {0}, elapsed {1}", filepath, tracer.duration());
if(!file.waitings.empty()) {
auto task = std::move(file.waitings.front());
async::schedule(task.waiting);
file.waitings.pop_front();
}
file.isIdle = true;
return outPath.str().str();
}
async::promise<proto::CompletionResult> Scheduler::codeComplete(llvm::StringRef filepath,
unsigned int line,
unsigned int column) {
auto iter = files.find(filepath);
if(iter == files.end()) {
log::fatal("File {0} is not building, skip code completion", filepath);
static std::string getPCMOutPath(llvm::StringRef srcPath) {
llvm::SmallString<128> outPath = srcPath;
path::replace_path_prefix(outPath, config::workplace(), config::frontend().cache_directory);
path::replace_extension(outPath, ".pcm");
if(auto dir = path::parent_path(outPath); !fs::exists(dir)) {
if(auto error = fs::create_directories(dir)) {
log::fatal("Failed to create directory {0}, because {1}", dir, error.message());
}
}
if(iter->second.isIdle) {
/// If the file is already existed and is building, append the task to the waiting list.
co_await async::suspend([&](auto handle) {
iter->second.waitings.emplace_back(Task{
.isBuild = true,
.waiting = handle,
});
});
return outPath.str().str();
}
async::promise<> Scheduler::updatePCH(CompilationParams& params, class Synchronizer& sync) {
llvm::StringRef srcPath = params.srcPath;
auto& pch = pchs[srcPath];
/// FIXME: judge need update here ...
if(!pch.needUpdate(params.content)) {
log::info("PCH for {0} is already up-to-date, reuse it", srcPath);
co_return;
}
llvm::SmallString<128> path = filepath;
/// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1
/// Construct the output path.
params.outPath = getPCHOutPath(srcPath);
CompliationParams params;
params.content = iter->second.content;
params.srcPath = path;
params.command = cmdMgr.lookupFirst(filepath);
/// through arguments to judge is it a module.
bool isModule = false;
co_await (isModule ? updatePCM() : updatePCH(params.srcPath, params.content, params.command));
params.addPCH(pchs.at(filepath));
/// Build PCH.
PCHInfo info;
Tracer tracer;
log::info("Run code completion at {0}:{1}:{2}", filepath, line, column);
log::info("Building PCH for {0}", srcPath);
auto task = [&] {
return feature::codeCompletion(params, line, column, filepath, {});
};
/// FIXME: consider header context.
params.computeBounds();
auto result = co_await async::schedule_task(std::move(task));
llvm::Error error = co_await async::schedule_task([&] -> llvm::Error {
auto result = compile(params, info);
if(!result) {
return result.takeError();
}
log::info("Code completion for {0} is done, elapsed {1}", filepath, tracer.duration());
/// FIXME: consider indexing PCH here.
co_return result;
return llvm::Error::success();
});
pchs[srcPath] = std::move(info);
if(error) {
log::warn("Failed to build PCH for {0}, because {1}", srcPath, error);
co_return;
} else {
log::info("PCH for {0} is up-to-date, elapsed {1}", srcPath, tracer.duration());
}
}
async::promise<void> Scheduler::add(llvm::StringRef path, llvm::StringRef content) {
co_await buildAST(path, content);
co_return;
llvm::Error Scheduler::addModuleDeps(CompilationParams& params,
const ModuleInfo& moduleInfo) const {
for(auto& mod: moduleInfo.mods) {
auto iter = pcms.find(mod);
if(iter == pcms.end()) {
return error("Cannot find PCM for module {0}", mod);
}
/// Add prerequired PCM.
if(auto error = addModuleDeps(params, iter->second)) {
return error;
}
params.addPCM(iter->second);
}
return llvm::Error::success();
}
async::promise<void> Scheduler::update(llvm::StringRef path, llvm::StringRef content) {
co_await buildAST(path, content);
co_return;
async::promise<> Scheduler::updatePCM(llvm::StringRef moduleName, class Synchronizer& sync) {
llvm::StringRef srcPath = sync.map(moduleName);
if(srcPath.empty()) {
log::warn("Cannot find source file for module {0}", moduleName);
co_return;
}
auto& pcm = pcms[moduleName];
if(!pcm.needUpdate()) {
log::info("PCM for {0} is already up-to-date, reuse it", srcPath);
}
CompilationParams params;
params.srcPath = srcPath;
params.command = sync.lookup(srcPath);
params.outPath = getPCMOutPath(srcPath);
auto moduleInfo = scanModule(params);
if(!moduleInfo) {
log::warn("Build AST for {0} failed, because {1}", srcPath, moduleInfo.takeError());
co_return;
}
/// Build prerequired PCM.
for(auto& mod: moduleInfo->mods) {
co_await updatePCM(mod, sync);
}
/// Build PCM.
PCMInfo info;
Tracer tracer;
log::info("Building PCM for {0}", srcPath);
/// Add deps.
if(auto error = addModuleDeps(params, *moduleInfo)) {
log::warn("Failed to build PCM for {0}, because {1}", srcPath, error);
co_return;
}
llvm::Error error = co_await async::schedule_task([&] -> llvm::Error {
auto result = compile(params, info);
if(!result) {
return result.takeError();
}
/// FIXME: consider indexing PCH here.
return llvm::Error::success();
});
pcms[moduleName] = std::move(info);
if(error) {
log::warn("Failed to build PCM for {0}, because {1}", srcPath, error);
co_return;
} else {
log::info("PCM for {0} is up-to-date, elapsed {1}", srcPath, tracer.duration());
}
}
async::promise<void> Scheduler::save(llvm::StringRef path) {
co_return;
async::promise<> Scheduler::update(llvm::StringRef filename,
llvm::StringRef content,
class Synchronizer& sync) {
CompilationParams params;
params.content = content;
params.srcPath = filename;
params.command = sync.lookup(filename);
auto moduleInfo = scanModule(params);
if(!moduleInfo) {
log::warn("Build AST for {0} failed, because {1}", filename, moduleInfo.takeError());
co_return;
}
if(moduleInfo->name.empty() && moduleInfo->mods.empty()) {
co_await updatePCH(params, sync);
params.bounds.reset();
params.addPCH(pchs[params.srcPath]);
} else {
for(auto& mod: moduleInfo->mods) {
co_await updatePCM(mod, sync);
}
if(auto error = addModuleDeps(params, *moduleInfo)) {
log::warn("Failed to build PCM for {0}, because {1}", filename, error);
co_return;
}
}
/// Build AST.
ASTInfo info;
Tracer tracer;
log::info("Building AST for {0}", filename);
co_await async::schedule_task([&] {
auto result = compile(params);
if(!result) {
log::warn("Failed to build AST for {0}, because {1}", filename, result.takeError());
return;
}
info = std::move(*result);
});
/// Build AST successfully.
log::info("AST for {0} is up-to-date, elapsed {1}", filename, tracer.duration());
}
async::promise<void> Scheduler::close(llvm::StringRef path) {
co_return;
void Scheduler::loadFromDisk() {
llvm::SmallString<128> fileName;
path::append(fileName, config::frontend().cache_directory, "cache.json");
auto buffer = llvm::MemoryBuffer::getFile(fileName);
if(!buffer) {
log::warn("Failed to load cache from disk, because {0}", buffer.getError().message());
return;
}
auto json = json::parse(buffer.get()->getBuffer());
if(!json) {
log::warn("Failed to parse cache from disk, because {0}", json.takeError());
return;
}
auto object = json->getAsObject();
if(!object) {
log::warn("Failed to parse cache from disk, because {0}", json.takeError());
return;
}
if(auto pchArray = object->getArray("pch")) {
for(auto& value: *pchArray) {
auto pch = json::deserialize<PCHInfo>(value);
pchs[pch.srcPath] = std::move(pch);
}
}
if(auto pcmArray = object->getArray("pcm")) {
for(auto& value: *pcmArray) {
auto pcm = json::deserialize<PCMInfo>(value);
pcms[pcm.name] = std::move(pcm);
}
}
log::info("Cache loaded from {0}", fileName);
}
void Scheduler::saveToDisk() const {
json::Object result;
json::Array pchArray;
for(auto& [name, pch]: pchs) {
pchArray.emplace_back(json::serialize(pch));
}
result.try_emplace("pch", std::move(pchArray));
json::Array pcmArray;
for(auto& [name, pcm]: pcms) {
pcmArray.emplace_back(json::serialize(pcm));
}
result.try_emplace("pcm", std::move(pcmArray));
llvm::SmallString<128> fileName;
path::append(fileName, config::frontend().cache_directory, "cache.json");
std::error_code EC;
llvm::raw_fd_ostream stream(fileName, EC, llvm::sys::fs::OF_Text);
if(EC) {
log::warn("Failed save cache to disk, because {0}", EC.message());
return;
}
stream << json::Value(std::move(result));
log::info("Cache saved to {0}", fileName);
}
} // namespace clice

2
src/Server/Server.cpp
View File

@@ -100,6 +100,8 @@ Server::Server() {
addMethod("textDocument/formatting", &Server::onFormatting);
addMethod("textDocument/rangeFormatting", &Server::onRangeFormatting);
addMethod("workspace/didChangeWatchedFiles", &Server::onDidChangeWatchedFiles);
addMethod("context/current", &Server::onContextCurrent);
addMethod("context/switch", &Server::onContextSwitch);
addMethod("context/all", &Server::onContextAll);

118
src/Server/Synchronizer.cpp Normal file
View File

@@ -0,0 +1,118 @@
#include "Compiler/Compiler.h"
#include "Server/Logger.h"
#include "Server/Synchronizer.h"
namespace clice {
void Synchronizer::sync(llvm::StringRef filename) {
/// Read the compile commands from the file.
json::Value json = nullptr;
if(auto buffer = llvm::MemoryBuffer::getFile(filename)) {
if(auto result = json::parse(buffer->get()->getBuffer())) {
/// llvm::json::Value will hold on string buffer.
/// Do not worry about the lifetime of the buffer.
/// Release buffer to save memory.
json = std::move(result.get());
} else {
log::warn("Failed to parse json file at {0}, because {1}",
filename,
result.takeError());
return;
}
} else {
log::warn("Failed to read file {0}", filename);
return;
}
assert(json.kind() != json::Value::Null && "json is nullptr");
if(json.kind() != json::Value::Array) {
log::warn("Compilation Database requires a array of object, but get {0}, input file: {1}",
refl::enum_name(json.kind()),
filename);
return;
}
auto elements = json.getAsArray();
assert(elements && "json is not an array");
for(auto& element: *elements) {
auto object = element.getAsObject();
if(!object) {
log::warn(
"Compilation Database requires an array of object, but get a array of {0}, input file: {1}",
refl::enum_name(element.kind()),
filename);
continue;
}
/// FIXME: currently we assume all path here is absolute.
/// Add `directory` field in the future.
llvm::SmallString<128> path;
if(auto file = object->getString("file")) {
if(auto error = fs::real_path(*file, path)) {
log::warn("Failed to get real path of {0}, because {1}", *file, error.message());
continue;
}
} else {
log::warn("The element does not have a file field, input file: {0}", filename);
continue;
}
auto command = object->getString("command");
if(!command) {
log::warn("The key:{0} does not have a command field, input file: {1}", path, filename);
continue;
}
commands[path] = *command;
}
log::info("Successfully loaded compile commands from {0}, total {1} commands",
filename,
commands.size());
/// Scan all files to build module map.
CompilationParams params;
for(auto& [path, command]: commands) {
params.srcPath = path;
params.command = command;
auto info = scanModule(params);
if(!info) {
log::warn("Failed to scan module from {0}, because {1}", path, info.takeError());
continue;
}
if(info->isInterfaceUnit) {
assert(!info->name.empty() && "module name is empty");
moduleMap[info->name] = path;
}
}
log::info("Successfully built module map, total {0} modules", moduleMap.size());
}
void Synchronizer::sync(llvm::StringRef name, llvm::StringRef path) {
moduleMap[name] = path;
}
llvm::StringRef Synchronizer::lookup(llvm::StringRef file) const {
auto iter = commands.find(file);
if(iter == commands.end()) {
return "";
}
return iter->second;
}
llvm::StringRef Synchronizer::map(llvm::StringRef name) const {
auto iter = moduleMap.find(name);
if(iter == moduleMap.end()) {
return "";
}
return iter->second;
}
} // namespace clice

27
src/Server/Workplace.cpp Normal file
View File

@@ -0,0 +1,27 @@
#include "Server/Server.h"
namespace clice {
async::promise<> Server::onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params) {
for(auto& event: params.changes) {
switch(event.type) {
case proto::FileChangeType::Created: {
break;
}
case proto::FileChangeType::Changed: {
auto path = URI::resolve(event.uri);
synchronizer.sync(path);
break;
}
case proto::FileChangeType::Deleted: {
break;
}
}
}
co_return;
}
} // namespace clice

22
unittests/Basic/URI.cpp
View File

@@ -5,7 +5,7 @@ namespace clice {
namespace {
TEST(URITest, ConstructorAndAccessors) {
TEST(URI, Basic) {
URI uri("https", "reviews.llvm.org", "/D41946");
EXPECT_EQ(uri.scheme(), "https");
@@ -13,7 +13,7 @@ TEST(URITest, ConstructorAndAccessors) {
EXPECT_EQ(uri.body(), "/D41946");
}
TEST(URITest, CopyConstructor) {
TEST(URI, Copy) {
URI uri1("https", "reviews.llvm.org", "/D41946");
URI uri2(uri1);
@@ -22,7 +22,7 @@ TEST(URITest, CopyConstructor) {
EXPECT_EQ(uri2.body(), "/D41946");
}
TEST(URITest, EqualityOperator) {
TEST(URI, Eq) {
URI uri1("https", "reviews.llvm.org", "/D41946");
URI uri2("https", "reviews.llvm.org", "/D41946");
URI uri3("http", "example.com", "/index.html");
@@ -31,11 +31,19 @@ TEST(URITest, EqualityOperator) {
EXPECT_FALSE(uri1 == uri3);
}
TEST(URITest, ParseFunction) {
auto expectedUri = URI::parse("https://reviews.llvm.org/D41946");
ASSERT_TRUE(static_cast<bool>(expectedUri));
TEST(URI, File) {
auto uri = URI::from("/home/user/file.txt");
EXPECT_EQ(uri.scheme(), "file");
EXPECT_EQ(uri.authority(), "");
EXPECT_EQ(uri.body(), "/home/user/file.txt");
EXPECT_EQ(uri.toString(), "file:///home/user/file.txt");
}
URI uri = expectedUri.get();
TEST(URI, Parse) {
auto expectedUri = URI::parse("https://reviews.llvm.org/D41946");
ASSERT_TRUE(bool(expectedUri));
URI& uri = expectedUri.get();
EXPECT_EQ(uri.scheme(), "https");
EXPECT_EQ(uri.authority(), "reviews.llvm.org");
EXPECT_EQ(uri.body(), "/D41946");

10
unittests/Compiler/Compiler.cpp
View File

@@ -16,7 +16,7 @@ int main(){
}
)cpp";
CompliationParams params;
CompilationParams params;
params.content = code;
params.srcPath = "main.cpp";
params.command = "clang++ -std=c++20 main.cpp";
@@ -34,7 +34,7 @@ int main(){
})cpp";
/// Test in no header file.
CompliationParams params;
CompilationParams params;
params.content = code;
params.srcPath = "main.cpp";
params.command = "clang++ -std=c++20 main.cpp";
@@ -91,7 +91,7 @@ int main(){
return;
}
CompliationParams params;
CompilationParams params;
params.content = code;
params.srcPath = "main.cpp";
params.outPath = outpath;
@@ -128,7 +128,7 @@ export int foo() {
return;
}
CompliationParams params;
CompilationParams params;
params.srcPath = "main.cppm";
params.content = code;
params.outPath = outpath;
@@ -162,7 +162,7 @@ export module A;
export int foo = 1;
)cpp";
CompliationParams params;
CompilationParams params;
params.content = code;
params.srcPath = "main.cppm";
params.command = "clang++ -std=c++20 main.cppm";

113
unittests/Compiler/Module.cpp Normal file
View File

@@ -0,0 +1,113 @@
#include "gtest/gtest.h"
#include "Compiler/Compiler.h"
#include "llvm/Support/ToolOutputFile.h"
namespace clice {
namespace {
PCMInfo buildPCM(llvm::StringRef file, llvm::StringRef code) {
llvm::SmallString<128> outPath;
fs::createUniquePath(llvm::Twine(file) + "%%%%%%.pcm", outPath, true);
CompilationParams params;
params.content = code;
params.srcPath = file;
params.outPath = outPath;
params.command = "clang++ -std=c++20 -x c++ " + file.str();
params.remappedFiles.emplace_back("./test.h", "export int foo2();");
PCMInfo pcm;
if(!compile(params, pcm)) {
llvm::errs() << "Failed to build PCM\n";
std::terminate();
}
return pcm;
}
ModuleInfo scan(llvm::StringRef content) {
CompilationParams params;
params.content = content;
params.srcPath = "main.ixx";
params.command = "clang++ -std=c++20 -x c++ main.ixx";
params.remappedFiles.emplace_back("./test.h", "export module A");
auto info = scanModule(params);
if(!info) {
llvm::errs() << "Failed to scan module\n";
std::terminate();
}
return std::move(*info);
}
TEST(Module, Scan) {
/// Simple case.
const char* content = R"(
export module A;
import B;
)";
auto info = scan(content);
ASSERT_EQ(info.isInterfaceUnit, true);
ASSERT_EQ(info.name, "A");
ASSERT_EQ(info.mods.size(), 1);
ASSERT_EQ(info.mods[0], "B");
/// With global module fragment and private module fragment.
content = R"(
module;
#include <iostream>
export module A;
import B;
import C;
module : private;
)";
info = scan(content);
ASSERT_EQ(info.isInterfaceUnit, true);
ASSERT_EQ(info.name, "A");
ASSERT_EQ(info.mods.size(), 2);
ASSERT_EQ(info.mods[0], "B");
ASSERT_EQ(info.mods[1], "C");
/// With module partition.
content = R"(
module;
#include <iostream>
export module A:B;
import B;
import C;
module : private;
)";
info = scan(content);
ASSERT_EQ(info.isInterfaceUnit, true);
ASSERT_EQ(info.name, "A:B");
ASSERT_EQ(info.mods.size(), 2);
ASSERT_EQ(info.mods[0], "B");
ASSERT_EQ(info.mods[1], "C");
content = R"(
module A;
import B;
import C;
)";
info = scan(content);
ASSERT_EQ(info.isInterfaceUnit, false);
ASSERT_EQ(info.name, "A");
ASSERT_EQ(info.mods.size(), 2);
ASSERT_EQ(info.mods[0], "B");
ASSERT_EQ(info.mods[1], "C");
}
TEST(Module, Normal) {
const char* content = R"(
export module A;
)";
auto pcm = buildPCM("A.ixx", content);
// ASSERT_EQ(pcm.isInterfaceUnit, true);
// ASSERT_EQ(pcm.name, "A");
// ASSERT_EQ(pcm.mods.size(), 0);
}
} // namespace
} // namespace clice

2
unittests/Feature/CodeCompletion.cpp
View File

@@ -14,7 +14,7 @@ int main() {
}
)cpp";
CompliationParams params;
CompilationParams params;
params.content = code;
params.srcPath = "main.cpp";
params.command = "clang++ -std=c++20 main.cpp";

2
unittests/Test.h
View File

@@ -149,7 +149,7 @@ inline void EXPECT_NE(const LHS& lhs,
class Tester {
public:
CompliationParams params;
CompilationParams params;
std::unique_ptr<llvm::vfs::InMemoryFileSystem> vfs;
ASTInfo info;