From 5326480cd67efe6856cf2e450e15cd29d5fb5e7a Mon Sep 17 00:00:00 2001 From: ykiko Date: Sat, 14 Dec 2024 13:40:13 +0800 Subject: [PATCH] Basic support C++20 named module. (#12) --- include/Basic/URI.h | 34 ++- include/Compiler/Compiler.h | 116 +++++--- include/Feature/CodeCompletion.h | 4 +- include/Server/Async.h | 14 +- include/Server/Command.h | 24 -- include/Server/Protocol.h | 59 +++- include/Server/Scheduler.h | 91 ++---- include/Server/Server.h | 8 + include/Server/Synchronizer.h | 43 +++ include/Support/Struct.h | 8 +- src/Basic/URI.cpp | 26 ++ src/Compiler/Command.cpp | 4 + src/Compiler/Compiler.cpp | 229 ++++++++++----- src/Compiler/Dependency.cpp | 42 --- src/Compiler/Semantic.cpp | 4 +- src/Feature/CodeCompletion.cpp | 2 +- src/Server/Async.cpp | 102 ++++--- src/Server/Command.cpp | 118 -------- src/Server/Document.cpp | 9 +- src/Server/Feature.cpp | 16 +- src/Server/Lifestyle.cpp | 27 +- src/Server/Scheduler.cpp | 399 ++++++++++++++++----------- src/Server/Server.cpp | 2 + src/Server/Synchronizer.cpp | 118 ++++++++ src/Server/Workplace.cpp | 27 ++ unittests/Basic/URI.cpp | 22 +- unittests/Compiler/Compiler.cpp | 10 +- unittests/Compiler/Module.cpp | 113 ++++++++ unittests/Feature/CodeCompletion.cpp | 2 +- unittests/Test.h | 2 +- 30 files changed, 1082 insertions(+), 593 deletions(-) delete mode 100644 include/Server/Command.h create mode 100644 include/Server/Synchronizer.h delete mode 100644 src/Compiler/Dependency.cpp delete mode 100644 src/Server/Command.cpp create mode 100644 src/Server/Synchronizer.cpp create mode 100644 src/Server/Workplace.cpp create mode 100644 unittests/Compiler/Module.cpp diff --git a/include/Basic/URI.h b/include/Basic/URI.h index 10205029..25bf22b1 100644 --- a/include/Basic/URI.h +++ b/include/Basic/URI.h @@ -1,7 +1,8 @@ #pragma once #include -#include + +#include "Support/Support.h" namespace clice { @@ -14,19 +15,34 @@ public: bool operator== (const URI&) const = default; - /// Returns decoded scheme e.g. "https" - llvm::StringRef scheme() const { return m_scheme; } - - /// Returns decoded authority e.g. "reviews.lvm.org" - llvm::StringRef authority() const { return m_authority; } - - /// Returns decoded body e.g. "/D41946" - llvm::StringRef body() const { return m_body; } + /// Construct a URI object from the given file path. + static URI from(llvm::StringRef file); + /// Parse the given URI string to create a URI object. static llvm::Expected parse(llvm::StringRef content); + /// Same as `parse`, but will crash if failed. static std::string resolve(llvm::StringRef content); + /// Returns decoded scheme e.g. "https" + llvm::StringRef scheme() const { + return m_scheme; + } + + /// Returns decoded authority e.g. "reviews.llvm.org" + llvm::StringRef authority() const { + return m_authority; + } + + /// Returns decoded body e.g. "/D41946" + llvm::StringRef body() const { + return m_body; + } + + std::string toString() const { + return std::format("{}://{}{}", m_scheme, m_authority, m_body); + } + private: std::string m_scheme; std::string m_authority; diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h index 9c9143f8..60621fc5 100644 --- a/include/Compiler/Compiler.h +++ b/include/Compiler/Compiler.h @@ -5,21 +5,26 @@ #include #include +#include "Basic/Location.h" +#include "llvm/ADT/StringSet.h" namespace clice { +struct CompilationParams; + /// All information about AST. class ASTInfo { public: ASTInfo() = default; - ASTInfo(std::unique_ptr action, + ASTInfo(std::unique_ptr action, std::unique_ptr instance, std::unique_ptr tokBuf, - llvm::DenseMap&& directives) : - action(std::move(action)), instance(std::move(instance)), m_TokBuf(std::move(tokBuf)), - m_Directives(std::move(directives)) { - m_Resolver = std::make_unique(this->instance->getSema()); + llvm::DenseMap&& directives, + std::vector deps) : + action(std::move(action)), m_Instance(std::move(instance)), m_TokBuf(std::move(tokBuf)), + m_Directives(std::move(directives)), m_Deps(std::move(deps)) { + m_Resolver = std::make_unique(this->m_Instance->getSema()); } ASTInfo(const ASTInfo&) = delete; @@ -33,32 +38,32 @@ public: } } - clang::Sema& sema() { - return instance->getSema(); + auto& sema() { + return m_Instance->getSema(); } - clang::ASTContext& context() { - return instance->getASTContext(); + auto& context() { + return m_Instance->getASTContext(); } - clang::SourceManager& srcMgr() { - return instance->getSourceManager(); + auto& srcMgr() { + return m_Instance->getSourceManager(); } - clang::Preprocessor& pp() { - return instance->getPreprocessor(); + auto& pp() { + return m_Instance->getPreprocessor(); } clang::TranslationUnitDecl* tu() { - return instance->getASTContext().getTranslationUnitDecl(); + return m_Instance->getASTContext().getTranslationUnitDecl(); } - clang::syntax::TokenBuffer& tokBuf() { + auto& tokBuf() { assert(m_TokBuf && "Token buffer is not available"); return *m_TokBuf; } - TemplateResolver& resolver() { + auto& resolver() { return *m_Resolver; } @@ -66,13 +71,21 @@ public: return m_Directives; } - Directive& directive(clang::FileID id) { + auto& directive(clang::FileID id) { return m_Directives[id]; } + auto& deps() { + return m_Deps; + } + + auto& instance() { + return *m_Instance; + } + /// Get the length of the token at the given location. auto getTokenLength(clang::SourceLocation loc) { - return clang::Lexer::MeasureTokenLength(loc, srcMgr(), instance->getLangOpts()); + return clang::Lexer::MeasureTokenLength(loc, srcMgr(), m_Instance->getLangOpts()); } /// Get the spelling of the token at the given location. @@ -85,13 +98,22 @@ public: } private: - std::unique_ptr action; - std::unique_ptr instance; + std::unique_ptr action; + std::unique_ptr m_Instance; std::unique_ptr m_TokBuf; std::unique_ptr m_Resolver; llvm::DenseMap m_Directives; + std::vector m_Deps; }; +/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. +/// Note this function will not check whether we need to update the PCH or PCM, caller should check +/// their reusability and update in time. +llvm::Expected compile(CompilationParams& params); + +/// Run code completion at the given location. +llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer); + struct PCHInfo { /// PCM file path. std::string path; @@ -115,20 +137,44 @@ struct PCHInfo { bool needUpdate(llvm::StringRef content); }; -struct PCMInfo { +/// Build PCH from given file path and content. +llvm::Expected compile(CompilationParams& params, PCHInfo& out); + +struct ModuleInfo { + /// Whether this module is an interface unit. + /// i.e. has export module declaration. + bool isInterfaceUnit = false; + + /// Module name. + std::string name; + + /// Dependent modules of this module. + std::vector mods; +}; + +/// Run the preprocessor to scan the given module unit to +/// collect its module name and dependencies. +llvm::Expected scanModule(CompilationParams& params); + +inherited_struct(PCMInfo, ModuleInfo) { /// PCM file path. std::string path; /// Source file path. std::string srcPath; - /// Module name. - std::string name; + /// Files involved in building this PCM(not include module). + std::vector deps; - bool needUpdate(); + bool needUpdate() { + return true; + } }; -struct CompliationParams { +/// Build PCM from given file path and content. +llvm::Expected compile(CompilationParams& params, PCMInfo& out); + +struct CompilationParams { /// Source file content. llvm::StringRef content; @@ -164,7 +210,7 @@ struct CompliationParams { clang::PreambleBounds pchBounds = {0, false}; /// Information about reuse PCM(name, path). - llvm::SmallVector> pcms; + llvm::StringMap pcms; /// Code completion file:line:column. llvm::StringRef file = ""; @@ -177,22 +223,10 @@ struct CompliationParams { } void addPCM(const PCMInfo& info) { - pcms.emplace_back(info.name, info.path); + assert((!pcms.contains(info.name) || pcms[info.name] == info.path) && + "Add a different PCM with the same name"); + pcms[info.name] = info.path; } }; -/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. -/// Note this function will not check whether we need to update the PCH or PCM, caller should check -/// their reusability and update in time. -llvm::Expected compile(CompliationParams& params); - -/// Build PCH from given file path and content. -llvm::Expected compile(CompliationParams& params, PCHInfo& out); - -/// Build PCM from given file path and content. -llvm::Expected compile(CompliationParams& params, PCMInfo& out); - -/// Run code completion at the given location. -llvm::Expected compile(CompliationParams& params, clang::CodeCompleteConsumer* consumer); - } // namespace clice diff --git a/include/Feature/CodeCompletion.h b/include/Feature/CodeCompletion.h index 3e72d080..193b5179 100644 --- a/include/Feature/CodeCompletion.h +++ b/include/Feature/CodeCompletion.h @@ -4,7 +4,7 @@ #include namespace clice { -struct CompliationParams; +struct CompilationParams; } namespace clice::proto { @@ -117,7 +117,7 @@ json::Value capability(json::Value clientCapabilities); /// Run code completion in given file and location. `compiler` should be /// set properly if any PCH or PCM is needed. Each completion requires a /// new compiler instance. -proto::CompletionResult codeCompletion(CompliationParams& compliation, +proto::CompletionResult codeCompletion(CompilationParams& compliation, uint32_t line, uint32_t column, llvm::StringRef file, diff --git a/include/Server/Async.h b/include/Server/Async.h index c21b5356..555e3106 100644 --- a/include/Server/Async.h +++ b/include/Server/Async.h @@ -35,7 +35,7 @@ extern uv_loop_t* loop; template T& uv_cast(U* u) { assert(u && u->data && "uv_cast: invalid uv handle"); - return *static_cast(u->data); + return *static_cast*>(u->data); } using Callback = llvm::unique_function(json::Value)>; @@ -44,7 +44,17 @@ void start_server(Callback callback); void start_server(Callback callback, const char* ip, unsigned int port); -void write(json::Value id, json::Value result); +/// Send a request to the client. +void request(llvm::StringRef method, json::Value params); + +/// Send a notification to the client. +void notify(llvm::StringRef method, json::Value params); + +/// Send a response to the client. +void response(json::Value id, json::Value result); + +/// Send an register capability to the client. +void registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions); template struct result { diff --git a/include/Server/Command.h b/include/Server/Command.h deleted file mode 100644 index f467aadf..00000000 --- a/include/Server/Command.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "Support/Support.h" -#include "clang/Tooling/CompilationDatabase.h" - -namespace clice { - -class CommandManager { -public: - void update(llvm::StringRef dir); - - /// Return the commands of first meet file. - llvm::StringRef lookupFirst(llvm::StringRef file); - - llvm::ArrayRef lookup(llvm::StringRef file); - -private: - /// CDB file -> file -> [commands] - using Commands = std::vector; - using CDB = llvm::StringMap; - llvm::StringMap CDBs; -}; - -} // namespace clice diff --git a/include/Server/Protocol.h b/include/Server/Protocol.h index 1237c38f..68520f7c 100644 --- a/include/Server/Protocol.h +++ b/include/Server/Protocol.h @@ -33,7 +33,64 @@ struct ClientInfo { string version; }; -struct ClientCapabilities {}; +struct DidChangeWatchedFilesClientCapabilities { + /// Did change watched files notification supports dynamic registration. + /// Please note that the current protocol doesn't support static + /// configuration for file changes from the server side. + bool dynamicRegistration = false; +}; + +struct ClientCapabilities { + /// Workspace specific client capabilities. + struct { + /// Capabilities specific to the `workspace/didChangeWatchedFiles` + /// notification. + DidChangeWatchedFilesClientCapabilities didChangeWatchedFiles; + } workspace; +}; + +struct FileSystemWatcher { + /// The glob pattern to watch. + std::string globPattern; + + enum WatchKind { + Create = 1, + Change = 2, + Delete = 4, + }; + + /// The kind of events of interest. + int kind = WatchKind::Create | WatchKind::Change | WatchKind::Delete; +}; + +struct DidChangeWatchedFilesRegistrationOptions { + /// The watchers to register. + std::vector watchers; +}; + +enum class FileChangeType { + /// The file got created. + Created = 1, + + /// The file got changed. + Changed = 2, + + /// The file got deleted. + Deleted = 3, +}; + +struct FileEvent { + /// The file's URI. + string uri; + + /// The change type. + FileChangeType type; +}; + +struct DidChangeWatchedFilesParams { + /// The actual file events. + std::vector changes; +}; struct Workplace { /// The associated URI for this workspace folder. diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index ee01033f..59c406f1 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -3,7 +3,6 @@ #include #include "Server/Async.h" -#include "Server/Command.h" #include "Server/Protocol.h" #include "Server/Trace.h" #include "Compiler/Compiler.h" @@ -45,88 +44,38 @@ struct llvm::DenseMapInfo { namespace clice { -struct File2 { - bool isIdle = true; - - llvm::DenseMap contexts; -}; - -struct File; - -struct Task { - /// Whether this task is a build task. - bool isBuild = false; - /// The coroutine handle of this task. - std::coroutine_handle<> waiting; -}; - -struct File { - bool isIdle = true; - std::string content; - /// The compiler instance of this file. - ASTInfo compiler; - - std::deque waitings; -}; - +/// Responsible for manage the files and schedule the tasks. class Scheduler { private: - async::promise updatePCH(llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef command); + async::promise<> updatePCH(CompilationParams& params, class Synchronizer& sync); - async::promise updatePCM() { - co_return; - } + /// Clang requires all direct and indirect dependent modules to be added during module building. + /// This function adds the dependencies of the given module to the compilation parameters. + /// Note: It is assumed that all dependent modules have already been built. + llvm::Error addModuleDeps(CompilationParams& params, const ModuleInfo& moduleInfo) const; - async::promise buildAST(llvm::StringRef path, llvm::StringRef content); + async::promise<> updatePCM(llvm::StringRef name, class Synchronizer& sync); public: - async::promise add(llvm::StringRef path, llvm::StringRef content); + async::promise<> update(llvm::StringRef filename, + llvm::StringRef content, + class Synchronizer& sync); - async::promise update(llvm::StringRef path, llvm::StringRef content); + /// Load all Information about PCHs and PCMs from disk. + void loadFromDisk(); - async::promise save(llvm::StringRef path); + /// Save all Information about PCHs and PCMs to disk. + /// So that we can reuse them next time. + void saveToDisk() const; - async::promise close(llvm::StringRef path); - - async::promise codeComplete(llvm::StringRef path, - unsigned int line, - unsigned int column); - - /// Schedule a task for a file. If the file is building, the task will be - /// appended to the task list of the file and wait for the building to finish. - /// Otherwise, the task will be executed immediately. - template - auto schedule(llvm::StringRef path, Task&& task) - -> async::promise()))> { - auto& file = files[path]; - if(!file.isIdle) { - co_await async::suspend([&](auto handle) { - file.waitings.push_back({.isBuild = false, .waiting = handle}); - }); - } - - file.isIdle = false; - auto& compiler = file.compiler; - - auto result = co_await async::schedule_task([&task, &compiler] { return task(compiler); }); - - if(!file.waitings.empty()) { - auto task = std::move(file.waitings.front()); - async::schedule(task.waiting); - file.waitings.pop_front(); - } - - file.isIdle = true; - - co_return result; - } + struct File {}; private: + /// [file name] -> [PCHInfo] llvm::StringMap pchs; - llvm::StringMap files; - CommandManager cmdMgr; + + /// [module name] -> [PCMInfo] + llvm::StringMap pcms; }; } // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index fb123f7d..1170eaf3 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -5,6 +5,7 @@ #include "Server/Config.h" #include "Server/Logger.h" #include "Server/Scheduler.h" +#include "Server/Synchronizer.h" #include "Server/Protocol.h" #include "Support/Support.h" @@ -127,6 +128,12 @@ private: async::promise onRangeFormatting(json::Value id, const proto::DocumentRangeFormattingParams& params); + /// ============================================================================ + /// Workspace Features + /// ============================================================================ + + async::promise onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params); + /// ============================================================================ /// Extension /// ============================================================================ @@ -139,6 +146,7 @@ private: private: Scheduler scheduler; + Synchronizer synchronizer; llvm::StringMap requests; llvm::StringMap notifications; }; diff --git a/include/Server/Synchronizer.h b/include/Server/Synchronizer.h new file mode 100644 index 00000000..615afd24 --- /dev/null +++ b/include/Server/Synchronizer.h @@ -0,0 +1,43 @@ +#pragma once + +#include "Support/Support.h" + +namespace clice { + +/// Responsible for synchronizing changes to the CDB file, +/// including updating compile commands and the module map. +class Synchronizer { +public: + /// FIXME: The CDB file can be very large, reaching the size of + /// several gigabytes (GB). Therefore, it's better for this function + /// to be an asynchronous function. + + /// Update the compile commands. + void sync(llvm::StringRef file); + + /// Update the module map for active file. + void sync(llvm::StringRef name, llvm::StringRef path); + + /// Lookup the compile commands of the given file. + llvm::StringRef lookup(llvm::StringRef file) const; + + /// Lookup the module interface unit file path of the given module name. + llvm::StringRef map(llvm::StringRef name) const; + +private: + /// FIXME: currently we assume that a file only occurs once in the CDB. + /// This is not always correct, but it is enough for now. + + /// A map between file path and compile commands. + llvm::StringMap commands; + + /// For C++20 module, we only can got dependent module name + /// in source context. But we need dependent module file path + /// to build PCM. So we will scan(preprocess) all project files + /// to build a module map between module name and module file path. + /// **Note that** this only includes module interface unit, for module + /// implementation unit, the scan could be delayed until compiling it. + llvm::StringMap moduleMap; +}; + +} // namespace clice diff --git a/include/Support/Struct.h b/include/Support/Struct.h index 8124d7b1..e04472f5 100644 --- a/include/Support/Struct.h +++ b/include/Support/Struct.h @@ -199,7 +199,7 @@ struct Inheritance : Ts... {}; /// Use to define a reflectable struct with inheritance. #define inherited_struct(name, ...) \ struct name##Body; \ - using name = clice::refl::Inheritance<__VA_ARGS__, name##Body>; \ + using name = clice::refl::Inheritance<__VA_ARGS__, name##Body>; \ struct name##Body template @@ -216,7 +216,11 @@ struct Struct> { template constexpr static auto collcet_members(Object&& object) { - return std::tuple_cat(Struct::collcet_members(static_cast(object))...); + if constexpr(std::is_const_v>) { + return std::tuple_cat(Struct::collcet_members(static_cast(object))...); + } else { + return std::tuple_cat(Struct::collcet_members(static_cast(object))...); + } } }; diff --git a/src/Basic/URI.cpp b/src/Basic/URI.cpp index 10e5933e..ff0dc912 100644 --- a/src/Basic/URI.cpp +++ b/src/Basic/URI.cpp @@ -5,6 +5,8 @@ namespace clice { +namespace { + /// returns true if the scheme is valid according to RFC 3986. bool isValidScheme(llvm::StringRef scheme) { if(scheme.empty()) { @@ -39,6 +41,30 @@ static std::string decodePercent(llvm::StringRef content) { return result; } +} // namespace + +URI URI::from(llvm::StringRef file) { + if(!path::is_absolute(file)) { + std::terminate(); + } + + llvm::SmallString<128> path; + + for(auto c: file) { + if(c == '\\') { + path.push_back('/'); + } else if(std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '/') { + path.push_back(c); + } else { + path.push_back('%'); + path.push_back(llvm::hexdigit(c >> 4)); + path.push_back(llvm::hexdigit(c & 0xF)); + } + } + + return URI("file", "", path); +} + llvm::Expected URI::parse(llvm::StringRef content) { URI result("", "", ""); llvm::StringRef uri = content; diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index 6357be3f..dd2d20fb 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -58,6 +58,10 @@ llvm::Error mangleCommand(llvm::StringRef command, continue; } + if(arg.starts_with("@CMakeFiles")) { + continue; + } + /// TODO: remove PCH. out.push_back(arg.data()); diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp index 46ffd28c..7afed45c 100644 --- a/src/Compiler/Compiler.cpp +++ b/src/Compiler/Compiler.cpp @@ -19,7 +19,7 @@ bool PCHInfo::needUpdate(llvm::StringRef content) { namespace { -auto createInvocation(CompliationParams& params) { +auto createInvocation(CompilationParams& params) { llvm::SmallString<1024> buffer; llvm::SmallVector args; @@ -45,7 +45,7 @@ auto createInvocation(CompliationParams& params) { return invocation; } -auto createInstance(CompliationParams& params) { +auto createInstance(CompilationParams& params) { auto instance = std::make_unique(); instance->setInvocation(createInvocation(params)); @@ -64,10 +64,12 @@ auto createInstance(CompliationParams& params) { assert(!instance->getPreprocessorOpts().RetainRemappedFileBuffers && "RetainRemappedFileBuffers should be false"); - instance->getPreprocessorOpts().addRemappedFile( - params.srcPath, - llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath) - .release()); + if(!params.content.empty()) { + instance->getPreprocessorOpts().addRemappedFile( + params.srcPath, + llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath) + .release()); + } for(auto& [file, content]: params.remappedFiles) { instance->getPreprocessorOpts().addRemappedFile( @@ -75,14 +77,19 @@ auto createInstance(CompliationParams& params) { llvm::MemoryBuffer::getMemBufferCopy(content, file).release()); } + if(!instance->createTarget()) { + /// FIXME: add error handle here. + std::terminate(); + } + return instance; } -void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) { +void applyPreamble(clang::CompilerInstance& instance, CompilationParams& params) { auto& PPOpts = instance.getPreprocessorOpts(); auto& pch = params.pch; auto& bounds = params.pchBounds; - auto& pcms = params.pcms; + if(bounds.Size != 0) { PPOpts.UsePredefines = false; PPOpts.ImplicitPCHInclude = std::move(pch); @@ -91,30 +98,34 @@ void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; } + auto& pcms = params.pcms; for(auto& [name, path]: pcms) { auto& HSOpts = instance.getHeaderSearchOpts(); - HSOpts.PrebuiltModuleFiles.try_emplace(std::move(name), std::move(path)); + HSOpts.PrebuiltModuleFiles.try_emplace(name.str(), std::move(path)); } } -llvm::Expected ExecuteAction(std::unique_ptr instance, - clang::frontend::ActionKind kind) { - std::unique_ptr action; - if(kind == clang::frontend::ActionKind::ParseSyntaxOnly) { - action = std::make_unique(); - } else if(kind == clang::frontend::ActionKind::GeneratePCH) { - action = std::make_unique(); - } else if(kind == clang::frontend::ActionKind::GenerateReducedModuleInterface) { - action = std::make_unique(); - } else { - llvm::errs() << "Unsupported action kind\n"; - std::terminate(); +/// Execute given action with the on the given instance. `callback` is called after +/// `BeginSourceFile`. Beacuse `BeginSourceFile` may create new preprocessor. +llvm::Error ExecuteAction(clang::CompilerInstance& instance, + clang::FrontendAction& action, + auto&& callback) { + if(!action.BeginSourceFile(instance, instance.getFrontendOpts().Inputs[0])) { + return error("Failed to begin source file"); } - if(!instance->createTarget()) { - return error("Failed to create target"); + callback(); + + if(auto error = action.Execute()) { + return error; } + return llvm::Error::success(); +} + +llvm::Expected ExecuteAction(std::unique_ptr instance, + std::unique_ptr action) { + if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { return error("Failed to begin source file"); } @@ -129,12 +140,13 @@ llvm::Expected ExecuteAction(std::unique_ptr i llvm::DenseMap directives; Directive::attach(pp, directives); - std::optional collector; + /// Collect tokens. + std::optional tokCollector; /// It is not necessary to collect tokens if we are running code completion. /// And in fact will cause assertion failure. if(!instance->hasCodeCompletionConsumer()) { - collector.emplace(pp); + tokCollector.emplace(pp); } if(auto error = action->Execute()) { @@ -142,19 +154,23 @@ llvm::Expected ExecuteAction(std::unique_ptr i } std::unique_ptr tokBuf; - if(collector) { - tokBuf = std::make_unique(std::move(*collector).consume()); + if(tokCollector) { + tokBuf = std::make_unique(std::move(*tokCollector).consume()); } + /// FIXME: getDependencies currently return ArrayRef, which actually results in + /// extra copy. It would be great to avoid this copy. + return ASTInfo(std::move(action), std::move(instance), std::move(tokBuf), - std::move(directives)); + std::move(directives), + {}); } } // namespace -void CompliationParams::computeBounds(llvm::StringRef header) { +void CompilationParams::computeBounds(llvm::StringRef header) { assert(!bounds.has_value() && "Bounds is already computed"); assert(!content.empty() && "Source content is required to compute bounds"); @@ -248,15 +264,101 @@ void CompliationParams::computeBounds(llvm::StringRef header) { } } -llvm::Expected compile(CompliationParams& params) { +/// Scan the module name. This will not run the preprocessor. +std::string scanModuleName(llvm::StringRef content) { + clang::LangOptions langOpts; + langOpts.Modules = true; + langOpts.CPlusPlus20 = true; + clang::Lexer lexer(clang::SourceLocation(), + langOpts, + content.begin(), + content.begin(), + content.end()); + + clang::Token token; + lexer.Lex(token); + + while(!token.is(clang::tok::eof)) { + llvm::outs() << token.getName() << "\n"; + if(token.is(clang::tok::kw_module)) { + lexer.Lex(token); + + if(token.is(clang::tok::coloncolon)) { + lexer.Lex(token); + } + + if(token.is(clang::tok::identifier)) { + return token.getIdentifierInfo()->getName().str(); + } + } + + lexer.Lex(token); + } + + return ""; +} + +llvm::Expected scanModule(CompilationParams& params) { + struct ModuleCollector : public clang::PPCallbacks { + ModuleInfo& info; + + ModuleCollector(ModuleInfo& info) : info(info) {} + + void moduleImport(clang::SourceLocation importLoc, + clang::ModuleIdPath path, + const clang::Module* imported) override { + assert(path.size() == 1); + info.mods.emplace_back(path[0].first->getName()); + } + }; + + ModuleInfo info; + clang::PreprocessOnlyAction action; + auto instance = createInstance(params); + + if(!action.BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { + return error("Failed to begin source file"); + } + + auto& pp = instance->getPreprocessor(); + + pp.addPPCallbacks(std::make_unique(info)); + + if(auto error = action.Execute()) { + return error; + } + + if(pp.isInNamedModule()) { + info.isInterfaceUnit = pp.isInNamedInterfaceUnit(); + info.name = pp.getNamedModuleName(); + } + + return info; +} + +llvm::Expected compile(CompilationParams& params) { auto instance = createInstance(params); applyPreamble(*instance, params); - return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly); + return ExecuteAction(std::move(instance), std::make_unique()); } -llvm::Expected compile(CompliationParams& params, PCHInfo& out) { +llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer) { + auto instance = createInstance(params); + + /// Set options to run code completion. + instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str(); + instance->getFrontendOpts().CodeCompletionAt.Line = params.line; + instance->getFrontendOpts().CodeCompletionAt.Column = params.column; + instance->setCodeCompletionConsumer(consumer); + + applyPreamble(*instance, params); + + return ExecuteAction(std::move(instance), std::make_unique()); +} + +llvm::Expected compile(CompilationParams& params, PCHInfo& out) { assert(params.bounds.has_value() && "Preamble bounds is required to build PCH"); auto instance = createInstance(params); @@ -268,25 +370,27 @@ llvm::Expected compile(CompliationParams& params, PCHInfo& out) { instance->getPreprocessorOpts().GeneratePreamble = true; instance->getLangOpts().CompilingPCH = true; - if(auto info = ExecuteAction(std::move(instance), clang::frontend::ActionKind::GeneratePCH)) { - out.path = params.outPath.str(); - out.srcPath = params.srcPath.str(); - - auto& bounds = *params.bounds; - out.preamble = params.content.substr(0, bounds.Size).str(); - if(bounds.PreambleEndsAtStartOfLine) { - out.preamble.append("@"); - } - - /// TODO: collect files involved in building this PCH. - - return std::move(*info); - } else { + auto info = ExecuteAction(std::move(instance), std::make_unique()); + if(!info) { return info.takeError(); } + + out.path = params.outPath.str(); + out.srcPath = params.srcPath.str(); + + auto& bounds = *params.bounds; + out.preamble = params.content.substr(0, bounds.Size).str(); + out.deps = info->deps(); + if(bounds.PreambleEndsAtStartOfLine) { + out.preamble.append("@"); + } + + /// TODO: collect files involved in building this PCH. + + return std::move(*info); } -llvm::Expected compile(CompliationParams& params, PCMInfo& out) { +llvm::Expected compile(CompilationParams& params, PCMInfo& out) { auto instance = createInstance(params); /// Set options to generate PCM. @@ -295,29 +399,26 @@ llvm::Expected compile(CompliationParams& params, PCMInfo& out) { applyPreamble(*instance, params); - if(auto info = ExecuteAction(std::move(instance), - clang::frontend::ActionKind::GenerateReducedModuleInterface)) { - out.path = params.outPath.str(); - out.name = info->context().getCurrentNamedModule()->Name; - - return std::move(*info); - } else { + auto info = ExecuteAction(std::move(instance), + std::make_unique()); + if(!info) { return info.takeError(); } -} -llvm::Expected compile(CompliationParams& params, clang::CodeCompleteConsumer* consumer) { - auto instance = createInstance(params); + assert(info->pp().isInNamedInterfaceUnit() && + "Only module interface unit could be built as PCM"); - /// Set options to run code completion. - instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str(); - instance->getFrontendOpts().CodeCompletionAt.Line = params.line; - instance->getFrontendOpts().CodeCompletionAt.Column = params.column; - instance->setCodeCompletionConsumer(consumer); + out.isInterfaceUnit = true; + out.name = info->pp().getNamedModuleName(); + for(auto& [name, path]: params.pcms) { + out.mods.emplace_back(name); + } - applyPreamble(*instance, params); + out.path = params.outPath.str(); + out.srcPath = params.srcPath.str(); + out.deps = info->deps(); - return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly); + return std::move(*info); } } // namespace clice diff --git a/src/Compiler/Dependency.cpp b/src/Compiler/Dependency.cpp deleted file mode 100644 index 861fbaa4..00000000 --- a/src/Compiler/Dependency.cpp +++ /dev/null @@ -1,42 +0,0 @@ - -#include -#include - -namespace clice::dependencies { - -namespace { - -/// module name -> file path. -llvm::StringMap moduleMap; -llvm::BumpPtrAllocator allocator; - -void scan() { - using namespace clang::tooling::dependencies; - DependencyScanningService service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::P1689); - // TODO: figure out vfs - DependencyScanningTool tool(service); - - // TODO: figure out - clang::tooling::CompileCommand command; - auto rule = tool.getP1689ModuleDependencyFile(command, "CWD"); -} - -} // namespace - -void load(llvm::ArrayRef dirs) { - for(auto dir: dirs) { - - std::string message; - auto CDB = clang::tooling::CompilationDatabase::loadFromDirectory(dir, message); - if(!CDB) {} - // TODO: - // remove unused compile commands. - // scan the whole project(if it is module file scan it). - // record include graph?? unsure - // use BumpPtrAllocator to store the commands for - // - reduce memory usage - // - expose `std::vector` to `Compiler::Invocation` - } -} - -} // namespace clice::dependencies diff --git a/src/Compiler/Semantic.cpp b/src/Compiler/Semantic.cpp index 3611ff17..ab8df1ab 100644 --- a/src/Compiler/Semantic.cpp +++ b/src/Compiler/Semantic.cpp @@ -35,6 +35,8 @@ SymbolKind SymbolKind::from(const clang::Decl* decl) { } } -SymbolKind SymbolKind::from(const clang::tok::TokenKind kind) {} +SymbolKind SymbolKind::from(const clang::tok::TokenKind kind) { + return {}; +} } // namespace clice diff --git a/src/Feature/CodeCompletion.cpp b/src/Feature/CodeCompletion.cpp index a241ffd7..0435f8d0 100644 --- a/src/Feature/CodeCompletion.cpp +++ b/src/Feature/CodeCompletion.cpp @@ -200,7 +200,7 @@ json::Value capability(json::Value clientCapabilities) { }; } -proto::CompletionResult codeCompletion(CompliationParams& params, +proto::CompletionResult codeCompletion(CompilationParams& params, uint32_t line, uint32_t column, llvm::StringRef file, diff --git a/src/Server/Async.cpp b/src/Server/Async.cpp index 8338a2cd..e6022510 100644 --- a/src/Server/Async.cpp +++ b/src/Server/Async.cpp @@ -75,6 +75,39 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) { } } +/// Write a JSON value to the client. +void write(json::Value value) { + struct Buffer { + uv_write_t req; + llvm::SmallString<128> header; + llvm::SmallString<4096> message; + }; + + Buffer* buffer = new Buffer(); + buffer->req.data = buffer; + + llvm::raw_svector_ostream os(buffer->message); + os << value; + + llvm::raw_svector_ostream sos(buffer->header); + sos << "Content-Length: " << buffer->message.size() << "\r\n\r\n"; + + uv_buf_t bufs[2] = { + uv_buf_init(buffer->header.data(), buffer->header.size()), + uv_buf_init(buffer->message.data(), buffer->message.size()), + }; + + auto on_write = [](uv_write_t* req, int status) { + if(status < 0) { + log::fatal("An error occurred while writing: {0}", uv_strerror(status)); + } + + delete static_cast(req->data); + }; + + uv_check_call(uv_write, &buffer->req, writer, bufs, 2, on_write); +} + } // namespace void start_server(Callback callback) { @@ -126,42 +159,45 @@ void start_server(Callback callback, const char* ip, unsigned int port) { uv_check_call(uv_run, async::loop, UV_RUN_DEFAULT); } -void write(json::Value id, json::Value result) { - json::Value response = json::Object{ - {"jsonrpc", "2.0" }, - {"id", id }, - {"result", result}, - }; +/// Send a request to the client. +void request(llvm::StringRef method, json::Value params) { + static std::uint32_t id = 0; + write(json::Object{ + {"jsonrpc", "2.0" }, + {"id", id += 1 }, + {"method", method }, + {"params", std::move(params)}, + }); +} - struct Buffer { - uv_write_t req; - llvm::SmallString<128> header; - llvm::SmallString<4096> message; - }; +/// Send a notification to the client. +void notify(llvm::StringRef method, json::Value params) { + write(json::Object{ + {"jsonrpc", "2.0" }, + {"method", method }, + {"params", std::move(params)}, + }); +} - Buffer* buffer = new Buffer(); - buffer->req.data = buffer; +void response(json::Value id, json::Value result) { + write(json::Object{ + {"jsonrpc", "2.0" }, + {"id", id }, + {"result", std::move(result)}, + }); +} - llvm::raw_svector_ostream os(buffer->message); - os << response; - - llvm::raw_svector_ostream sos(buffer->header); - sos << "Content-Length: " << buffer->message.size() << "\r\n\r\n"; - - uv_buf_t bufs[2] = { - uv_buf_init(buffer->header.data(), buffer->header.size()), - uv_buf_init(buffer->message.data(), buffer->message.size()), - }; - - auto on_write = [](uv_write_t* req, int status) { - if(status < 0) { - log::fatal("An error occurred while writing: {0}", uv_strerror(status)); - } - - delete static_cast(req->data); - }; - - uv_check_call(uv_write, &buffer->req, writer, bufs, 2, on_write); +/// Send an register capability to the client. +void registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions) { + request("client/registerCapability", + json::Object{ + {"registrations", + json::Array{json::Object{ + {"id", id}, + {"method", method}, + {"registerOptions", std::move(registerOptions)}, + }}}, + }); } } // namespace clice::async diff --git a/src/Server/Command.cpp b/src/Server/Command.cpp deleted file mode 100644 index ec982285..00000000 --- a/src/Server/Command.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include "Server/Command.h" -#include "Server/Config.h" -#include "Server/Logger.h" - -namespace clice { - -void CommandManager::update(llvm::StringRef dir) { - llvm::SmallString<128> path; - path::append(path, dir, "compile_commands.json"); - - auto buffer = llvm::MemoryBuffer::getFile(path); - if(!buffer) { - log::warn("Failed to read compile_commands.json from {}, because {}", - dir, - buffer.getError().message()); - return; - } - - auto json = json::parse(buffer.get()->getBuffer()); - if(!json) { - log::warn("Failed to parse json file at {}, because {}", path, json.takeError()); - return; - } - - if(!json->getAsArray()) { - log::warn("Invalid JSON format at {}, Compilation Database requires Array, but get {}", - path, - refl::enum_name(json->kind())); - return; - } - - auto& CDB = CDBs[dir]; - - /// Clear old commands. - /// FIXME: it would be better to have cache in some way. - CDB.clear(); - - for(auto& value: *json->getAsArray()) { - auto element = value.getAsObject(); - if(!element) { - log::warn("Invalid JSON format at {}, Compilation Database requires Object, but get {}", - path, - refl::enum_name(value.kind())); - continue; - } - - /// Source file path. - llvm::SmallString<128> path; - - auto file = element->getString("file"); - if(!file) { - log::warn("the element in {} does not have a file field", - path, - refl::enum_name(value.kind())); - continue; - } - - if(path::is_relative(file.value())) { - auto working = element->getString("directory"); - if(!working) { - log::warn( - "Invalid JSON format at {}, {} is relative path, but directory is not provided", - path, - file.value()); - } - - path::append(path, working.value(), file.value()); - } else { - path::append(path, file.value()); - } - - /// Command to compile the source file. - llvm::SmallString<1024> rawCommand; - - if(auto command = element->getString("command")) { - rawCommand = command.value(); - } else if(auto arguments = element->getArray("arguments")) { - /// FIXME: - std::terminate(); - } else { - log::warn( - "Invalid JSON format, the element in {} does not have a command or arguments field", - path, - refl::enum_name(value.kind())); - continue; - } - - CDB[path].emplace_back(rawCommand.str()); - } - - log::info("Compilation Database at {} is up-to-date", dir); -} - -llvm::StringRef CommandManager::lookupFirst(llvm::StringRef file) { - for(auto& [dir, cdb]: CDBs) { - auto iter = cdb.find(file); - if(iter != cdb.end()) { - return iter->second.front(); - } - } - - return {}; -} - -llvm::ArrayRef CommandManager::lookup(llvm::StringRef file) { - for(auto& [dir, cdb]: CDBs) { - /// FIXME: currently we directly return the first match. - /// it is better to provide a callback to handle all matches. - auto iter = cdb.find(file); - if(iter != cdb.end()) { - return iter->second; - } - } - - return {}; -} - -} // namespace clice diff --git a/src/Server/Document.cpp b/src/Server/Document.cpp index 31e08d38..dbd5f0b4 100644 --- a/src/Server/Document.cpp +++ b/src/Server/Document.cpp @@ -5,25 +5,24 @@ namespace clice { async::promise Server::onDidOpen(const proto::DidOpenTextDocumentParams& params) { auto path = URI::resolve(params.textDocument.uri); llvm::StringRef content = params.textDocument.text; - - co_await scheduler.add(path, content); + co_await scheduler.update(path, content, synchronizer); } async::promise Server::onDidChange(const proto::DidChangeTextDocumentParams& document) { auto path = URI::resolve(document.textDocument.uri); llvm::StringRef content = document.contentChanges[0].text; - co_await scheduler.update(path, content); + co_await scheduler.update(path, content, synchronizer); } async::promise Server::onDidSave(const proto::DidSaveTextDocumentParams& document) { auto path = URI::resolve(document.textDocument.uri); - co_await scheduler.save(path); + /// co_await scheduler.save(path); co_return; } async::promise Server::onDidClose(const proto::DidCloseTextDocumentParams& document) { auto path = URI::resolve(document.textDocument.uri); - co_await scheduler.close(path); + /// co_await scheduler.close(path); co_return; } diff --git a/src/Server/Feature.cpp b/src/Server/Feature.cpp index b8f73f95..d1a0efd6 100644 --- a/src/Server/Feature.cpp +++ b/src/Server/Feature.cpp @@ -90,10 +90,10 @@ async::promise Server::onDocumentSymbol(json::Value id, async::promise Server::onSemanticTokens(json::Value id, const proto::SemanticTokensParams& params) { auto path = URI::resolve(params.textDocument.uri); - auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) { - return feature::semanticTokens(compiler, ""); - }); - async::write(std::move(id), json::serialize(tokens)); + // auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) { + // return feature::semanticTokens(compiler, ""); + // }); + /// async::response(std::move(id), json::serialize(tokens)); co_return; } @@ -104,10 +104,10 @@ async::promise Server::onInlayHint(json::Value id, const proto::InlayHintP async::promise Server::onCodeCompletion(json::Value id, const proto::CompletionParams& params) { auto path = URI::resolve(params.textDocument.uri); - auto result = co_await scheduler.codeComplete(path, - params.position.line + 1, - params.position.character + 1); - async::write(std::move(id), json::serialize(result)); + // auto result = co_await scheduler.codeComplete(path, + // params.position.line + 1, + // params.position.character + 1); + // async::response(std::move(id), json::serialize(result)); co_return; } diff --git a/src/Server/Lifestyle.cpp b/src/Server/Lifestyle.cpp index 322b80c7..98bb1adb 100644 --- a/src/Server/Lifestyle.cpp +++ b/src/Server/Lifestyle.cpp @@ -6,12 +6,35 @@ async::promise Server::onInitialize(json::Value id, const proto::Initializ auto workplace = URI::resolve(params.workspaceFolders[0].uri); config::init(workplace); + if(!params.capabilities.workspace.didChangeWatchedFiles.dynamicRegistration) { + log::fatal( + "clice requires the client to support file event watching to monitor updates to CDB files"); + } + proto::InitializeResult result = {}; - async::write(std::move(id), json::serialize(result)); + async::response(std::move(id), json::serialize(result)); + + /// Load the compile commands from the workspace. + for(auto dir: config::frontend().compile_commands_directorys) { + synchronizer.sync(dir + "/compile_commands.json"); + } + co_return; } async::promise Server::onInitialized(const proto::InitializedParams& params) { + proto::DidChangeWatchedFilesRegistrationOptions options; + for(auto& dir: config::frontend().compile_commands_directorys) { + options.watchers.emplace_back(proto::FileSystemWatcher{ + dir + "/compile_commands.json", + }); + } + async::registerCapacity("watchedFiles", + "workspace/didChangeWatchedFiles", + json::serialize(options)); + + /// Load all information about PCHs and PCMs from disk. + scheduler.loadFromDisk(); co_return; } @@ -20,6 +43,8 @@ async::promise Server::onExit(const proto::None&) { } async::promise Server::onShutdown(json::Value id, const proto::None&) { + /// Save all information about PCHs and PCMs to disk. + scheduler.saveToDisk(); co_return; } diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index 8ae1452e..18dcef12 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -4,185 +4,276 @@ namespace clice { -async::promise Scheduler::updatePCH(llvm::StringRef filepath, - llvm::StringRef content, - llvm::StringRef command) { - auto [iter, success] = pchs.try_emplace(filepath); - if(success || iter->second.needUpdate(content)) { - Tracer tracer; +static std::string getPCHOutPath(llvm::StringRef srcPath) { + llvm::SmallString<128> outPath = srcPath; + path::replace_path_prefix(outPath, config::workplace(), config::frontend().cache_directory); + path::replace_extension(outPath, ".pch"); - CompliationParams params; - params.content = content; - params.srcPath = filepath; - params.outPath = filepath; - params.command = command; - - path::replace_path_prefix(params.outPath, - config::workplace(), - config::frontend().cache_directory); - path::replace_extension(params.outPath, ".pch"); - - log::info("Start building PCH for {0} at {1}", params.srcPath, params.outPath); - - PCHInfo pch; - - co_await async::schedule_task([&] { - auto dir = path::parent_path(params.outPath); - if(!fs::exists(dir)) { - if(auto error = fs::create_directories(dir)) { - log::fatal("Failed to create directory {0}, because {1}, build PCH stopped", - dir, - error.message()); - return; - } - } - - if(auto info = compile(params, pch); !info) { - log::fatal("Failed to build PCH for {0}, because {1}", - filepath.str(), - info.takeError()); - return; - } - }); - - log::info("PCH for {0} is up-to-date, elapsed {1}ms", - filepath.str(), - tracer.duration().count()); - - pchs[filepath] = std::move(pch); - } else { - log::info("Reuse PCH for {0} from {1}", filepath.str(), iter->second.path); - } - co_return; -} - -async::promise Scheduler::buildAST(llvm::StringRef filepath, llvm::StringRef content) { - llvm::SmallString<128> path = filepath; - - auto [iter, success] = files.try_emplace(filepath); - if(!success && !iter->second.isIdle) { - /// If the file is already existed and is building, append the task to the waiting list. - co_await async::suspend([&](auto handle) { - iter->second.waitings.emplace_back(Task{ - .isBuild = true, - .waiting = handle, - }); - }); - } - - files[filepath].isIdle = false; - - static bool initCmd = false; - if(!initCmd) { - cmdMgr.update(config::frontend().compile_commands_directorys[0]); - initCmd = true; - } - - CompliationParams params; - params.srcPath = path; - params.content = content; - params.command = cmdMgr.lookupFirst(filepath); - params.addPCH(pchs.at(filepath)); - - /// through arguments to judge is it a module. - bool isModule = false; - co_await (isModule ? updatePCM() : updatePCH(filepath, content, params.command)); - - Tracer tracer; - - log::info("Start building AST for {0}, command: [{1}]", filepath, params.command.str()); - - auto task = [&] { - /// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be - /// Invalid Because other changed the `pchs` map. We also cannot to retrieve the `pch` from - /// the `pchs` map in this task, beacuse it is called in thread pool which will result in - /// data race. So temporarily copy the `pch` here. There must be a better way to solve this - /// problem. - auto info = clice::compile(params); - if(!info) { - log::fatal("Failed to build AST for {0}", filepath); + if(auto dir = path::parent_path(outPath); !fs::exists(dir)) { + if(auto error = fs::create_directories(dir)) { + log::fatal("Failed to create directory {0}, because {1}", dir, error.message()); } - return std::move(*info); - }; - - auto compiler = co_await async::schedule_task(std::move(task)); - - auto& file = files[path]; - file.content = content; - file.compiler = std::move(compiler); - - log::info("Build AST successfully for {0}, elapsed {1}", filepath, tracer.duration()); - - if(!file.waitings.empty()) { - auto task = std::move(file.waitings.front()); - async::schedule(task.waiting); - file.waitings.pop_front(); } - file.isIdle = true; + return outPath.str().str(); } -async::promise Scheduler::codeComplete(llvm::StringRef filepath, - unsigned int line, - unsigned int column) { - auto iter = files.find(filepath); - if(iter == files.end()) { - log::fatal("File {0} is not building, skip code completion", filepath); +static std::string getPCMOutPath(llvm::StringRef srcPath) { + llvm::SmallString<128> outPath = srcPath; + path::replace_path_prefix(outPath, config::workplace(), config::frontend().cache_directory); + path::replace_extension(outPath, ".pcm"); + + if(auto dir = path::parent_path(outPath); !fs::exists(dir)) { + if(auto error = fs::create_directories(dir)) { + log::fatal("Failed to create directory {0}, because {1}", dir, error.message()); + } } - if(iter->second.isIdle) { - /// If the file is already existed and is building, append the task to the waiting list. - co_await async::suspend([&](auto handle) { - iter->second.waitings.emplace_back(Task{ - .isBuild = true, - .waiting = handle, - }); - }); + return outPath.str().str(); +} + +async::promise<> Scheduler::updatePCH(CompilationParams& params, class Synchronizer& sync) { + llvm::StringRef srcPath = params.srcPath; + + auto& pch = pchs[srcPath]; + /// FIXME: judge need update here ... + if(!pch.needUpdate(params.content)) { + log::info("PCH for {0} is already up-to-date, reuse it", srcPath); + co_return; } - llvm::SmallString<128> path = filepath; - /// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1 + /// Construct the output path. + params.outPath = getPCHOutPath(srcPath); - CompliationParams params; - params.content = iter->second.content; - params.srcPath = path; - params.command = cmdMgr.lookupFirst(filepath); - - /// through arguments to judge is it a module. - bool isModule = false; - co_await (isModule ? updatePCM() : updatePCH(params.srcPath, params.content, params.command)); - params.addPCH(pchs.at(filepath)); + /// Build PCH. + PCHInfo info; Tracer tracer; - log::info("Run code completion at {0}:{1}:{2}", filepath, line, column); + log::info("Building PCH for {0}", srcPath); - auto task = [&] { - return feature::codeCompletion(params, line, column, filepath, {}); - }; + /// FIXME: consider header context. + params.computeBounds(); - auto result = co_await async::schedule_task(std::move(task)); + llvm::Error error = co_await async::schedule_task([&] -> llvm::Error { + auto result = compile(params, info); + if(!result) { + return result.takeError(); + } - log::info("Code completion for {0} is done, elapsed {1}", filepath, tracer.duration()); + /// FIXME: consider indexing PCH here. - co_return result; + return llvm::Error::success(); + }); + + pchs[srcPath] = std::move(info); + + if(error) { + log::warn("Failed to build PCH for {0}, because {1}", srcPath, error); + co_return; + } else { + log::info("PCH for {0} is up-to-date, elapsed {1}", srcPath, tracer.duration()); + } } -async::promise Scheduler::add(llvm::StringRef path, llvm::StringRef content) { - co_await buildAST(path, content); - co_return; +llvm::Error Scheduler::addModuleDeps(CompilationParams& params, + const ModuleInfo& moduleInfo) const { + for(auto& mod: moduleInfo.mods) { + auto iter = pcms.find(mod); + + if(iter == pcms.end()) { + return error("Cannot find PCM for module {0}", mod); + } + + /// Add prerequired PCM. + if(auto error = addModuleDeps(params, iter->second)) { + return error; + } + + params.addPCM(iter->second); + } + + return llvm::Error::success(); } -async::promise Scheduler::update(llvm::StringRef path, llvm::StringRef content) { - co_await buildAST(path, content); - co_return; +async::promise<> Scheduler::updatePCM(llvm::StringRef moduleName, class Synchronizer& sync) { + llvm::StringRef srcPath = sync.map(moduleName); + if(srcPath.empty()) { + log::warn("Cannot find source file for module {0}", moduleName); + co_return; + } + + auto& pcm = pcms[moduleName]; + if(!pcm.needUpdate()) { + log::info("PCM for {0} is already up-to-date, reuse it", srcPath); + } + + CompilationParams params; + params.srcPath = srcPath; + params.command = sync.lookup(srcPath); + params.outPath = getPCMOutPath(srcPath); + + auto moduleInfo = scanModule(params); + if(!moduleInfo) { + log::warn("Build AST for {0} failed, because {1}", srcPath, moduleInfo.takeError()); + co_return; + } + + /// Build prerequired PCM. + for(auto& mod: moduleInfo->mods) { + co_await updatePCM(mod, sync); + } + + /// Build PCM. + PCMInfo info; + + Tracer tracer; + log::info("Building PCM for {0}", srcPath); + + /// Add deps. + if(auto error = addModuleDeps(params, *moduleInfo)) { + log::warn("Failed to build PCM for {0}, because {1}", srcPath, error); + co_return; + } + + llvm::Error error = co_await async::schedule_task([&] -> llvm::Error { + auto result = compile(params, info); + if(!result) { + return result.takeError(); + } + + /// FIXME: consider indexing PCH here. + + return llvm::Error::success(); + }); + + pcms[moduleName] = std::move(info); + + if(error) { + log::warn("Failed to build PCM for {0}, because {1}", srcPath, error); + co_return; + } else { + log::info("PCM for {0} is up-to-date, elapsed {1}", srcPath, tracer.duration()); + } } -async::promise Scheduler::save(llvm::StringRef path) { - co_return; +async::promise<> Scheduler::update(llvm::StringRef filename, + llvm::StringRef content, + class Synchronizer& sync) { + CompilationParams params; + params.content = content; + params.srcPath = filename; + params.command = sync.lookup(filename); + + auto moduleInfo = scanModule(params); + if(!moduleInfo) { + log::warn("Build AST for {0} failed, because {1}", filename, moduleInfo.takeError()); + co_return; + } + + if(moduleInfo->name.empty() && moduleInfo->mods.empty()) { + co_await updatePCH(params, sync); + params.bounds.reset(); + params.addPCH(pchs[params.srcPath]); + } else { + for(auto& mod: moduleInfo->mods) { + co_await updatePCM(mod, sync); + } + + if(auto error = addModuleDeps(params, *moduleInfo)) { + log::warn("Failed to build PCM for {0}, because {1}", filename, error); + co_return; + } + } + + /// Build AST. + ASTInfo info; + + Tracer tracer; + + log::info("Building AST for {0}", filename); + + co_await async::schedule_task([&] { + auto result = compile(params); + if(!result) { + log::warn("Failed to build AST for {0}, because {1}", filename, result.takeError()); + return; + } + + info = std::move(*result); + }); + + /// Build AST successfully. + + log::info("AST for {0} is up-to-date, elapsed {1}", filename, tracer.duration()); } -async::promise Scheduler::close(llvm::StringRef path) { - co_return; +void Scheduler::loadFromDisk() { + llvm::SmallString<128> fileName; + path::append(fileName, config::frontend().cache_directory, "cache.json"); + + auto buffer = llvm::MemoryBuffer::getFile(fileName); + if(!buffer) { + log::warn("Failed to load cache from disk, because {0}", buffer.getError().message()); + return; + } + + auto json = json::parse(buffer.get()->getBuffer()); + if(!json) { + log::warn("Failed to parse cache from disk, because {0}", json.takeError()); + return; + } + + auto object = json->getAsObject(); + if(!object) { + log::warn("Failed to parse cache from disk, because {0}", json.takeError()); + return; + } + + if(auto pchArray = object->getArray("pch")) { + for(auto& value: *pchArray) { + auto pch = json::deserialize(value); + pchs[pch.srcPath] = std::move(pch); + } + } + + if(auto pcmArray = object->getArray("pcm")) { + for(auto& value: *pcmArray) { + auto pcm = json::deserialize(value); + pcms[pcm.name] = std::move(pcm); + } + } + + log::info("Cache loaded from {0}", fileName); +} + +void Scheduler::saveToDisk() const { + json::Object result; + + json::Array pchArray; + for(auto& [name, pch]: pchs) { + pchArray.emplace_back(json::serialize(pch)); + } + result.try_emplace("pch", std::move(pchArray)); + + json::Array pcmArray; + for(auto& [name, pcm]: pcms) { + pcmArray.emplace_back(json::serialize(pcm)); + } + result.try_emplace("pcm", std::move(pcmArray)); + + llvm::SmallString<128> fileName; + path::append(fileName, config::frontend().cache_directory, "cache.json"); + + std::error_code EC; + llvm::raw_fd_ostream stream(fileName, EC, llvm::sys::fs::OF_Text); + + if(EC) { + log::warn("Failed save cache to disk, because {0}", EC.message()); + return; + } + + stream << json::Value(std::move(result)); + log::info("Cache saved to {0}", fileName); } } // namespace clice diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index d8b9fa13..a08a16be 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -100,6 +100,8 @@ Server::Server() { addMethod("textDocument/formatting", &Server::onFormatting); addMethod("textDocument/rangeFormatting", &Server::onRangeFormatting); + addMethod("workspace/didChangeWatchedFiles", &Server::onDidChangeWatchedFiles); + addMethod("context/current", &Server::onContextCurrent); addMethod("context/switch", &Server::onContextSwitch); addMethod("context/all", &Server::onContextAll); diff --git a/src/Server/Synchronizer.cpp b/src/Server/Synchronizer.cpp new file mode 100644 index 00000000..635abfb7 --- /dev/null +++ b/src/Server/Synchronizer.cpp @@ -0,0 +1,118 @@ +#include "Compiler/Compiler.h" +#include "Server/Logger.h" +#include "Server/Synchronizer.h" + +namespace clice { + +void Synchronizer::sync(llvm::StringRef filename) { + /// Read the compile commands from the file. + json::Value json = nullptr; + + if(auto buffer = llvm::MemoryBuffer::getFile(filename)) { + if(auto result = json::parse(buffer->get()->getBuffer())) { + /// llvm::json::Value will hold on string buffer. + /// Do not worry about the lifetime of the buffer. + /// Release buffer to save memory. + json = std::move(result.get()); + } else { + log::warn("Failed to parse json file at {0}, because {1}", + filename, + result.takeError()); + return; + } + } else { + log::warn("Failed to read file {0}", filename); + return; + } + + assert(json.kind() != json::Value::Null && "json is nullptr"); + + if(json.kind() != json::Value::Array) { + log::warn("Compilation Database requires a array of object, but get {0}, input file: {1}", + refl::enum_name(json.kind()), + filename); + return; + } + + auto elements = json.getAsArray(); + assert(elements && "json is not an array"); + + for(auto& element: *elements) { + auto object = element.getAsObject(); + if(!object) { + log::warn( + "Compilation Database requires an array of object, but get a array of {0}, input file: {1}", + refl::enum_name(element.kind()), + filename); + continue; + } + + /// FIXME: currently we assume all path here is absolute. + /// Add `directory` field in the future. + + llvm::SmallString<128> path; + + if(auto file = object->getString("file")) { + if(auto error = fs::real_path(*file, path)) { + log::warn("Failed to get real path of {0}, because {1}", *file, error.message()); + continue; + } + } else { + log::warn("The element does not have a file field, input file: {0}", filename); + continue; + } + + auto command = object->getString("command"); + if(!command) { + log::warn("The key:{0} does not have a command field, input file: {1}", path, filename); + continue; + } + + commands[path] = *command; + } + + log::info("Successfully loaded compile commands from {0}, total {1} commands", + filename, + commands.size()); + + /// Scan all files to build module map. + CompilationParams params; + for(auto& [path, command]: commands) { + params.srcPath = path; + params.command = command; + auto info = scanModule(params); + if(!info) { + log::warn("Failed to scan module from {0}, because {1}", path, info.takeError()); + continue; + } + + if(info->isInterfaceUnit) { + assert(!info->name.empty() && "module name is empty"); + moduleMap[info->name] = path; + } + } + + log::info("Successfully built module map, total {0} modules", moduleMap.size()); +} + +void Synchronizer::sync(llvm::StringRef name, llvm::StringRef path) { + moduleMap[name] = path; +} + +llvm::StringRef Synchronizer::lookup(llvm::StringRef file) const { + auto iter = commands.find(file); + if(iter == commands.end()) { + return ""; + } + return iter->second; +} + +llvm::StringRef Synchronizer::map(llvm::StringRef name) const { + auto iter = moduleMap.find(name); + if(iter == moduleMap.end()) { + return ""; + } + return iter->second; +} + +} // namespace clice diff --git a/src/Server/Workplace.cpp b/src/Server/Workplace.cpp new file mode 100644 index 00000000..d8475dc0 --- /dev/null +++ b/src/Server/Workplace.cpp @@ -0,0 +1,27 @@ +#include "Server/Server.h" + +namespace clice { + +async::promise<> Server::onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params) { + for(auto& event: params.changes) { + switch(event.type) { + case proto::FileChangeType::Created: { + break; + } + + case proto::FileChangeType::Changed: { + auto path = URI::resolve(event.uri); + synchronizer.sync(path); + break; + } + + case proto::FileChangeType::Deleted: { + break; + } + } + } + + co_return; +} + +} // namespace clice diff --git a/unittests/Basic/URI.cpp b/unittests/Basic/URI.cpp index 344889fa..7b47ccf5 100644 --- a/unittests/Basic/URI.cpp +++ b/unittests/Basic/URI.cpp @@ -5,7 +5,7 @@ namespace clice { namespace { -TEST(URITest, ConstructorAndAccessors) { +TEST(URI, Basic) { URI uri("https", "reviews.llvm.org", "/D41946"); EXPECT_EQ(uri.scheme(), "https"); @@ -13,7 +13,7 @@ TEST(URITest, ConstructorAndAccessors) { EXPECT_EQ(uri.body(), "/D41946"); } -TEST(URITest, CopyConstructor) { +TEST(URI, Copy) { URI uri1("https", "reviews.llvm.org", "/D41946"); URI uri2(uri1); @@ -22,7 +22,7 @@ TEST(URITest, CopyConstructor) { EXPECT_EQ(uri2.body(), "/D41946"); } -TEST(URITest, EqualityOperator) { +TEST(URI, Eq) { URI uri1("https", "reviews.llvm.org", "/D41946"); URI uri2("https", "reviews.llvm.org", "/D41946"); URI uri3("http", "example.com", "/index.html"); @@ -31,11 +31,19 @@ TEST(URITest, EqualityOperator) { EXPECT_FALSE(uri1 == uri3); } -TEST(URITest, ParseFunction) { - auto expectedUri = URI::parse("https://reviews.llvm.org/D41946"); - ASSERT_TRUE(static_cast(expectedUri)); +TEST(URI, File) { + auto uri = URI::from("/home/user/file.txt"); + EXPECT_EQ(uri.scheme(), "file"); + EXPECT_EQ(uri.authority(), ""); + EXPECT_EQ(uri.body(), "/home/user/file.txt"); + EXPECT_EQ(uri.toString(), "file:///home/user/file.txt"); +} - URI uri = expectedUri.get(); +TEST(URI, Parse) { + auto expectedUri = URI::parse("https://reviews.llvm.org/D41946"); + ASSERT_TRUE(bool(expectedUri)); + + URI& uri = expectedUri.get(); EXPECT_EQ(uri.scheme(), "https"); EXPECT_EQ(uri.authority(), "reviews.llvm.org"); EXPECT_EQ(uri.body(), "/D41946"); diff --git a/unittests/Compiler/Compiler.cpp b/unittests/Compiler/Compiler.cpp index fa59fc98..384d9c95 100644 --- a/unittests/Compiler/Compiler.cpp +++ b/unittests/Compiler/Compiler.cpp @@ -16,7 +16,7 @@ int main(){ } )cpp"; - CompliationParams params; + CompilationParams params; params.content = code; params.srcPath = "main.cpp"; params.command = "clang++ -std=c++20 main.cpp"; @@ -34,7 +34,7 @@ int main(){ })cpp"; /// Test in no header file. - CompliationParams params; + CompilationParams params; params.content = code; params.srcPath = "main.cpp"; params.command = "clang++ -std=c++20 main.cpp"; @@ -91,7 +91,7 @@ int main(){ return; } - CompliationParams params; + CompilationParams params; params.content = code; params.srcPath = "main.cpp"; params.outPath = outpath; @@ -128,7 +128,7 @@ export int foo() { return; } - CompliationParams params; + CompilationParams params; params.srcPath = "main.cppm"; params.content = code; params.outPath = outpath; @@ -162,7 +162,7 @@ export module A; export int foo = 1; )cpp"; - CompliationParams params; + CompilationParams params; params.content = code; params.srcPath = "main.cppm"; params.command = "clang++ -std=c++20 main.cppm"; diff --git a/unittests/Compiler/Module.cpp b/unittests/Compiler/Module.cpp new file mode 100644 index 00000000..a6aee260 --- /dev/null +++ b/unittests/Compiler/Module.cpp @@ -0,0 +1,113 @@ +#include "gtest/gtest.h" +#include "Compiler/Compiler.h" + +#include "llvm/Support/ToolOutputFile.h" + +namespace clice { + +namespace { + +PCMInfo buildPCM(llvm::StringRef file, llvm::StringRef code) { + llvm::SmallString<128> outPath; + fs::createUniquePath(llvm::Twine(file) + "%%%%%%.pcm", outPath, true); + + CompilationParams params; + params.content = code; + params.srcPath = file; + params.outPath = outPath; + params.command = "clang++ -std=c++20 -x c++ " + file.str(); + params.remappedFiles.emplace_back("./test.h", "export int foo2();"); + + PCMInfo pcm; + if(!compile(params, pcm)) { + llvm::errs() << "Failed to build PCM\n"; + std::terminate(); + } + + return pcm; +} + +ModuleInfo scan(llvm::StringRef content) { + CompilationParams params; + params.content = content; + params.srcPath = "main.ixx"; + params.command = "clang++ -std=c++20 -x c++ main.ixx"; + params.remappedFiles.emplace_back("./test.h", "export module A"); + auto info = scanModule(params); + if(!info) { + llvm::errs() << "Failed to scan module\n"; + std::terminate(); + } + return std::move(*info); +} + +TEST(Module, Scan) { + /// Simple case. + const char* content = R"( +export module A; +import B; + )"; + auto info = scan(content); + ASSERT_EQ(info.isInterfaceUnit, true); + ASSERT_EQ(info.name, "A"); + ASSERT_EQ(info.mods.size(), 1); + ASSERT_EQ(info.mods[0], "B"); + + /// With global module fragment and private module fragment. + content = R"( +module; +#include +export module A; +import B; +import C; +module : private; +)"; + info = scan(content); + ASSERT_EQ(info.isInterfaceUnit, true); + ASSERT_EQ(info.name, "A"); + ASSERT_EQ(info.mods.size(), 2); + ASSERT_EQ(info.mods[0], "B"); + ASSERT_EQ(info.mods[1], "C"); + + /// With module partition. + content = R"( +module; +#include +export module A:B; +import B; +import C; +module : private; +)"; + info = scan(content); + ASSERT_EQ(info.isInterfaceUnit, true); + ASSERT_EQ(info.name, "A:B"); + ASSERT_EQ(info.mods.size(), 2); + ASSERT_EQ(info.mods[0], "B"); + ASSERT_EQ(info.mods[1], "C"); + + content = R"( +module A; +import B; +import C; +)"; + info = scan(content); + ASSERT_EQ(info.isInterfaceUnit, false); + ASSERT_EQ(info.name, "A"); + ASSERT_EQ(info.mods.size(), 2); + ASSERT_EQ(info.mods[0], "B"); + ASSERT_EQ(info.mods[1], "C"); +} + +TEST(Module, Normal) { + const char* content = R"( +export module A; +)"; + auto pcm = buildPCM("A.ixx", content); + // ASSERT_EQ(pcm.isInterfaceUnit, true); + // ASSERT_EQ(pcm.name, "A"); + // ASSERT_EQ(pcm.mods.size(), 0); +} + +} // namespace + +} // namespace clice diff --git a/unittests/Feature/CodeCompletion.cpp b/unittests/Feature/CodeCompletion.cpp index ff226187..d341d8ec 100644 --- a/unittests/Feature/CodeCompletion.cpp +++ b/unittests/Feature/CodeCompletion.cpp @@ -14,7 +14,7 @@ int main() { } )cpp"; - CompliationParams params; + CompilationParams params; params.content = code; params.srcPath = "main.cpp"; params.command = "clang++ -std=c++20 main.cpp"; diff --git a/unittests/Test.h b/unittests/Test.h index d589d155..3848d1a9 100644 --- a/unittests/Test.h +++ b/unittests/Test.h @@ -149,7 +149,7 @@ inline void EXPECT_NE(const LHS& lhs, class Tester { public: - CompliationParams params; + CompilationParams params; std::unique_ptr vfs; ASTInfo info;