From 7874fbacb813ffcfaf26ea924d21ccdb0078125c Mon Sep 17 00:00:00 2001 From: ykiko Date: Fri, 11 Apr 2025 22:22:23 +0800 Subject: [PATCH] Implement text document sync (#119) --- include/Async/ThreadPool.h | 12 +- include/Compiler/Compilation.h | 14 +- include/Compiler/Preamble.h | 3 + include/Feature/CodeCompletion.h | 52 ++++- include/Server/DocumentManager.h | 18 ++ include/Server/LSPConverter.h | 10 +- include/Server/Protocol.h | 12 ++ include/Server/Scheduler.h | 73 +++++-- include/Server/Server.h | 41 ++-- include/Support/Format.h | 5 +- include/Test/CTest.h | 2 +- src/Compiler/Compilation.cpp | 35 +++- src/Feature/CodeCompletion.cpp | 139 +++++++++---- src/Server/IncludeGraph.cpp | 6 +- src/Server/LSPConverter.cpp | 48 ++++- src/Server/Scheduler.cpp | 188 ++++++++++++++++++ src/Server/Server.cpp | 282 ++++++++++++--------------- unittests/Async/Event.cpp | 28 +++ unittests/Async/ThreadPool.cpp | 12 +- unittests/Compiler/Module.cpp | 6 +- unittests/Compiler/Preamble.cpp | 4 +- unittests/Feature/CodeCompletion.cpp | 16 +- unittests/Feature/SignatureHelp.cpp | 14 +- 23 files changed, 744 insertions(+), 276 deletions(-) create mode 100644 include/Server/DocumentManager.h create mode 100644 src/Server/Scheduler.cpp diff --git a/include/Async/ThreadPool.h b/include/Async/ThreadPool.h index 4d99e82d..52d2f4db 100644 --- a/include/Async/ThreadPool.h +++ b/include/Async/ThreadPool.h @@ -46,9 +46,15 @@ struct thread_pool : value, uv, uv_work_t, Ret, int> } // namespace awaiter -template ()())> -auto submit(Work&& work) { - return awaiter::thread_pool, Ret>{{}, {}, std::forward(work)}; +template > +async::Task submit(Work&& work) { + using W = std::remove_cvref_t; + auto result = co_await awaiter::thread_pool{{}, {}, std::forward(work)}; + if(!result) { + /// Thread pool task should never fails. + std::abort(); + } + co_return std::move(*result); } } // namespace clice::async diff --git a/include/Compiler/Compilation.h b/include/Compiler/Compilation.h index 727439d1..6fae2b64 100644 --- a/include/Compiler/Compilation.h +++ b/include/Compiler/Compilation.h @@ -28,17 +28,21 @@ struct CompilationParams { llvm::IntrusiveRefCntPtr vfs = new ThreadSafeFS(); - /// Remapped files. Currently, this is only used for testing. - llvm::SmallVector> remappedFiles; - /// Information about reuse PCH. std::pair pch; /// Information about reuse PCM(name, path). llvm::StringMap pcms; - /// Code completion file:line:column. - std::tuple completion; + /// Code completion file:offset. + std::tuple completion; + + /// The memory buffers for all remapped file. + llvm::StringMap> buffers; + + void addRemappedFile(llvm::StringRef path, llvm::StringRef content) { + buffers.try_emplace(path, llvm::MemoryBuffer::getMemBufferCopy(content)); + } }; namespace impl { diff --git a/include/Compiler/Preamble.h b/include/Compiler/Preamble.h index 4811be20..9fe2fe64 100644 --- a/include/Compiler/Preamble.h +++ b/include/Compiler/Preamble.h @@ -13,6 +13,9 @@ class ASTInfo; struct CompilationParams; struct PCHInfo { + /// The building time. + std::int64_t mtime; + /// The path of the output PCH file. std::string path; diff --git a/include/Feature/CodeCompletion.h b/include/Feature/CodeCompletion.h index 81e71e32..7cc4c59a 100644 --- a/include/Feature/CodeCompletion.h +++ b/include/Feature/CodeCompletion.h @@ -3,6 +3,7 @@ #include #include +#include "AST/SourceCode.h" #include "llvm/ADT/StringRef.h" namespace clice { @@ -17,9 +18,56 @@ struct CodeCompletionOption {}; namespace feature { -struct CodeCompletionItem {}; +enum class CompletionItemKind { + None = 0, + Text, + Method, + Function, + Constructor, + Field, + Variable, + Class, + Interface, + Module, + Property, + Unit, + Value, + Enum, + Keyword, + Snippet, + Color, + File, + Reference, + Folder, + EnumMember, + Constant, + Struct, + Event, + Operator, + TypeParameter +}; -using CodeCompletionResult = std::vector; +struct CompletionItem { + /// The label displayed when user select the item. + std::string label; + + std::string detail; + + /// TODO: + /// std::string sortText; + + CompletionItemKind kind; + + bool deprecated; + + struct Edit { + std::string text; + + LocalSourceRange range; + } edit; +}; + +using CodeCompletionResult = std::vector; CodeCompletionResult codeCompletion(CompilationParams& params, const config::CodeCompletionOption& option); diff --git a/include/Server/DocumentManager.h b/include/Server/DocumentManager.h new file mode 100644 index 00000000..3b063bd9 --- /dev/null +++ b/include/Server/DocumentManager.h @@ -0,0 +1,18 @@ +#pragma once + +#include "llvm/ADT/StringMap.h" + +namespace clice { + +class Document {}; + +/// Responsible for all opened files. +class DocumentManager { +public: + +private: + /// TODO: Use an LRU for this. + llvm::StringMap documents; +}; + +} // namespace clice diff --git a/include/Server/LSPConverter.h b/include/Server/LSPConverter.h index a495ec99..0f6342fc 100644 --- a/include/Server/LSPConverter.h +++ b/include/Server/LSPConverter.h @@ -9,6 +9,8 @@ #include "Feature/DocumentLink.h" #include "Feature/DocumentSymbol.h" #include "Feature/SemanticToken.h" +#include "Feature/CodeCompletion.h" +#include "Feature/SignatureHelp.h" namespace clice { @@ -32,11 +34,13 @@ public: } public: + /// Convert URI to file path with path mapping. + std::string convert(llvm::StringRef URI); + /// Convert a position into an offset relative to the beginning of the file. std::uint32_t convert(llvm::StringRef content, proto::Position position); - /// Convert `TextDocumentParams` to file path. - std::string convert(proto::TextDocumentParams params); + proto::Position convert(llvm::StringRef content, std::uint32_t offset); json::Value convert(llvm::StringRef content, const feature::Hover& hover); @@ -50,6 +54,8 @@ public: json::Value convert(llvm::StringRef content, const feature::SemanticTokens& tokens); + json::Value convert(llvm::StringRef content, const std::vector& items); + private: PositionEncodingKind kind; std::string workspacePath; diff --git a/include/Server/Protocol.h b/include/Server/Protocol.h index 442241c8..a2f968ab 100644 --- a/include/Server/Protocol.h +++ b/include/Server/Protocol.h @@ -83,6 +83,13 @@ struct TextDocumentIdentifier { DocumentUri uri; }; +struct VersionedTextDocumentIdentifier { + /// The text document's URI. + DocumentUri uri; + + std::uint32_t version; +}; + struct TextDocumentPositionParams { /// The text document. TextDocumentIdentifier textDocument; @@ -179,6 +186,11 @@ struct SemanticTokenOptions { bool full = true; }; +struct CompletionOptions { + std::vector triggerCharacters = {".", "<", ">", ":", "\"", "/", "*"}; + bool resolveProvider = false; +}; + struct HeaderContext { /// The path of context file. std::string file; diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index 38f59afb..b9bc1c42 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -1,35 +1,72 @@ #pragma once #include "Async/Async.h" -#include "Compiler/Command.h" -#include "Feature/CodeCompletion.h" -#include "Feature/SignatureHelp.h" +#include "Compiler/AST.h" +#include "Compiler/Module.h" +#include "Compiler/Preamble.h" namespace clice { -class ASTInfo; +class LSPConverter; +class CompilationDatabase; + +struct OpenFile { + /// The file version, every edition will increase it. + std::uint32_t version = 0; + + /// The file content. + std::string content; + + /// We build PCH for every opened file. + std::optional PCH; + async::Task<> PCHBuild; + async::Event PCHBuiltEvent; + + /// For each opened file, we would like to build an AST for it. + std::shared_ptr AST; + async::Task<> ASTBuild; + async::Lock ASTBuiltLock; + + /// For header with context, it may have multiple ASTs, use + /// an chain to store them. + std::unique_ptr next; +}; class Scheduler { public: - /// Build the given source file with given content. If there is not - /// preamble for it, build the preamble first. - async::Task build(llvm::StringRef file, llvm::StringRef content); + Scheduler(LSPConverter& converter, CompilationDatabase& database) : + converter(converter), database(database) {} - /// Build the given source directly without preamble. - async::Task build(llvm::StringRef file); + /// Add or update a document. + void addDocument(std::string path, std::string content); - async::Task codeCompletion(llvm::StringRef file, - llvm::StringRef content, - std::uint32_t line, - std::uint32_t column); + /// Close a document. + void closeDocument(std::string path); - async::Task signatureHelp(llvm::StringRef file, - llvm::StringRef content, - std::uint32_t line, - std::uint32_t column); + llvm::StringRef getDocumentContent(llvm::StringRef path); + + /// Get the specific AST of given file. + async::Task semanticToken(std::string path); + + async::Task completion(std::string path, std::uint32_t offset); private: - CompilationDatabase database; + async::Task isPCHOutdated(llvm::StringRef file, llvm::StringRef preamble); + + async::Task<> buildPCH(std::string file, std::string preamble); + + async::Task<> buildAST(std::string file, std::string content); + +private: + LSPConverter& converter; + CompilationDatabase& database; + + /// The task that runs in the thread pool. The number of tasks is fixed, + /// and we won't attempt to expand the vector, so the references are + /// guaranteed to remain valid. + std::vector> running; + + llvm::StringMap openFiles; }; } // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index aed8ae16..36ff6035 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -3,8 +3,8 @@ #include "Config.h" #include "Indexer.h" #include "Protocol.h" +#include "Scheduler.h" #include "LSPConverter.h" - #include "Async/Async.h" #include "Compiler/Command.h" @@ -16,24 +16,6 @@ public: async::Task<> onReceive(json::Value value); - /// Handle requests, a request must have a response. - async::Task onRequest(llvm::StringRef method, json::Value value); - - /// Handle requests started with `textDocument/`. - async::Task onTextDocument(llvm::StringRef method, json::Value value); - - /// Handle requests started with `context/`. - async::Task onContext(llvm::StringRef method, json::Value value); - - /// Handle requests started with `index/`. - async::Task onIndex(llvm::StringRef method, json::Value value); - - /// Handle notifications, a notification doesn't require response. - async::Task<> onNotification(llvm::StringRef method, json::Value value); - - /// Handle notifications `context/ - async::Task<> onFileOperation(llvm::StringRef method, json::Value value); - private: /// Send a request to the client. async::Task<> request(llvm::StringRef method, json::Value params); @@ -52,12 +34,31 @@ private: json::Value registerOptions); private: + async::Task onInitialize(json::Value value); -public: + async::Task<> onDidOpen(json::Value value); + + async::Task<> onDidChange(json::Value value); + + async::Task<> onDidSave(json::Value value); + + async::Task<> onDidClose(json::Value value); + + async::Task onSemanticToken(json::Value value); + + async::Task onCodeCompletion(json::Value value); + +private: std::uint32_t id = 0; Indexer indexer; + Scheduler scheduler; LSPConverter converter; CompilationDatabase database; + + using OnRequest = async::Task (Server::*)(json::Value); + using OnNotification = async::Task<> (Server::*)(json::Value); + llvm::StringMap onRequests; + llvm::StringMap onNotifications; }; } // namespace clice diff --git a/include/Support/Format.h b/include/Support/Format.h index 475129a3..b6a7e786 100644 --- a/include/Support/Format.h +++ b/include/Support/Format.h @@ -147,7 +147,7 @@ std::string dump(const Object& object) { result += is_sequence ? "]" : "}"; return result; } else if constexpr(std::is_enum_v) { - return std::format("{}", refl::enum_name(object)); + return std::format("\"{}\"", refl::enum_name(object)); } else if constexpr(refl::reflectable_enum) { return std::format("\"{}\"", object); } else if constexpr(refl::reflectable_struct) { @@ -168,9 +168,10 @@ std::string dump(const Object& object) { template std::string pretty_dump(const Object& object, std::size_t indent = 2) { - auto repr = dump(object); + std::string repr = dump(object); auto json = json::parse(repr); if(!json) { + println("{} {}", json.takeError(), repr); std::abort(); } llvm::SmallString<128> buffer = {std::format("{{0:{}}}", indent)}; diff --git a/include/Test/CTest.h b/include/Test/CTest.h index e3771f62..2873dc25 100644 --- a/include/Test/CTest.h +++ b/include/Test/CTest.h @@ -29,7 +29,7 @@ public: } void addFile(llvm::StringRef name, llvm::StringRef content) { - params.remappedFiles.emplace_back(name, annoate(content)); + params.addRemappedFile(name, annoate(content)); } llvm::StringRef annoate(llvm::StringRef content) { diff --git a/src/Compiler/Compilation.cpp b/src/Compiler/Compilation.cpp index bc89f2d9..e6e724df 100644 --- a/src/Compiler/Compilation.cpp +++ b/src/Compiler/Compilation.cpp @@ -56,6 +56,7 @@ std::unique_ptr createInstance(CompilationParams& param assert(!instance->getPreprocessorOpts().RetainRemappedFileBuffers && "RetainRemappedFileBuffers should be false"); + if(!params.content.empty()) { instance->getPreprocessorOpts().addRemappedFile( params.srcPath, @@ -63,11 +64,11 @@ std::unique_ptr createInstance(CompilationParams& param .release()); } - for(auto& [file, content]: params.remappedFiles) { - instance->getPreprocessorOpts().addRemappedFile( - file, - llvm::MemoryBuffer::getMemBufferCopy(content, file).release()); + /// Add all remapped file. + for(auto& [file, buffer]: params.buffers) { + instance->getPreprocessorOpts().addRemappedFile(file, buffer.release()); } + params.buffers.clear(); if(!instance->createTarget()) { std::abort(); @@ -174,7 +175,30 @@ std::expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer) { auto instance = impl::createInstance(params); - auto& [file, line, column] = params.completion; + auto& [file, offset] = params.completion; + + /// The location of clang is 1-1 based. + std::uint32_t line = 1; + std::uint32_t column = 1; + + llvm::StringRef content; + if(file == params.srcPath) { + content = params.content; + } else { + auto it = params.buffers.find(file); + assert(it != params.buffers.end() && "completion must occur in remapped file."); + content = it->second->getBuffer(); + } + + for(auto c: content.substr(0, offset)) { + if(c == '\n') { + line += 1; + column = 1; + continue; + } + column += 1; + } + /// Set options to run code completion. instance->getFrontendOpts().CodeCompletionAt.FileName = std::move(file); instance->getFrontendOpts().CodeCompletionAt.Line = line; @@ -203,7 +227,6 @@ std::expected compile(CompilationParams& params, PCHInfo& out.preamble = params.content.substr(0, *params.bound); out.command = params.command.str(); out.deps = info->deps(); - return std::move(*info); } else { return std::unexpected(info.error()); diff --git a/src/Feature/CodeCompletion.cpp b/src/Feature/CodeCompletion.cpp index 1fa328cd..15e9bf83 100644 --- a/src/Feature/CodeCompletion.cpp +++ b/src/Feature/CodeCompletion.cpp @@ -1,6 +1,8 @@ +#include "AST/Utility.h" #include "AST/SymbolKind.h" #include "Compiler/Compilation.h" #include "Feature/CodeCompletion.h" +#include "clang/Sema/Sema.h" #include "clang/Sema/CodeCompleteConsumer.h" namespace clice::feature { @@ -16,9 +18,9 @@ struct CompletionPrefix { // If there is none, begin() == end() == name.begin(). llvm::StringRef qualifier; - static CompletionPrefix from(llvm::StringRef content, std::size_t offset) { + static CompletionPrefix from(llvm::StringRef content, std::uint32_t offset) { assert(offset <= content.size()); - CompletionPrefix result; + CompletionPrefix prefix; llvm::StringRef rest = content.take_front(offset); @@ -28,7 +30,7 @@ struct CompletionPrefix { rest = rest.drop_back(); } - result.name = content.slice(rest.size(), offset); + prefix.name = content.slice(rest.size(), offset); // Consume qualifiers. while(rest.consume_back("::") && !rest.ends_with(":")) { @@ -38,65 +40,136 @@ struct CompletionPrefix { } } - result.qualifier = content.slice(rest.size(), result.name.begin() - content.begin()); - return result; + prefix.qualifier = content.slice(rest.size(), prefix.name.begin() - content.begin()); + return prefix; } }; class CodeCompletionCollector final : public clang::CodeCompleteConsumer { public: - CodeCompletionCollector(std::vector& completions) : - clang::CodeCompleteConsumer({}), completions(completions), - allocator(new clang::GlobalCodeCompletionAllocator()), info(allocator) {} + CodeCompletionCollector(std::uint32_t offset) : + clang::CodeCompleteConsumer({}), offset(offset), + info(std::make_shared()) {} + + CompletionItem processCandidate(clang::CodeCompletionResult& candidate) { + CompletionItem item; + + switch(candidate.Kind) { + case clang::CodeCompletionResult::RK_Keyword: { + item.label = candidate.Keyword; + item.kind = CompletionItemKind::Keyword; + break; + } + case clang::CodeCompletionResult::RK_Pattern: { + item.label = candidate.Pattern->getAsString(); + item.kind = CompletionItemKind::Snippet; + break; + } + case clang::CodeCompletionResult::RK_Macro: { + item.label = candidate.Macro->getName(); + item.kind = CompletionItemKind::Unit; + break; + } + case clang::CodeCompletionResult::RK_Declaration: { + auto decl = candidate.Declaration; + item.label = getDeclName(decl); + item.kind = CompletionItemKind::Function; + break; + } + } + + item.deprecated = false; + item.edit.text = item.label; + item.edit.range = editRange; + + return item; + } + + void initCompletionInfo(clang::Sema& sema) { + if(init) { + return; + } + + auto& PP = sema.getPreprocessor(); + auto& SM = sema.getSourceManager(); + auto loc = PP.getCodeCompletionLoc(); + auto content = SM.getBufferData(SM.getFileID(loc)); + + editRange = {offset, offset}; + + /// FIXME: consume the prefix of completion prefix, because we may complete + /// full qualified name. + assert(editRange.begin > 0); + + while(clang::isAsciiIdentifierContinue(content[editRange.begin - 1])) { + editRange.begin -= 1; + } + + if(editRange.end < content.size()) { + while(clang::isAsciiIdentifierContinue(content[editRange.end])) { + editRange.end += 1; + } + } + + init = true; + } void ProcessCodeCompleteResults(clang::Sema& sema, clang::CodeCompletionContext context, clang::CodeCompletionResult* candidates, unsigned count) final { - for(auto& candidate: llvm::make_range(candidates, candidates + count)) { - switch(candidate.Kind) { - case clang::CodeCompletionResult::RK_Declaration: { - break; - } - case clang::CodeCompletionResult::RK_Keyword: { - break; - } - case clang::CodeCompletionResult::RK_Macro: { - break; - } - case clang::CodeCompletionResult::RK_Pattern: { - break; - } - } + initCompletionInfo(sema); - println("{}", refl::enum_name(candidate.Kind)); + for(auto& candidate: llvm::make_range(candidates, candidates + count)) { + completions.emplace_back(processCandidate(candidate)); } } clang::CodeCompletionAllocator& getAllocator() final { - return *allocator; + return info.getAllocator(); } clang::CodeCompletionTUInfo& getCodeCompletionTUInfo() final { return info; } + auto dump() { + return std::move(completions); + } + private: - std::shared_ptr allocator; + clang::ASTContext* Ctx; + bool init = false; + /// TODO: + /// 1. 计算 token 边界，思考该怎么计算比较合适 + /// 比如 std::vec^ 选择 vector => std::vector + /// 比如 vec 选择 std::vector => std::vector + /// 不仅要考虑前缀，也要考虑 token 后缀的替换 + /// 之后记得试一下 clion 里面对 prefix 和 suffix 的处理 + /// + /// 2. 如果发现用户的光标正在补全头文件，则可以把该行头文件之 + /// 前的代码全 substr 掉，然后再在结果上加几行或者 offset，这样 + /// 可以大大优化补全头文件的速度，毕竟头文件补全只和编译命令有关 + /// 由于 #include 后面可能跟着宏，所以确保出现 <> 或者 "" 再进行这种 + /// 优化 + std::uint32_t offset; + LocalSourceRange editRange; clang::CodeCompletionTUInfo info; - std::vector& completions; + std::vector completions; }; } // namespace -std::vector codeCompletion(CompilationParams& params, - const config::CodeCompletionOption& option) { - std::vector completions; - auto consumer = new CodeCompletionCollector(completions); +std::vector codeCompletion(CompilationParams& params, + const config::CodeCompletionOption& option) { + auto& [file, offset] = params.completion; + auto consumer = new CodeCompletionCollector(offset); if(auto info = compile(params, consumer)) { - for(auto& item: completions) {} + return consumer->dump(); + /// TODO: Handle error here. + } else { + return {}; } - return completions; } } // namespace clice::feature diff --git a/src/Server/IncludeGraph.cpp b/src/Server/IncludeGraph.cpp index 1516c2ce..f45cdb33 100644 --- a/src/Server/IncludeGraph.cpp +++ b/src/Server/IncludeGraph.cpp @@ -127,9 +127,9 @@ async::Task<> IncludeGraph::index(llvm::StringRef file, CompilationDatabase& dat llvm::DenseMap files; /// Otherwise, we need to update all header contexts. - addContexts(**info, tu, files); + addContexts(*info, tu, files); - co_await updateIndices(**info, tu, files); + co_await updateIndices(*info, tu, files); } std::string IncludeGraph::getIndexPath(llvm::StringRef file) { @@ -271,7 +271,7 @@ async::Task<> IncludeGraph::updateIndices(ASTInfo& info, auto& SM = info.srcMgr(); - for(auto& [fid, index]: *indices) { + for(auto& [fid, index]: indices) { if(fid == SM.getMainFileID()) { if(tu->indexPath.empty()) { tu->indexPath = getIndexPath(tu->srcPath); diff --git a/src/Server/LSPConverter.cpp b/src/Server/LSPConverter.cpp index fdc35d49..5af6c41a 100644 --- a/src/Server/LSPConverter.cpp +++ b/src/Server/LSPConverter.cpp @@ -202,6 +202,16 @@ public: return it->second; } + proto::Range lookup(LocalSourceRange range) { + auto it = cache.find(range.begin); + assert(it != cache.end() && "Offset is not cached"); + auto begin = it->second; + it = cache.find(range.end); + assert(it != cache.end() && "Offset is not cached"); + auto end = it->second; + return proto::Range{begin, end}; + } + private: std::uint32_t line = 0; /// The offset of the last line end. @@ -219,6 +229,19 @@ private: } // namespace +std::uint32_t LSPConverter::convert(llvm::StringRef content, proto::Position position) { + return toOffset(content, encoding(), position); +} + +proto::Position LSPConverter::convert(llvm::StringRef content, std::uint32_t offset) { + PositionConverter converter(content, encoding()); + return converter.toPosition(offset); +} + +std::string LSPConverter::convert(llvm::StringRef URI) { + return fs::toPath(URI); +} + json::Value LSPConverter::convert(llvm::StringRef content, const feature::Hover& hover) { return json::Value(nullptr); } @@ -373,6 +396,27 @@ json::Value LSPConverter::convert(llvm::StringRef content, const feature::Semant }; } +json::Value LSPConverter::convert(llvm::StringRef content, + const std::vector& items) { + PositionConverter converter(content, encoding()); + converter.toPositions(items, [](auto& item) { return item.edit.range; }); + + json::Array result; + for(auto& item: items) { + json::Object object{ + {"label", item.label }, + {"kind", static_cast(item.kind)}, + {"textEdit", + json::Object{ + {"newText", item.edit.text}, + {"range", json::serialize(converter.lookup(item.edit.range))}, + } }, + }; + result.emplace_back(std::move(object)); + } + return result; +} + namespace proto { struct InitializeParams { @@ -398,7 +442,7 @@ struct InitializeResult { struct ServerCapabilities { std::string positionEncoding; - TextDocumentSyncKind textDocumentSync = TextDocumentSyncKind::Incremental; + TextDocumentSyncKind textDocumentSync = TextDocumentSyncKind::Full; bool declarationProvider = true; bool definitionProvider = true; @@ -415,7 +459,7 @@ struct InitializeResult { SemanticTokenOptions semanticTokensProvider; /// TODO: - /// completionProvider + CompletionOptions completionProvider; /// signatureHelpProvider /// codeLensProvider /// codeActionProvider diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp new file mode 100644 index 00000000..b641d9a4 --- /dev/null +++ b/src/Server/Scheduler.cpp @@ -0,0 +1,188 @@ +#include "Server/Config.h" +#include "Server/Scheduler.h" +#include "Server/LSPConverter.h" +#include "Support/Logger.h" +#include "Support/FileSystem.h" +#include "Compiler/Command.h" +#include "Compiler/Compilation.h" + +namespace clice { + +void Scheduler::addDocument(std::string path, std::string content) { + auto& openFile = openFiles[path]; + openFile.content = content; + + auto& task = openFile.ASTBuild; + + /// If there is already an AST build task, cancel it. + if(!task.empty()) { + task.cancel(); + task.dispose(); + } + + /// Create and schedule a new task. + task = buildAST(std::move(path), std::move(content)); + task.schedule(); +} + +llvm::StringRef Scheduler::getDocumentContent(llvm::StringRef path) { + return openFiles[path].content; +} + +async::Task Scheduler::semanticToken(std::string path) { + auto openFile = &openFiles[path]; + auto guard = co_await openFile->ASTBuiltLock.try_lock(); + + openFile = &openFiles[path]; + auto content = openFile->content; + auto AST = openFile->AST; + + auto tokens = co_await async::submit([&] { return feature::semanticTokens(*AST); }); + + co_return converter.convert(content, tokens); +} + +async::Task Scheduler::completion(std::string path, std::uint32_t offset) { + /// Wait for PCH building. + auto openFile = &openFiles[path]; + if(!openFile->PCHBuild.empty()) { + co_await openFile->PCHBuiltEvent; + } + + openFile = &openFiles[path]; + auto& PCH = openFile->PCH; + + /// Set compilation params ... . + CompilationParams params; + params.command = database.getCommand(path); + params.srcPath = path; + params.content = openFile->content; + params.pch = {PCH->path, PCH->preamble.size()}; + params.completion = {path, offset}; + + auto result = co_await async::submit([&] { return feature::codeCompletion(params, {}); }); + + openFile = &openFiles[path]; + co_return converter.convert(openFile->content, result); +} + +async::Task Scheduler::isPCHOutdated(llvm::StringRef path, llvm::StringRef preamble) { + auto openFile = &openFiles[path]; + + /// If there is not PCH, directly build it. + if(!openFile->PCH) { + co_return true; + } + + /// Check command and preamble matchs. + auto command = database.getCommand(path); + if(openFile->PCH->command != command || openFile->PCH->preamble != preamble) { + co_return true; + } + + /// TODO: Check mtime. + + co_return false; +} + +async::Task<> Scheduler::buildPCH(std::string path, std::string content) { + auto bound = computePreambleBound(content); + + auto openFile = &openFiles[path]; + bool outdated = true; + if(openFile->PCH) { + outdated = co_await isPCHOutdated(path, llvm::StringRef(content).substr(0, bound)); + } + + /// If not need update, return directly. + if(!outdated) { + co_return; + } + + /// The actual PCH build task. + constexpr static auto PCHBuildTask = [](Scheduler& scheduler, + std::string path, + std::uint32_t bound, + std::string content) -> async::Task<> { + CompilationParams params; + params.srcPath = path; + params.command = scheduler.database.getCommand(path); + params.content = content; + params.bound = bound; + params.outPath = path::join(config::index.dir, path::filename(path) + ".pch"); + + PCHInfo info; + auto result = co_await async::submit([&] { return compile(params, info); }); + if(!result) { + /// FIXME: Fails needs cancel waiting tasks. + log::warn("Building PCH fails for {}", path); + co_return; + } + + auto& openFile = scheduler.openFiles[path]; + /// Update the built PCH info. + openFile.PCH = std::move(info); + /// Dispose the task so that it will destroyed when task complete. + openFile.PCHBuild.dispose(); + /// Resume waiters on this event. + openFile.PCHBuiltEvent.set(); + openFile.PCHBuiltEvent.clear(); + + log::info("Building PCH successfully for {}", path); + }; + + openFile = &openFiles[path]; + + /// If there is already an PCH build task, cancel it. + auto& task = openFile->PCHBuild; + if(!task.empty()) { + task.cancel(); + task.dispose(); + } + + /// Schedule the new building task. + task = PCHBuildTask(*this, std::move(path), bound, std::move(content)); + task.schedule(); + + /// Waiting for PCH building. + co_await openFile->PCHBuiltEvent; +} + +async::Task<> Scheduler::buildAST(std::string path, std::string content) { + /// PCH is already updated. + co_await buildPCH(path, content); + + auto PCH = openFiles[path].PCH; + if(!PCH) { + log::fatal("Expected PCH built at this point"); + } + + CompilationParams params; + params.srcPath = path; + params.command = database.getCommand(path); + params.content = content; + params.pch = {PCH->path, PCH->preamble.size()}; + + /// Check result + auto info = co_await async::submit([&] { return compile(params); }); + if(!info) { + /// FIXME: Fails needs cancel waiting tasks. + log::warn("Building AST fails for {}", path); + co_return; + } + + auto& file = openFiles[path]; + + /// Try get the lock, the waiter on the lock will be resumed when + /// guard is destroyed. + auto guard = co_await file.ASTBuiltLock.try_lock(); + + /// Update built AST info. + file.AST = std::make_shared(std::move(*info)); + /// Dispose the task so that it will destroyed when task complete. + file.ASTBuild.dispose(); + + log::info("Building AST successfully for {}", path); +} + +} // namespace clice diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index ad9b6c94..6bd4f7c0 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -3,162 +3,6 @@ namespace clice { -Server::Server() : indexer(database, config::index) {} - -async::Task<> Server::onReceive(json::Value value) { - auto object = value.getAsObject(); - if(!object) [[unlikely]] { - log::fatal("Invalid LSP message, not an object: {}", value); - } - - /// If the json object has an `id`, it's a request, - /// which needs a response. Otherwise, it's a notification. - auto id = object->get("id"); - - llvm::StringRef method; - if(auto result = object->getString("method")) { - method = *result; - } else [[unlikely]] { - log::warn("Invalid LSP message, method not found: {}", value); - if(id) { - co_await response(std::move(*id), - proto::ErrorCodes::InvalidRequest, - "Method not found"); - } - co_return; - } - - json::Value params = json::Value(nullptr); - if(auto result = object->get("params")) { - params = std::move(*result); - } - - /// Handle request and notification separately. - /// TODO: Record the time of handling request and notification. - if(id) { - log::info("Handling request: {}", method); - auto result = co_await onRequest(method, std::move(params)); - co_await response(std::move(*id), std::move(result)); - log::info("Handled request: {}", method); - } else { - log::info("Handling notification: {}", method); - co_await onNotification(method, std::move(params)); - log::info("Handled notification: {}", method); - } - - co_return; -} - -async::Task Server::onRequest(llvm::StringRef method, json::Value value) { - if(method == "initialize") { - auto result = converter.initialize(std::move(value)); - config::init(converter.workspace()); - - /// FIXME: Use a better way to handle compile commands. - for(auto&& dir: config::server.compile_commands_dirs) { - database.updateCommands(dir + "/compile_commands.json"); - } - - indexer.load(); - - co_return json::serialize(result); - } else if(method == "shutdown") { - indexer.save(); - } else if(method.consume_front("textDocument/")) { - co_return co_await onTextDocument(method, std::move(value)); - } else if(method.consume_front("context/")) { - co_return co_await onContext(method, std::move(value)); - } else if(method.consume_front("index/")) { - co_return co_await onIndex(method, std::move(value)); - } - - co_return json::Value(nullptr); -} - -async::Task Server::onTextDocument(llvm::StringRef method, json::Value value) { - using SemanticTokensParams = proto::TextDocumentParams; - using FoldingRangeParams = proto::TextDocumentParams; - using DocumentLinkParams = proto::TextDocumentParams; - using DocumentSymbolParams = proto::TextDocumentParams; - - if(method == "semanticTokens/full") { - auto params2 = json::deserialize(value); - auto path = fs::toPath(params2.textDocument.uri); - - std::string buffer; - if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { - co_return converter.convert(index->content(), index->semanticTokens()); - } else { - co_return json::Value(nullptr); - } - - } else if(method == "foldingRange") { - auto params2 = json::deserialize(value); - auto path = fs::toPath(params2.textDocument.uri); - - std::string buffer; - if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { - co_return converter.convert(index->content(), index->foldingRanges()); - } else { - co_return json::Value(nullptr); - } - - } else if(method == "documentLink") { - auto params2 = json::deserialize(value); - auto path = fs::toPath(params2.textDocument.uri); - - std::string buffer; - if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { - co_return converter.convert(index->content(), index->documentLinks()); - } else { - co_return json::Value(nullptr); - } - } - - co_return json::Value(nullptr); -} - -async::Task Server::onContext(llvm::StringRef method, json::Value value) { - if(method == "current") { - auto param2 = json::deserialize(value); - auto path = fs::toPath(param2.textDocument.uri); - auto result = indexer.currentContext(path); - co_return result ? json::serialize(*result) : json::Value(nullptr); - } else if(method == "switch") { - auto params = json::deserialize(value); - auto header = fs::toPath(params.header); - indexer.switchContext(header, params.context); - } else if(method == "all") { - auto param2 = json::deserialize(value); - auto path = fs::toPath(param2.textDocument.uri); - auto result = indexer.allContexts(path); - co_return json::serialize(result); - } else if(method == "resolve") { - co_return json::serialize( - indexer.resolveContext(json::deserialize(value))); - } - - co_return json::Value(nullptr); -} - -async::Task Server::onIndex(llvm::StringRef method, json::Value value) { - co_return json::Value(nullptr); -} - -async::Task<> Server::onNotification(llvm::StringRef method, json::Value value) { - if(method.consume_front("textDocument/")) { - /// co_await onFileOperation(method, std::move(value)); - } - - if(method.consume_front("index/")) { - if(method == "all") { - indexer.indexAll(); - } - } - - co_return; -} - async::Task<> Server::request(llvm::StringRef method, json::Value params) { co_await async::net::write(json::Object{ {"jsonrpc", "2.0" }, @@ -211,4 +55,130 @@ async::Task<> Server::registerCapacity(llvm::StringRef id, }); } +Server::Server() : indexer(database, config::index), scheduler(converter, database) { + onRequests.try_emplace("initialize", &Server::onInitialize); + onRequests.try_emplace("textDocument/semanticTokens/full", &Server::onSemanticToken); + onRequests.try_emplace("textDocument/completion", &Server::onCodeCompletion); + onNotifications.try_emplace("textDocument/didOpen", &Server::onDidOpen); + onNotifications.try_emplace("textDocument/didChange", &Server::onDidChange); + onNotifications.try_emplace("textDocument/didSave", &Server::onDidSave); + onNotifications.try_emplace("textDocument/didClose", &Server::onDidClose); +} + +async::Task<> Server::onReceive(json::Value value) { + auto object = value.getAsObject(); + if(!object) [[unlikely]] { + log::fatal("Invalid LSP message, not an object: {}", value); + } + + /// If the json object has an `id`, it's a request, + /// which needs a response. Otherwise, it's a notification. + auto id = object->get("id"); + + llvm::StringRef method; + if(auto result = object->getString("method")) { + method = *result; + } else [[unlikely]] { + log::warn("Invalid LSP message, method not found: {}", value); + if(id) { + co_await response(std::move(*id), + proto::ErrorCodes::InvalidRequest, + "Method not found"); + } + co_return; + } + + json::Value params = json::Value(nullptr); + if(auto result = object->get("params")) { + params = std::move(*result); + } + + /// Handle request and notification separately. + /// TODO: Record the time of handling request and notification. + if(id) { + log::info("Handling request: {}", method); + if(auto iter = onRequests.find(method); iter != onRequests.end()) { + auto result = co_await (this->*(iter->second))(std::move(params)); + co_await response(std::move(*id), std::move(result)); + } + log::info("Handled request: {}", method); + } else { + log::info("Handling notification: {}", method); + if(auto iter = onNotifications.find(method); iter != onNotifications.end()) { + co_await (this->*(iter->second))(std::move(params)); + } + log::info("Handled notification: {}", method); + } + + co_return; +} + +async::Task Server::onInitialize(json::Value value) { + auto result = converter.initialize(std::move(value)); + config::init(converter.workspace()); + + for(auto& dir: config::server.compile_commands_dirs) { + database.updateCommands(dir + "/compile_commands.json"); + } + + co_return result; +} + +async::Task<> Server::onDidOpen(json::Value value) { + struct DidOpenTextDocumentParams { + proto::TextDocumentItem textDocument; + }; + + auto params = json::deserialize(value); + auto path = converter.convert(params.textDocument.uri); + scheduler.addDocument(std::move(path), std::move(params.textDocument.text)); + co_return; +} + +async::Task<> Server::onDidChange(json::Value value) { + struct DidChangeTextDocumentParams { + proto::VersionedTextDocumentIdentifier textDocument; + + struct TextDocumentContentChangeEvent { + std::string text; + }; + + std::vector contentChanges; + }; + + auto params = json::deserialize(value); + auto path = converter.convert(params.textDocument.uri); + scheduler.addDocument(std::move(path), std::move(params.contentChanges[0].text)); + + co_return; +} + +async::Task<> Server::onDidSave(json::Value value) { + co_return; +} + +async::Task<> Server::onDidClose(json::Value value) { + co_return; +} + +async::Task Server::onSemanticToken(json::Value value) { + struct SemanticTokensParams { + proto::TextDocumentIdentifier textDocument; + }; + + auto params = json::deserialize(value); + auto path = converter.convert(params.textDocument.uri); + co_return co_await scheduler.semanticToken(std::move(path)); +} + +async::Task Server::onCodeCompletion(json::Value value) { + using CompletionParams = proto::TextDocumentPositionParams; + auto params = json::deserialize(value); + + auto path = converter.convert(params.textDocument.uri); + auto content = scheduler.getDocumentContent(path); + auto offset = converter.convert(content, params.position); + co_return co_await scheduler.completion(std::move(path), offset); +} + } // namespace clice diff --git a/unittests/Async/Event.cpp b/unittests/Async/Event.cpp index cf672557..fe760cba 100644 --- a/unittests/Async/Event.cpp +++ b/unittests/Async/Event.cpp @@ -33,6 +33,34 @@ TEST(Async, Event) { async::run(task1(), task2(), main()); } +TEST(Async, EventClear) { + async::Event event; + + int x = 0; + + auto task1 = [&]() -> async::Task<> { + EXPECT_EQ(x, 0); + co_await event; + EXPECT_EQ(x, 1); + x = 2; + }; + + auto task2 = [&]() -> async::Task<> { + EXPECT_EQ(x, 0); + co_await event; + EXPECT_EQ(x, 2); + x = 3; + }; + + auto main = [&]() -> async::Task<> { + x = 1; + event.set(); + co_return; + }; + + async::run(task1(), task2(), main()); +} + } // namespace } // namespace clice::testing diff --git a/unittests/Async/ThreadPool.cpp b/unittests/Async/ThreadPool.cpp index b85a9d16..85f5138a 100644 --- a/unittests/Async/ThreadPool.cpp +++ b/unittests/Async/ThreadPool.cpp @@ -6,7 +6,7 @@ namespace clice::testing { namespace { TEST(Async, ThreadPool) { - auto task_gen = []() -> async::Result { + auto task_gen = []() -> async::Task { co_return co_await async::submit([]() { std::this_thread::sleep_for(std::chrono::milliseconds(100)); return std::this_thread::get_id(); @@ -31,13 +31,9 @@ TEST(Async, ThreadPool) { auto id2 = task2.result(); auto id3 = task3.result(); - EXPECT_TRUE(id1.has_value()); - EXPECT_TRUE(id2.has_value()); - EXPECT_TRUE(id3.has_value()); - - EXPECT_NE(*id1, *id2); - EXPECT_NE(*id1, *id3); - EXPECT_NE(*id2, *id3); + EXPECT_NE(id1, id2); + EXPECT_NE(id1, id3); + EXPECT_NE(id2, id3); } } // namespace diff --git a/unittests/Compiler/Module.cpp b/unittests/Compiler/Module.cpp index 483e0c45..18421bae 100644 --- a/unittests/Compiler/Module.cpp +++ b/unittests/Compiler/Module.cpp @@ -15,7 +15,7 @@ PCMInfo buildPCM(llvm::StringRef file, llvm::StringRef code) { params.srcPath = file; params.outPath = outPath; params.command = "clang++ -std=c++20 -x c++ " + file.str(); - params.remappedFiles.emplace_back("./test.h", "export int foo2();"); + params.addRemappedFile("./test.h", "export int foo2();"); PCMInfo pcm; if(!compile(params, pcm)) { @@ -31,7 +31,7 @@ ModuleInfo scan(llvm::StringRef content) { params.content = content; params.srcPath = "main.ixx"; params.command = "clang++ -std=c++20 -x c++ main.ixx"; - params.remappedFiles.emplace_back("./test.h", "export module A"); + params.addRemappedFile("./test.h", "export module A"); auto info = scanModule(params); if(!info) { llvm::errs() << "Failed to scan module\n"; @@ -154,4 +154,4 @@ export module B; } // namespace -} // namespace clice +} // namespace clice::testing diff --git a/unittests/Compiler/Preamble.cpp b/unittests/Compiler/Preamble.cpp index 27cd259b..8fb30925 100644 --- a/unittests/Compiler/Preamble.cpp +++ b/unittests/Compiler/Preamble.cpp @@ -73,7 +73,7 @@ int x = foo(); params.bound = computePreambleBound(content); llvm::SmallString<128> path; - params.remappedFiles.emplace_back(deps[0], test); + params.addRemappedFile(deps[0], test); /// Build PCH. PCHInfo out; @@ -189,7 +189,7 @@ export int x = foo(); params.bound = computePreambleBound(content); llvm::SmallString<128> path; - params.remappedFiles.emplace_back(deps[0], test); + params.addRemappedFile(deps[0], test); /// Build PCH. PCHInfo out; diff --git a/unittests/Feature/CodeCompletion.cpp b/unittests/Feature/CodeCompletion.cpp index 7d833727..068c02cb 100644 --- a/unittests/Feature/CodeCompletion.cpp +++ b/unittests/Feature/CodeCompletion.cpp @@ -6,22 +6,32 @@ namespace clice::testing { namespace { TEST(Feature, CodeCompletion) { - const char* code = R"cpp( + llvm::StringRef code = R"cpp( +#include +#include +#include +#include +///#include <> int foo = 2; int main() { foo = 2; + std::vec$(pos)tor } )cpp"; + Annotation annotation = {code}; CompilationParams params; - params.content = code; + params.content = annotation.source(); params.srcPath = "main.cpp"; params.command = "clang++ -std=c++20 main.cpp"; - params.completion = {"main.cpp", 5, 6}; + params.completion = {"main.cpp", annotation.offset("pos")}; config::CodeCompletionOption options = {}; auto result = feature::codeCompletion(params, options); + // for(auto& item: result) { + // println("kind {} label {}", item.label, refl::enum_name(item.kind)); + // } } } // namespace diff --git a/unittests/Feature/SignatureHelp.cpp b/unittests/Feature/SignatureHelp.cpp index 6222a153..4015555c 100644 --- a/unittests/Feature/SignatureHelp.cpp +++ b/unittests/Feature/SignatureHelp.cpp @@ -22,14 +22,14 @@ int main() { params.content = code; params.srcPath = "main.cpp"; params.command = "clang++ -std=c++20 main.cpp"; - params.completion = {"main.cpp", 9, 10}; + /// params.completion = {"main.cpp", 9, 10}; - config::SignatureHelpOption options = {}; - auto result = feature::signatureHelp(params, options); - /// EXPECT - /// foo(int x, int y) - /// foo(int x) - /// foo() + /// config::SignatureHelpOption options = {}; + /// auto result = feature::signatureHelp(params, options); + /// EXPECT + /// foo(int x, int y) + /// foo(int x) + /// foo() } } // namespace