diff --git a/docs/configuration.md b/docs/configuration.md index eb49e9bc..dc12dff0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -177,3 +177,41 @@ If `index.instantiation` is `true`, clice will traverse declarations in template If `index.instantiation` is `false`, clice will not index entities inside template instantiations, and `go-to-definition` will return no results.
+ +## Feature + +### Semantic Tokens + +| Name | Type | Default | +| ------------------------------------ | ------------------ | ------- | +| `feature.semanticTokens.typeMap` | `array` of `table` | `[]` | +| `feature.semanticTokens.modifierMap` | `array` of `table` | `[]` | + +Maps the customized semantic symbol kinds or modifier to LSP semantic token types or modifiers. + +Example: + +```toml +[feature.semanticTokens] +typeMap = [ + {"from": "header", "to": "string"}, + {"from": "attribute", "to": "decorator"}, +] + +modifierMap = [ + {"from": "const", "to": "readonly"}, + {"from": "pureVirtual", "to": "abstract"}, +] +``` + +For all clice symbol kinds, please refer to [SymbolKind](https://github.com/clice-project/clice/blob/main/include/AST/SymbolKind.h). The first letter of the name should be translated to lowercase. For all LSP semantic token types, refer to [SemanticTokenKind](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens). + + +| Name | Type | Default | +| --------------------------------- | --------- | ------- | +| `feature.semanticTokens.standard` | `boolean` | `false` | + +If `true`, clice will map the semantic token types and modifiers to the standard ones automatically. + +### Folding Range + diff --git a/include/Basic/Lifecycle.h b/include/Basic/Lifecycle.h index eafc3cbe..d5cf4c57 100644 --- a/include/Basic/Lifecycle.h +++ b/include/Basic/Lifecycle.h @@ -78,6 +78,59 @@ struct SemanticTokensOptions { bool full = true; }; +struct SemanticTokens { + /// The actual tokens. + std::vector data; +}; + +/// A set of predefined range kinds. +enum class FoldingRangeKind { + /// Folding range for a comment. + Comment, + + /// Folding range for imports or includes. + Imports, + + /// Folding range for a region. + Region, +}; + +/// Represents a folding range. To be valid, start and end line must be bigger +/// than zero and smaller than the number of lines in the document. Clients +/// are free to ignore invalid ranges. +struct FoldingRange { + /// The zero-based start line of the range to fold. The folded area starts + /// after the line's last character. To be valid, the end must be zero or + /// larger and smaller than the number of lines in the document. + uint32_t startLine; + + /// The zero-based character offset from where the folded range starts. If + /// not defined, defaults to the length of the start line. + std::optional startCharacter; + + /// The zero-based end line of the range to fold. The folded area ends with + /// the line's last character. To be valid, the end must be zero or larger + /// and smaller than the number of lines in the document. + uint32_t endLine; + + /// The zero-based character offset before the folded range ends. If not + /// defined, defaults to the length of the end line. + std::optional endCharacter; + + /// Describes the kind of the folding range such as `comment` or `region`. + /// The kind is used to categorize folding ranges and used by commands like + /// 'Fold all comments'. See [FoldingRangeKind](#FoldingRangeKind) for an + /// enumeration of standardized kinds. + FoldingRangeKind kind; + + /// The text that the client should show when the specified range is + /// collapsed. If not defined or not supported by the client, a default + /// will be chosen by the client. + /// + /// @since 3.17.0 - proposed + std::optional collapsedText; +}; + /// Server Capability. struct ServerCapabilities { /// The position encoding the server picked from the encodings offered @@ -118,6 +171,9 @@ struct ServerCapabilities { /// The server provides semantic tokens support. SemanticTokensOptions semanticTokensProvider; + + /// The server provides folding provider support. + bool foldingRangeProvider = true; }; struct InitializeResult { diff --git a/include/Compiler/Command.h b/include/Compiler/Command.h index 3a27dafd..f5660810 100644 --- a/include/Compiler/Command.h +++ b/include/Compiler/Command.h @@ -1,8 +1,7 @@ #pragma once #include -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" namespace clice { @@ -18,4 +17,50 @@ std::expected mangleCommand(llvm::StringRef command, llvm::SmallVectorImpl& out, llvm::SmallVectorImpl& buffer); +/// `CompilationDatabase` is responsible for managing the compile commands. +/// +/// FIXME: currently we assume that a file only occurs once in the CDB. +/// This is not always correct, but it is enough for now. +class CompilationDatabase { +public: + /// Update the compile commands with the given file. + void updateCommands(llvm::StringRef file); + + /// Update the compile commands with the given file and compile command. + void updateCommand(llvm::StringRef file, llvm::StringRef command); + + /// Update the module map with the given file and module name. + void updateModule(llvm::StringRef file, llvm::StringRef name); + + /// Lookup the compile commands of the given file. + llvm::StringRef getCommand(llvm::StringRef file); + + /// Lookup the module interface unit file path of the given module name. + llvm::StringRef getModuleFile(llvm::StringRef name); + + auto size() const { + return commands.size(); + } + + auto begin() { + return commands.begin(); + } + + auto end() { + return commands.end(); + } + +private: + /// A map between file path and compile commands. + llvm::StringMap commands; + + /// For C++20 module, we only can got dependent module name + /// in source context. But we need dependent module file path + /// to build PCM. So we will scan(preprocess) all project files + /// to build a module map between module name and module file path. + /// **Note that** this only includes module interface unit, for module + /// implementation unit, the scan could be delayed until compiling it. + llvm::StringMap moduleMap; +}; + } // namespace clice diff --git a/include/Feature/DocumentSymbol.h b/include/Feature/DocumentSymbol.h index 50ddf51b..8f198e05 100644 --- a/include/Feature/DocumentSymbol.h +++ b/include/Feature/DocumentSymbol.h @@ -1,3 +1,5 @@ +#pragma once + #include "Basic/Document.h" #include "Basic/SourceCode.h" #include "Index/Shared.h" diff --git a/include/Index/FeatureIndex.h b/include/Index/FeatureIndex.h index f066992e..62f1795b 100644 --- a/include/Index/FeatureIndex.h +++ b/include/Index/FeatureIndex.h @@ -31,7 +31,9 @@ public: } } - llvm::ArrayRef semanticTokens() const; + std::vector semanticTokens() const; + + std::vector foldingRanges() const; public: char* base; diff --git a/include/Index/Shared2.h b/include/Index/Shared2.h new file mode 100644 index 00000000..0b46d662 --- /dev/null +++ b/include/Index/Shared2.h @@ -0,0 +1,29 @@ +#pragma once + +#include "SymbolIndex.h" +#include "FeatureIndex.h" +#include "Async/FileSystem.h" + +namespace clice::index { + +struct Index2 { + std::optional symbol; + llvm::XXH128_hash_t symbolHash = {0, 0}; + + std::optional feature; + llvm::XXH128_hash_t featureHash = {0, 0}; + + static Shared build(ASTInfo& AST); + + async::Task<> write(std::string path) { + if(symbol) { + co_await async::fs::write(path + ".sidx", symbol->base, symbol->size); + } + + if(feature) { + co_await async::fs::write(path + ".fidx", feature->base, feature->size); + } + } +}; + +} // namespace clice::index diff --git a/include/Server/Cache.h b/include/Server/Cache.h deleted file mode 100644 index 6166a877..00000000 --- a/include/Server/Cache.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include - -#include "Async/Async.h" -#include "Database.h" -#include "Compiler/Module.h" -#include "Compiler/Preamble.h" - -#include "llvm/ADT/StringMap.h" - -namespace clice { - -struct CacheOption { - /// The directory to store the cache files. - std::string dir; -}; - -/// This class is responsible for PCH and PCM building. -class CacheController { -public: - CacheController(CacheOption& option, CompilationDatabase& database); - - /// Generate `cache.json` to store the cache information. - void loadFromDisk(); - - /// Load the cache information from `cache.json`. - void saveToDisk(); - - /// Complete the PCH or PCM information required for the compilation arguments. - /// If no suitable PCH or PCM is available, a build will be triggered. - async::Task<> prepare(CompilationParams& params); - - async::Task<> updatePCH(); - -private: - const CacheOption& option; - - CompilationDatabase& database; - - struct CachedPCHInfo : PCHInfo { - /// The hash of the preamble, for fast comparison. - std::uint64_t hash; - - /// The reference count of this PCH. When server exit, all PCH with zero - /// reference count will be removed. - std::uint32_t reference; - }; - - /// All PCHs. - std::deque pchs; - - /// A map between source file and its PCH. - llvm::StringMap pchMap; - - /// [module name] -> [PCMInfo] - llvm::StringMap pcms; -}; - -} // namespace clice diff --git a/include/Server/Config.h b/include/Server/Config.h index b563545d..386fcfbc 100644 --- a/include/Server/Config.h +++ b/include/Server/Config.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" @@ -8,7 +9,7 @@ namespace clice::config { /// Read the config file, call when the program starts. -void load(llvm::StringRef execute, llvm::StringRef filename); +std::expected load(llvm::StringRef execute, llvm::StringRef filename); /// Initialize the config, replace all predefined variables in the config file. /// called in `Server::initialize`. diff --git a/include/Server/Database.h b/include/Server/Database.h deleted file mode 100644 index dfa13485..00000000 --- a/include/Server/Database.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include "llvm/ADT/StringMap.h" - -namespace clice { - -/// `CompilationDatabase` is responsible for managing the compile commands. -/// -/// FIXME: currently we assume that a file only occurs once in the CDB. -/// This is not always correct, but it is enough for now. -class CompilationDatabase { -public: - /// Update the compile commands with the given file. - void updateCommands(llvm::StringRef file); - - /// Update the compile commands with the given file and compile command. - void updateCommand(llvm::StringRef file, llvm::StringRef command); - - /// Update the module map with the given file and module name. - void updateModule(llvm::StringRef file, llvm::StringRef name); - - /// Lookup the compile commands of the given file. - llvm::StringRef getCommand(llvm::StringRef file); - - /// Lookup the module interface unit file path of the given module name. - llvm::StringRef getModuleFile(llvm::StringRef name); - - auto size() const { - return commands.size(); - } - - auto begin() { - return commands.begin(); - } - - auto end() { - return commands.end(); - } - -private: - /// A map between file path and compile commands. - llvm::StringMap commands; - - /// For C++20 module, we only can got dependent module name - /// in source context. But we need dependent module file path - /// to build PCM. So we will scan(preprocess) all project files - /// to build a module map between module name and module file path. - /// **Note that** this only includes module interface unit, for module - /// implementation unit, the scan could be delayed until compiling it. - llvm::StringMap moduleMap; -}; - -} // namespace clice diff --git a/include/Server/IncludeGraph.h b/include/Server/IncludeGraph.h index f7887813..912d090b 100644 --- a/include/Server/IncludeGraph.h +++ b/include/Server/IncludeGraph.h @@ -1,10 +1,10 @@ #pragma once #include "Config.h" -#include "Database.h" #include "Protocol.h" #include "Async/Async.h" #include "Basic/SourceConverter.h" +#include "Compiler/Command.h" #include "Compiler/Compilation.h" #include "Support/JSON.h" #include "Index/SymbolIndex.h" @@ -25,11 +25,12 @@ struct HeaderIndex { }; struct Context { - /// The include chain that introduces this context. - uint32_t include = -1; - - /// The index information of this context. + /// The index of header context in indices. uint32_t index = -1; + + /// The location index in corresponding tu's + /// all include locations. + uint32_t include = -1; }; struct IncludeLocation { @@ -43,22 +44,10 @@ struct IncludeLocation { /// a header may be included by multiple files, so we use /// a string pool to cache the file name to reduce the memory /// usage. - uint32_t filename = -1; + uint32_t file = -1; }; -struct Header { - /// The path of the header file. - std::string srcPath; - - /// All indices of this header. - std::vector indices; - - /// All header contexts of this header. - llvm::DenseMap> contexts; - - /// The active translation unit and the index of the context. - std::pair active = {nullptr, -1}; -}; +struct Header; struct TranslationUnit { /// The source file path. @@ -83,33 +72,46 @@ struct TranslationUnit { uint32_t version = 0; }; -namespace proto { - -struct IncludeLocation { - /// The line number of the include directive. - uint32_t line; - - /// The filename of the included header. - std::string filename; -}; - struct HeaderContext { - /// The path of the source file. - std::string srcFile; + TranslationUnit* tu = nullptr; - /// The path of the context file. - std::string contextFile; + Context context; - /// The index of the context. - uint32_t index = -1; - - /// The version of the context. - uint32_t version = 0; + bool valid() { + return tu != nullptr; + } }; -using HeaderContextGroups = std::vector>; +struct Header { + /// The path of the header file. + std::string srcPath; -} // namespace proto + /// The active header context. + HeaderContext active; + + /// All indices of the header. + std::vector indices; + + /// All header contexts of this header. + llvm::DenseMap> contexts; + + /// Given a translation unit and a include location, return its + /// its corresponding index. + std::optional getIndex(TranslationUnit* tu, uint32_t include) { + auto it = contexts.find(tu); + if(it == contexts.end()) { + return std::nullopt; + } + + for(auto& context: it->second) { + if(context.include == include) { + return context.index; + } + } + + return std::nullopt; + } +}; class IncludeGraph { protected: @@ -123,62 +125,6 @@ protected: async::Task<> index(llvm::StringRef file, CompilationDatabase& database); -public: - /// Return all header context of the given file. - /// FIXME: The results are grouped by the index file. And a header actually - /// may have thousands of contexts, of course, users don't want to see all - /// of them. For each index file, we return the first 10 contexts. In the future - /// we may add a parameter to control the number of contexts or set filter. - proto::HeaderContextGroups contextAll(llvm::StringRef file); - - /// Return current header context of the given file. - std::optional contextCurrent(llvm::StringRef file); - - /// Switch to the given header context. - void contextSwitch(const proto::HeaderContext& context); - - /// Resolve the header context to the include chain. - std::vector contextResolve(const proto::HeaderContext& context); - -private: - struct SymbolID { - uint64_t hash; - std::string name; - }; - - /// Return all indices of the given translation unit. If the file is empty, - /// return all indices of the IncludeGraph. - std::vector indices(TranslationUnit* tu = nullptr); - - /// Resolve the symbol at the given position. - async::Task> resolve(const proto::TextDocumentPositionParams& params); - - using LookupCallback = llvm::unique_function; - - async::Task<> lookup(llvm::ArrayRef targets, - llvm::ArrayRef files, - LookupCallback callback); - -public: - /// Lookup the reference information according to the given position. - async::Task lookup(const proto::ReferenceParams& params, - RelationKind kind); - - /// According to the given file and offset, resolve the symbol at the offset. - async::Task - prepareHierarchy(const proto::HierarchyPrepareParams& params); - - async::Task - incomingCalls(const proto::HierarchyParams& params); - - async::Task - outgoingCalls(const proto::HierarchyParams& params); - - async::Task typeHierarchy(const proto::HierarchyParams& params, - bool super); - private: std::string getIndexPath(llvm::StringRef file); @@ -190,7 +136,8 @@ private: uint32_t addIncludeChain(std::vector& locations, llvm::DenseMap& files, clang::SourceManager& SM, - clang::FileID fid); + clang::FileID fid, + ASTInfo& AST); void addContexts(ASTInfo& info, TranslationUnit* tu, @@ -200,7 +147,7 @@ private: TranslationUnit* tu, llvm::DenseMap& files); -private: +protected: const config::IndexOptions& options; llvm::StringMap headers; llvm::StringMap tus; diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h index 6d135ba1..9ffa7b39 100644 --- a/include/Server/Indexer.h +++ b/include/Server/Indexer.h @@ -1,15 +1,14 @@ #pragma once #include "Config.h" -#include "Database.h" -#include "Async/Async.h" -#include "llvm/ADT/StringSet.h" #include "IncludeGraph.h" +#include "Async/Async.h" +#include "Compiler/Command.h" +#include "Index/FeatureIndex.h" +#include "llvm/ADT/StringSet.h" namespace clice { -class IncludeGraph; - class Indexer : public IncludeGraph { public: Indexer(CompilationDatabase& database, const config::IndexOptions& options); @@ -26,6 +25,78 @@ public: void load(); +public: + Header* getHeader(llvm::StringRef file) const; + + TranslationUnit* getTranslationUnit(llvm::StringRef file) const; + + /// Return current header context of given header file. If the header + /// does't have an active context, the result will be invalid. + std::optional currentContext(llvm::StringRef header) const; + + /// Switch the context of the header to given context. If success, + /// return true. + bool switchContext(llvm::StringRef header, proto::HeaderContext context); + + /// Resolve the given header context to a group of locations. + std::vector resolveContext(proto::HeaderContext context) const; + + /// Return all header contexts of given header file, note that a header may have thousands + /// of header contexts, of course we won't return them all at once. We would return a group + /// of contexts for each different header context. The maximum of group count is determined + /// by limit. Optionally, you can specify a 'contextFile' to filter the results, returning only + /// contexts related to that file. + std::vector + allContexts(llvm::StringRef headerFile, + uint32_t limit = 10, + llvm::StringRef contextFile = llvm::StringRef()) const; + +public: + struct SymbolID { + uint64_t hash; + std::string name; + }; + + /// Return all indices of the given translation unit. If the file is empty, + /// return all indices of the IncludeGraph. + std::vector indices(TranslationUnit* tu = nullptr); + + /// Resolve the symbol at the given position. + async::Task> resolve(const proto::TextDocumentPositionParams& params); + + using LookupCallback = llvm::unique_function; + + async::Task<> lookup(llvm::ArrayRef targets, + llvm::ArrayRef files, + LookupCallback callback); + + /// Lookup the reference information according to the given position. + async::Task lookup(const proto::ReferenceParams& params, + RelationKind kind); + + /// According to the given file and offset, resolve the symbol at the offset. + async::Task + prepareHierarchy(const proto::HierarchyPrepareParams& params); + + async::Task + incomingCalls(const proto::HierarchyParams& params); + + async::Task + outgoingCalls(const proto::HierarchyParams& params); + + async::Task typeHierarchy(const proto::HierarchyParams& params, + bool super); + +public: + async::Task> getFeatureIndex(std::string& buffer, + llvm::StringRef file) const; + + async::Task> semanticTokens(llvm::StringRef file) const; + + async::Task> foldingRanges(llvm::StringRef file) const; + private: async::Task<> index(std::string file); diff --git a/include/Server/LSPConverter.h b/include/Server/LSPConverter.h index 049818de..7596ac6b 100644 --- a/include/Server/LSPConverter.h +++ b/include/Server/LSPConverter.h @@ -7,6 +7,7 @@ #include "Feature/FoldingRange.h" #include "Feature/DocumentSymbol.h" #include "Feature/SemanticTokens.h" +#include "Server/Protocol.h" namespace clice { @@ -15,6 +16,26 @@ class LSPConverter { public: using Result = async::Task; + proto::InitializeResult initialize(json::Value value); + + auto encoding() { + return params.capabilities.general.positionEncodings[0]; + } + + auto& capabilities() { + return params.capabilities; + } + + /// The path of the workspace. + llvm::StringRef workspace(); + +public: + proto::SemanticTokens transform(llvm::StringRef content, + llvm::ArrayRef tokens); + + std::vector transform(llvm::StringRef content, + llvm::ArrayRef foldings); + Result convert(llvm::StringRef path, llvm::ArrayRef tokens); Result convert(llvm::StringRef path, llvm::ArrayRef foldings); @@ -22,7 +43,8 @@ public: Result convert(const feature::Hover& hover); private: - proto::PositionEncodingKind kind; + proto::InitializeParams params; + std::string workspacePath; }; } // namespace clice diff --git a/include/Server/Protocol.h b/include/Server/Protocol.h index 05c16ba6..707ecbe6 100644 --- a/include/Server/Protocol.h +++ b/include/Server/Protocol.h @@ -1,3 +1,77 @@ #pragma once -#include "Basic/Lifecycle.h" \ No newline at end of file +#include "Basic/Lifecycle.h" + +namespace clice::proto { + +struct TextDocumentParams { + /// The text document. + TextDocumentIdentifier textDocument; +}; + +enum class SemanticTokenTypes { + Namespace, + Type, + Class, + Enum, + Interface, + Struct, + TypeParameter, + Parameter, + Variable, + Property, + EnumMember, + Event, + Function, + Method, + Macro, + Keyword, + Modifier, + Comment, + String, + Number, + Regexp, + Operator, + Decorator +}; + +using SemanticTokensParams = TextDocumentParams; + +using FoldingRangeParams = TextDocumentParams; + +struct HeaderContext { + /// The path of context file. + std::string file; + + /// The version of context file's AST. + uint32_t version; + + /// The include location id for further resolving. + uint32_t include; +}; + +struct IncludeLocation { + /// The line of include drective. + uint32_t line = -1; + + /// The file path of include drective. + std::string file; +}; + +struct HeaderContextGroup { + /// The index path of this header Context. + std::string indexFile; + + /// The header contexts. + std::vector contexts; +}; + +struct HeaderContextSwitchParams { + /// The header file path which wants to switch context. + std::string header; + + /// The context + HeaderContext context; +}; + +} // namespace clice::proto diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h deleted file mode 100644 index ecc33df7..00000000 --- a/include/Server/Scheduler.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "Cache.h" - -namespace clice { - -struct Rule { - /// The file name pattern. - std::string pattern; - - /// ... - std::vector append; - - /// ... - std::vector remove; - - std::string readonly; - - std::string header; - - std::vector context; -}; - -/// This class is responsible for managing all opened files. -class Scheduler { -public: - Scheduler(CompilationDatabase& database, llvm::ArrayRef rules) : - database(database), rules(rules) {} - - async::Task<> open(llvm::StringRef path); - - async::Task<> update(llvm::StringRef path); - - async::Task<> close(llvm::StringRef path); - -private: - CompilationDatabase& database; - - llvm::ArrayRef rules; - - struct File {}; - - llvm::StringMap files; -}; - -} // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index a349e535..5e811143 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -3,19 +3,92 @@ #include "Config.h" #include "Indexer.h" #include "Protocol.h" -#include "Database.h" -#include "Scheduler.h" +#include "LSPConverter.h" #include "Async/Async.h" +#include "Compiler/Command.h" namespace clice { +namespace proto { + +enum class ErrorCodes { + // Defined by JSON-RPC + ParseError = -32700, + InvalidRequest = -32600, + MethodNotFound = -32601, + InvalidParams = -32602, + InternalError = -32603, + + /** + * Error code indicating that a server received a notification or + * request before the server has received the `initialize` request. + */ + ServerNotInitialized = -32002, + UnknownErrorCode = -32001, + + /** + * A request failed but it was syntactically correct, e.g the + * method name was known and the parameters were valid. The error + * message should contain human readable information about why + * the request failed. + * + * @since 3.17.0 + */ + RequestFailed = -32803, + + /** + * The server cancelled the request. This error code should + * only be used for requests that explicitly support being + * server cancellable. + * + * @since 3.17.0 + */ + ServerCancelled = -32802, + + /** + * The server detected that the content of a document got + * modified outside normal conditions. A server should + * NOT send this error code if it detects a content change + * in it unprocessed messages. The result even computed + * on an older state might still be useful for the client. + * + * If a client decides that a result is not of any use anymore + * the client should cancel the request. + */ + ContentModified = -32801, + + /** + * The client has canceled a request and a server has detected + * the cancel. + */ + RequestCancelled = -32800, +}; + +} + class Server { public: Server(); async::Task<> onReceive(json::Value value); + /// Handle requests, a request must have a response. + async::Task onRequest(llvm::StringRef method, json::Value value); + + /// Handle requests started with `textDocument/`. + async::Task onTextDocument(llvm::StringRef method, json::Value value); + + /// Handle requests started with `context/`. + async::Task onContext(llvm::StringRef method, json::Value value); + + /// Handle requests started with `index/`. + async::Task onIndex(llvm::StringRef method, json::Value value); + + /// Handle notifications, a notification doesn't require response. + async::Task<> onNotification(llvm::StringRef method, json::Value value); + +private: /// Send a request to the client. async::Task<> request(llvm::StringRef method, json::Value params); @@ -25,146 +98,21 @@ public: /// Send a response to the client. async::Task<> response(json::Value id, json::Value result); + async::Task<> response(json::Value id, proto::ErrorCodes code, llvm::StringRef message = ""); + /// Send an register capability to the client. async::Task<> registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions); + +private: + async::Task<> initialize(json::Value value); + +public: std::uint32_t id = 0; - -private: - using onRequest = llvm::unique_function(json::Value, json::Value)>; - using onNotification = llvm::unique_function(json::Value)>; - - template - void addMethod(llvm::StringRef name, - async::Task<> (Server::*method)(json::Value, const Param&)) { - requests.try_emplace(name, - [this, method](json::Value id, json::Value value) -> async::Task<> { - co_await (this->*method)(std::move(id), - json::deserialize(value)); - }); - } - - template - void addMethod(llvm::StringRef name, async::Task<> (Server::*method)(const Param&)) { - notifications.try_emplace(name, [this, method](json::Value value) -> async::Task<> { - co_await (this->*method)(json::deserialize(value)); - }); - } - - llvm::StringMap requests; - llvm::StringMap notifications; - -private: - /// ============================================================================ - /// Lifecycle Message - /// ============================================================================ - - async::Task<> onInitialize(json::Value id, const proto::InitializeParams& params); - - async::Task<> onInitialized(const proto::InitializedParams& params); - - async::Task<> onShutdown(json::Value id, const proto::None&); - - async::Task<> onExit(const proto::None&); - - /// ============================================================================ - /// Document Synchronization - /// ============================================================================ - - async::Task<> onDidOpen(const proto::DidOpenTextDocumentParams& document); - - async::Task<> onDidChange(const proto::DidChangeTextDocumentParams& document); - - async::Task<> onDidSave(const proto::DidSaveTextDocumentParams& document); - - async::Task<> onDidClose(const proto::DidCloseTextDocumentParams& document); - - /// ============================================================================ - /// Language Features - /// ============================================================================ - - // async::Task<> onGotoDeclaration(json::Value id, const proto::DeclarationParams& params); - // - // async::Task<> onGotoDefinition(json::Value id, const proto::DefinitionParams& params); - // - // async::Task<> onGotoTypeDefinition(json::Value id, const proto::TypeDefinitionParams& - // params); - // - // async::Task<> onGotoImplementation(json::Value id, const proto::ImplementationParams& - // params); - // - // async::Task<> onFindReferences(json::Value id, const proto::ReferenceParams& params); - // - // async::Task<> onPrepareCallHierarchy(json::Value id, - // const proto::CallHierarchyPrepareParams& params); - // - // async::Task<> onIncomingCall(json::Value id, - // const proto::CallHierarchyIncomingCallsParams& params); - // - // async::Task<> onOutgoingCall(json::Value id, - // const proto::CallHierarchyOutgoingCallsParams& params); - // - // async::Task<> onPrepareTypeHierarchy(json::Value id, - // const proto::TypeHierarchyPrepareParams& params); - // - // async::Task<> onSupertypes(json::Value id, const proto::TypeHierarchySupertypesParams& - // params); - // - // async::Task<> onSubtypes(json::Value id, const proto::TypeHierarchySubtypesParams& params); - - // async::Task<> onDocumentHighlight(json::Value id, const proto::DocumentHighlightParams& - // params); - // - // async::Task<> onDocumentLink(json::Value id, const proto::DocumentLinkParams& params); - // - // async::Task<> onHover(json::Value id, const proto::HoverParams& params); - // - // async::Task<> onCodeLens(json::Value id, const proto::CodeLensParams& params); - // - // async::Task<> onFoldingRange(json::Value id, const proto::FoldingRangeParams& params); - // - // async::Task<> onDocumentSymbol(json::Value id, const proto::DocumentSymbolParams& params); - // - // async::Task<> onSemanticTokens(json::Value id, const proto::SemanticTokensParams& params); - // - // async::Task<> onInlayHint(json::Value id, const proto::InlayHintParams& params); - // - // async::Task<> onCodeCompletion(json::Value id, const proto::CompletionParams& params); - // - // async::Task<> onSignatureHelp(json::Value id, const proto::SignatureHelpParams& params); - // - // async::Task<> onCodeAction(json::Value id, const proto::CodeActionParams& params); - // - // async::Task<> onFormatting(json::Value id, const proto::DocumentFormattingParams& params); - // - // async::Task<> onRangeFormatting(json::Value id, - // const proto::DocumentRangeFormattingParams& params); - - /// ============================================================================ - /// Workspace Features - /// ============================================================================ - - async::Task<> onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params); - - /// ============================================================================ - /// Extension - /// ============================================================================ - - async::Task<> onIndexCurrent(const proto::TextDocumentIdentifier& params); - - async::Task<> onIndexAll(const proto::None&); - - async::Task<> onContextCurrent(const proto::TextDocumentIdentifier& params); - - async::Task<> onContextAll(const proto::TextDocumentIdentifier& params); - - async::Task<> onContextSwitch(const proto::TextDocumentIdentifier& params); - - SourceConverter converter; - CompilationDatabase database; Indexer indexer; - Scheduler scheduler; + LSPConverter converter; + CompilationDatabase database; }; } // namespace clice diff --git a/include/Support/Enum.h b/include/Support/Enum.h index f12ad3af..c4ae997c 100644 --- a/include/Support/Enum.h +++ b/include/Support/Enum.h @@ -266,21 +266,18 @@ public: using underlying_type = underlying; - constexpr Enum(underlying value) : m_Value(value) { + constexpr Enum(underlying value) { static_assert( requires { Derived::All; }, "Derived enum must define all possible enum values."); - auto check = [](auto value) { - for(auto kind: Derived::All) { - if(kind == value) { - return true; - } + for(auto& element: Derived::All) { + if(element == value) { + m_Value = element; } - return false; - }; + } - assert(check(value) && "Invalid enum value."); + assert(!m_Value.empty() && "Invalid enum value."); } constexpr Enum(const Enum&) = default; diff --git a/include/Support/FileSystem.h b/include/Support/FileSystem.h index d70eca03..f42d656a 100644 --- a/include/Support/FileSystem.h +++ b/include/Support/FileSystem.h @@ -38,15 +38,15 @@ using namespace llvm::sys::fs; inline std::string resource_dir = ""; -inline llvm::Error init_resource_dir(llvm::StringRef execute) { +inline std::expected init_resource_dir(llvm::StringRef execute) { llvm::SmallString<128> path; path::append(path, path::parent_path(execute), ".."); path::append(path, "lib", "clang", "20"); if(auto error = real_path(path, path)) { - return llvm::make_error(error.message(), error); + return std::unexpected(error); } resource_dir = path.str(); - return llvm::Error::success(); + return std::expected(); } inline std::expected createTemporaryFile(llvm::StringRef prefix, diff --git a/src/Async/Network.cpp b/src/Async/Network.cpp index 1edcf859..468c1796 100644 --- a/src/Async/Network.cpp +++ b/src/Async/Network.cpp @@ -29,7 +29,8 @@ void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) { if(nread == UV_EOF) [[unlikely]] { uv_read_stop(stream); uv_close(uv_cast(*stream), nullptr); - uv_close(uv_cast(*writer), nullptr); + /// FIXME: Figure out why the writer is already closed. + ///uv_close(uv_cast(*writer), nullptr); return; } diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index cd9f6bb4..74a021e3 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -1,6 +1,7 @@ +#include "Support/Logger.h" #include "Compiler/Command.h" +#include "Compiler/Compilation.h" #include "Support/FileSystem.h" -#include "Support/Format.h" namespace clice { @@ -70,4 +71,122 @@ std::expected mangleCommand(llvm::StringRef command, return {}; } +/// Update the compile commands with the given file. +void CompilationDatabase::updateCommands(llvm::StringRef filename) { + auto path = path::real_path(filename); + filename = path; + + /// Read the compile commands from the file. + json::Value json = nullptr; + + if(auto buffer = llvm::MemoryBuffer::getFile(filename)) { + if(auto result = json::parse(buffer->get()->getBuffer())) { + /// llvm::json::Value will hold on string buffer. + /// Do not worry about the lifetime of the buffer. + /// Release buffer to save memory. + json = std::move(result.get()); + } else { + log::warn("Failed to parse json file at {0}, because {1}", + filename, + result.takeError()); + return; + } + } else { + log::warn("Failed to read file {0}", filename); + return; + } + + assert(json.kind() != json::Value::Null && "json is nullptr"); + + if(json.kind() != json::Value::Array) { + log::warn( + "Compilation CompilationDatabase requires a array of object, but get {0}, input file: {1}", + refl::enum_name(json.kind()), + filename); + return; + } + + auto elements = json.getAsArray(); + assert(elements && "json is not an array"); + + for(auto& element: *elements) { + auto object = element.getAsObject(); + if(!object) { + log::warn( + "Compilation CompilationDatabase requires an array of object, but get a array of {0}, input file: {1}", + refl::enum_name(element.kind()), + filename); + continue; + } + + /// FIXME: currently we assume all path here is absolute. + /// Add `directory` field in the future. + + llvm::SmallString<128> path; + + if(auto file = object->getString("file")) { + if(auto error = fs::real_path(*file, path)) { + log::warn("Failed to get real path of {0}, because {1}", *file, error.message()); + continue; + } + } else { + log::warn("The element does not have a file field, input file: {0}", filename); + continue; + } + + auto command = object->getString("command"); + if(!command) { + log::warn("The key:{0} does not have a command field, input file: {1}", path, filename); + continue; + } + + commands[path] = *command; + } + + log::info("Successfully loaded compile commands from {0}, total {1} commands", + filename, + commands.size()); + + /// Scan all files to build module map. + // CompilationParams params; + // for(auto& [path, command]: commands) { + // params.srcPath = path; + // params.command = command; + // + // auto name = scanModuleName(params); + // if(!name.empty()) { + // moduleMap[name] = path; + // } + //} + + log::info("Successfully built module map, total {0} modules", moduleMap.size()); +} + +void CompilationDatabase::updateCommand(llvm::StringRef file, llvm::StringRef command) { + commands[path::real_path(file)] = command; +} + +/// Update the module map with the given file and module name. +void CompilationDatabase::updateModule(llvm::StringRef file, llvm::StringRef name) { + moduleMap[path::real_path(file)] = file; +} + +/// Lookup the compile commands of the given file. +llvm::StringRef CompilationDatabase::getCommand(llvm::StringRef file) { + auto iter = commands.find(file); + if(iter == commands.end()) { + return ""; + } + return iter->second; +} + +/// Lookup the module interface unit file path of the given module name. +llvm::StringRef CompilationDatabase::getModuleFile(llvm::StringRef name) { + auto iter = moduleMap.find(name); + if(iter == moduleMap.end()) { + return ""; + } + return iter->second; +} + } // namespace clice diff --git a/src/Driver/clice.cc b/src/Driver/clice.cc index 3315e71e..2bdd6590 100644 --- a/src/Driver/clice.cc +++ b/src/Driver/clice.cc @@ -1,61 +1,128 @@ -#include "Support/Logger.h" #include "Server/Server.h" +#include "Support/Logger.h" +#include "Support/Format.h" + +#include "llvm/Support/InitLLVM.h" #include "llvm/Support/CommandLine.h" +namespace cl = llvm::cl; using namespace clice; -namespace cl { +namespace { -llvm::cl::opt config("config", - llvm::cl::desc("The path of the config file"), - llvm::cl::value_desc("path")); +static cl::OptionCategory category("clice options"); -llvm::cl::opt mode("mode", llvm::cl::desc("Use pipe mode")); +cl::opt + mode("mode", + cl::cat(category), + cl::value_desc("pipe|socket|indexer"), + cl::desc("The mode of clice, default is pipe, socket is usually used for debugging")); -llvm::cl::opt resource_dir("resource-dir", llvm::cl::desc("Resource dir path")); +cl::opt config_path( + "config", + cl::cat(category), + cl::value_desc("path"), + cl::desc( + "The path of the clice config file, if not specified, the default config will be used")); -} // namespace cl +cl::opt resource_dir( + "resource-dir", + cl::cat(category), + cl::value_desc("path"), + cl::desc(R"(The path of the clang resource directory, default is "../../lib/clang/version")")); + +void printVersion(llvm::raw_ostream& os) { + os << std::format("clice version: {}\n", clice::config::version) + << std::format("llvm version: {}\n", clice::config::llvm_version); +} + +/// Check the command line arguments and initialize the clice. +bool checkArguments(int argc, const char** argv) { + /// Hide unrelated options. + cl::HideUnrelatedOptions(category); + + // Set version printer and parse command line options + cl::SetVersionPrinter(printVersion); + cl::ParseCommandLineOptions(argc, + argv, + "clice is a new generation of language server for C/C++"); -int main(int argc, const char** argv) { for(int i = 0; i < argc; ++i) { - log::info("argv[{0}] = {1}", i, argv[i]); + log::info("argv[{}] = {}", i, argv[i]); } - llvm::cl::SetVersionPrinter([](llvm::raw_ostream& os) { os << "clice version: 0.0.1\n"; }); - llvm::cl::ParseCommandLineOptions(argc, argv, "clice language server"); - - if(cl::config.empty()) { - log::warn("No config file specified; using default configuration."); + // Handle configuration file loading + if(config_path.empty()) { + log::info("No configuration file specified, using default settings"); } else { - config::load(argv[0], cl::config.getValue()); - log::info("Successfully loaded configuration file from {0}.", cl::config.getValue()); - } - - /// Get the resource directory. - if(!cl::resource_dir.empty()) { - fs::resource_dir = cl::resource_dir.getValue(); - } else { - if(auto error = fs::init_resource_dir(argv[0])) { - log::fatal("Failed to get resource directory, because {0}", error); - return 1; + llvm::StringRef path = config_path; + // Try to load the configuration file and check the result + if(auto result = config::load(argv[0], path); result) { + log::info("Configuration file loaded successfully from: {}", path); + } else { + log::warn("Failed to load configuration file from: {} because {}", + path, + result.error()); + return false; } } - static Server server; - auto loop = [](json::Value value) -> async::Task<> { - co_await server.onReceive(value); - }; + // Initialize resource directory + if(resource_dir.empty()) { + log::info("No resource directory specified, using default resource directory"); + // Try to initialize default resource directory + if(auto result = fs::init_resource_dir(argv[0]); !result) { + log::warn("Cannot find default resource directory, because {}", result.error()); + return false; + } + } else { + // Set and check the specified resource directory + fs::resource_dir = resource_dir.getValue(); + if(fs::exists(fs::resource_dir)) { + log::info("Resource directory found: {}", fs::resource_dir); + } else { + log::warn("Resource directory not found: {}", fs::resource_dir); + return false; + } + } + + return true; +} + +} // namespace + +int main(int argc, const char** argv) { + llvm::InitLLVM guard(argc, argv); + llvm::setBugReportMsg( + "Please report bugs to https://github.com/clice-project/clice/issues and include the crash backtrace"); + + if(!checkArguments(argc, argv)) { + return 1; + } async::init(); - if(cl::mode == "pipe") { + /// The global server instance. + static Server instance; + auto loop = [&](json::Value value) -> async::Task<> { + co_await instance.onReceive(value); + }; + + if(mode == "pipe") { async::net::listen(loop); log::info("Server starts listening on stdin/stdout"); - } else if(cl::mode == "socket") { + } else if(mode == "socket") { async::net::listen("127.0.0.1", 50051, loop); log::info("Server starts listening on {}:{}", "127.0.0.1", 50051); + } else if(mode == "indexer") { + /// TODO: + } else { + log::fatal("Invalid mode: {}", mode.getValue()); + return 1; } async::run(); + + return 0; } diff --git a/src/Driver/unit_tests.cc b/src/Driver/unit_tests.cc index 3895dfd0..15010f33 100644 --- a/src/Driver/unit_tests.cc +++ b/src/Driver/unit_tests.cc @@ -37,8 +37,8 @@ int main(int argc, char** argv) { if(!cl::resource_dir.empty()) { fs::resource_dir = cl::resource_dir.getValue(); } else { - if(auto error = fs::init_resource_dir(argv[0])) { - llvm::outs() << std::format("Failed to get resource directory, because {}\n", error); + if(auto result = fs::init_resource_dir(argv[0]); !result) { + llvm::outs() << std::format("Failed to get resource directory, because {}\n", result.error()); return 1; } } diff --git a/src/Index/FeatureIndex.cpp b/src/Index/FeatureIndex.cpp index 4af36a89..74d8eb84 100644 --- a/src/Index/FeatureIndex.cpp +++ b/src/Index/FeatureIndex.cpp @@ -44,8 +44,25 @@ Shared indexFeature(ASTInfo& info) { return result; } -llvm::ArrayRef FeatureIndex::semanticTokens() const { +std::vector FeatureIndex::semanticTokens() const { return binary::Proxy{base, base}.get<"tokens">().as_array(); } +std::vector FeatureIndex::foldingRanges() const { + auto array = binary::Proxy{base, base}.get<"foldings">(); + + std::vector result; + result.reserve(array.size()); + + /// FIXME: Use iterator or other thing to make cast easier. + for(std::size_t i = 0, n = array.size(); i < n; ++i) { + auto&& range = array[i]; + result.emplace_back(range.get<"range">().value(), + range.get<"kind">().value(), + range.get<"text">().as_string().str()); + } + + return result; +} + } // namespace clice::index diff --git a/src/Index/Index2.cpp b/src/Index/Index2.cpp new file mode 100644 index 00000000..60c1f4ae --- /dev/null +++ b/src/Index/Index2.cpp @@ -0,0 +1,36 @@ +#include "Index/Shared2.h" +#include "Compiler/Compilation.h" + +namespace clice::index { + +Shared Index2::build(ASTInfo& AST) { + llvm::DenseMap indices; + + auto symbolIndices = index::index(AST); + for(auto& [fid, index]: symbolIndices) { + indices[fid].symbol.emplace(std::move(index)); + } + + auto featureIndices = index::indexFeature(AST); + for(auto& [fid, index]: featureIndices) { + indices[fid].feature.emplace(std::move(index)); + } + + for(auto& [fid, index]: indices) { + if(index.symbol) { + auto data = llvm::ArrayRef(reinterpret_cast(index.symbol->base), + index.symbol->size); + index.symbolHash = llvm::xxh3_128bits(data); + } + + if(index.feature) { + auto data = llvm::ArrayRef(reinterpret_cast(index.feature->base), + index.feature->size); + index.featureHash = llvm::xxh3_128bits(data); + } + } + + return indices; +} + +} // namespace clice::index diff --git a/src/Server/Cache.cpp b/src/Server/Cache.cpp deleted file mode 100644 index 3248824a..00000000 --- a/src/Server/Cache.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Server/Cache.h" - -namespace clice { - -void CacheController::loadFromDisk() { - -} - -void CacheController::saveToDisk() { - -} - -async::Task<> CacheController::prepare(CompilationParams& params) { - /// 首先要区分这个文件是不是 module file. - bool isModule = false; - - if(isModule){ - /// Update dependent mods .... - } - - /// 准备构建 PCH ... - - co_return; -} - -} // namespace clice diff --git a/src/Server/Config.cpp b/src/Server/Config.cpp index 0de8527c..96de7048 100644 --- a/src/Server/Config.cpp +++ b/src/Server/Config.cpp @@ -73,19 +73,18 @@ static void parse(Object& object, auto&& value) { } } -void load(llvm::StringRef execute, llvm::StringRef filename) { +std::expected load(llvm::StringRef execute, llvm::StringRef filename) { predefined["version"] = "0.0.1"; predefined["binary"] = execute; predefined["llvm_version"] = "20"; auto toml = toml::parse_file(filename); if(toml.failed()) { - log::fatal("Failed to parse config file: {0}. Because: {1}", - filename, - toml.error().description()); + return std::unexpected(toml.error().description()); } parse(config, toml.table()); + return {}; } /// replace all predefined variables in the text. diff --git a/src/Server/Database.cpp b/src/Server/Database.cpp deleted file mode 100644 index eda0e58a..00000000 --- a/src/Server/Database.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "Support/Logger.h" -#include "Server/Database.h" -#include "Support/FileSystem.h" -#include "Compiler/Compilation.h" - -namespace clice { - -/// Update the compile commands with the given file. -void CompilationDatabase::updateCommands(llvm::StringRef filename) { - auto path = path::real_path(filename); - filename = path; - - /// Read the compile commands from the file. - json::Value json = nullptr; - - if(auto buffer = llvm::MemoryBuffer::getFile(filename)) { - if(auto result = json::parse(buffer->get()->getBuffer())) { - /// llvm::json::Value will hold on string buffer. - /// Do not worry about the lifetime of the buffer. - /// Release buffer to save memory. - json = std::move(result.get()); - } else { - log::warn("Failed to parse json file at {0}, because {1}", - filename, - result.takeError()); - return; - } - } else { - log::warn("Failed to read file {0}", filename); - return; - } - - assert(json.kind() != json::Value::Null && "json is nullptr"); - - if(json.kind() != json::Value::Array) { - log::warn("Compilation Database requires a array of object, but get {0}, input file: {1}", - refl::enum_name(json.kind()), - filename); - return; - } - - auto elements = json.getAsArray(); - assert(elements && "json is not an array"); - - for(auto& element: *elements) { - auto object = element.getAsObject(); - if(!object) { - log::warn( - "Compilation Database requires an array of object, but get a array of {0}, input file: {1}", - refl::enum_name(element.kind()), - filename); - continue; - } - - /// FIXME: currently we assume all path here is absolute. - /// Add `directory` field in the future. - - llvm::SmallString<128> path; - - if(auto file = object->getString("file")) { - if(auto error = fs::real_path(*file, path)) { - log::warn("Failed to get real path of {0}, because {1}", *file, error.message()); - continue; - } - } else { - log::warn("The element does not have a file field, input file: {0}", filename); - continue; - } - - auto command = object->getString("command"); - if(!command) { - log::warn("The key:{0} does not have a command field, input file: {1}", path, filename); - continue; - } - - commands[path] = *command; - } - - log::info("Successfully loaded compile commands from {0}, total {1} commands", - filename, - commands.size()); - - /// Scan all files to build module map. - // CompilationParams params; - // for(auto& [path, command]: commands) { - // params.srcPath = path; - // params.command = command; - // - // auto name = scanModuleName(params); - // if(!name.empty()) { - // moduleMap[name] = path; - // } - //} - - log::info("Successfully built module map, total {0} modules", moduleMap.size()); -} - -void CompilationDatabase::updateCommand(llvm::StringRef file, llvm::StringRef command) { - commands[path::real_path(file)] = command; -} - -/// Update the module map with the given file and module name. -void CompilationDatabase::updateModule(llvm::StringRef file, llvm::StringRef name) { - moduleMap[path::real_path(file)] = file; -} - -/// Lookup the compile commands of the given file. -llvm::StringRef CompilationDatabase::getCommand(llvm::StringRef file) { - auto iter = commands.find(file); - if(iter == commands.end()) { - return ""; - } - return iter->second; -} - -/// Lookup the module interface unit file path of the given module name. -llvm::StringRef CompilationDatabase::getModuleFile(llvm::StringRef name) { - auto iter = moduleMap.find(name); - if(iter == moduleMap.end()) { - return ""; - } - return iter->second; -} - -} // namespace clice diff --git a/src/Server/Document.cpp b/src/Server/Document.cpp deleted file mode 100644 index 66c49505..00000000 --- a/src/Server/Document.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "Basic/SourceConverter.h" -#include "Server/Server.h" - -namespace clice { - -async::Task<> Server::onDidOpen(const proto::DidOpenTextDocumentParams& params) { - co_return; -} - -async::Task<> Server::onDidChange(const proto::DidChangeTextDocumentParams& document) { - co_return; -} - -async::Task<> Server::onDidSave(const proto::DidSaveTextDocumentParams& document) { - co_return; -} - -async::Task<> Server::onDidClose(const proto::DidCloseTextDocumentParams& document) { - co_return; -} - -} // namespace clice diff --git a/src/Server/Extension.cpp b/src/Server/Extension.cpp index 85bbb0e5..fccc8093 100644 --- a/src/Server/Extension.cpp +++ b/src/Server/Extension.cpp @@ -2,27 +2,27 @@ namespace clice { -async::Task<> Server::onIndexCurrent(const proto::TextDocumentIdentifier& params) { - auto path = SourceConverter::toPath(params.uri); - /// co_await indexer.index(path); - co_return; -} - -async::Task<> Server::onIndexAll(const proto::None& params) { - /// co_await indexer.indexAll(); - co_return; -} - -async::Task<> Server::onContextCurrent(const proto::TextDocumentIdentifier& params) { - co_return; -} - -async::Task<> Server::onContextAll(const proto::TextDocumentIdentifier& params) { - co_return; -} - -async::Task<> Server::onContextSwitch(const proto::TextDocumentIdentifier& params) { - co_return; -} +// async::Task<> Server::onIndexCurrent(const proto::TextDocumentIdentifier& params) { +// auto path = SourceConverter::toPath(params.uri); +// /// co_await indexer.index(path); +// co_return; +// } +// +// async::Task<> Server::onIndexAll(const proto::None& params) { +// /// co_await indexer.indexAll(); +// co_return; +// } +// +// async::Task<> Server::onContextCurrent(const proto::TextDocumentIdentifier& params) { +// co_return; +// } +// +// async::Task<> Server::onContextAll(const proto::TextDocumentIdentifier& params) { +// co_return; +// } +// +// async::Task<> Server::onContextSwitch(const proto::TextDocumentIdentifier& params) { +// co_return; +// } } // namespace clice diff --git a/src/Server/IncludeGraph.cpp b/src/Server/IncludeGraph.cpp index 7960f03b..6b60b6cf 100644 --- a/src/Server/IncludeGraph.cpp +++ b/src/Server/IncludeGraph.cpp @@ -3,6 +3,7 @@ #include "Server/IncludeGraph.h" #include "Index/SymbolIndex.h" #include "Index/FeatureIndex.h" +#include "Index/Shared2.h" #include "Support/Compare.h" #include "Support/Logger.h" #include "llvm/ADT/DenseSet.h" @@ -133,269 +134,6 @@ async::Task<> IncludeGraph::index(llvm::StringRef file, CompilationDatabase& dat co_await updateIndices(**info, tu, files); } -proto::HeaderContextGroups IncludeGraph::contextAll(llvm::StringRef file) { - llvm::StringMap> groups; - - if(auto iter = headers.find(file); iter != headers.end()) { - auto header = iter->second; - for(auto& [tu, contexts]: header->contexts) { - for(auto& context: contexts) { - auto& group = groups[tu->indexPath]; - if(group.size() < 10) { - group.emplace_back(file.str(), tu->srcPath, context.index, tu->version); - } - } - } - } - - proto::HeaderContextGroups result; - for(auto& [_, group]: groups) { - result.emplace_back(std::move(group)); - } - return result; -} - -std::optional IncludeGraph::contextCurrent(llvm::StringRef file) { - if(auto iter = headers.find(file); iter != headers.end()) { - auto header = iter->second; - auto [tu, index] = header->active; - if(tu) { - return proto::HeaderContext{ - .srcFile = file.str(), - .contextFile = tu->srcPath, - .index = index, - .version = tu->version, - }; - } - - /// If no active translation unit, we just return the first context. - if(!header->contexts.empty()) { - /// FIXME: Is it possible that a tu does not have any context? - auto& [tu, contexts] = *header->contexts.begin(); - header->active = {tu, 0}; - return proto::HeaderContext{ - .srcFile = file.str(), - .contextFile = tu->srcPath, - .index = 0, - .version = tu->version, - }; - } - } - - return std::nullopt; -} - -void IncludeGraph::contextSwitch(const proto::HeaderContext& context) { - Header* header = nullptr; - if(auto iter = headers.find(context.srcFile); iter != headers.end()) { - header = iter->second; - } else { - return; - } - - TranslationUnit* tu = nullptr; - if(auto iter = tus.find(context.contextFile); iter != tus.end()) { - tu = iter->second; - } else { - return; - } - - /// Check whether the context is valid. - if(tu->version != context.version) { - return; - } - - /// Switch to the new context. - header->active = {tu, context.index}; -} - -std::vector - IncludeGraph::contextResolve(const proto::HeaderContext& context) { - Header* header = nullptr; - if(auto iter = headers.find(context.srcFile); iter != headers.end()) { - header = iter->second; - } else { - return {}; - } - - TranslationUnit* tu = nullptr; - if(auto iter = tus.find(context.contextFile); iter != tus.end()) { - tu = iter->second; - } else { - return {}; - } - - if(tu->version != context.version) { - return {}; - } - - std::vector locations; - - auto include = header->contexts[tu][context.index].include; - while(include != -1) { - auto& location = tu->locations[include]; - locations.push_back(proto::IncludeLocation{ - .line = location.line - 1, - .filename = pathPool[location.filename], - }); - include = location.include; - } - - return locations; -} - -std::vector IncludeGraph::indices(TranslationUnit* tu) { - std::vector indices; - if(tu) { - indices.emplace_back(tu->indexPath); - for(auto& header: tu->headers) { - for(auto& context: header->contexts[tu]) { - indices.emplace_back(header->indices[context.index].path); - } - } - } else { - for(auto& [_, tu]: tus) { - indices.emplace_back(tu->indexPath); - } - - for(auto& [_, header]: headers) { - for(auto& index: header->indices) { - indices.emplace_back(index.path); - } - } - } - return indices; -} - -async::Task> - IncludeGraph::resolve(const proto::TextDocumentPositionParams& params) { - std::vector ids; - auto path = SourceConverter::toPath(params.textDocument.uri); - - std::uint32_t offset = 0; - { - auto content = co_await async::fs::read(path); - if(!content) { - co_return ids; - } - offset = SC.toOffset(*content, params.position); - } - - std::vector indices; - if(auto iter = tus.find(path); iter != tus.end()) { - indices.emplace_back(iter->second->indexPath); - } else if(auto iter = headers.find(path); iter != headers.end()) { - /// FIXME: What is the expected result of resolving a symbol that may refers different - /// declaration in different header contexts? Currently, we just return the first one. - for(auto& index: iter->second->indices) { - indices.emplace_back(index.path); - } - } - - co_await async::gather(indices, [&](const std::string& path) -> async::Task { - auto binary = co_await async::fs::read(path + ".sidx"); - if(!binary) { - co_return true; - } - - llvm::SmallVector symbols; - co_await async::submit([&] { - index::SymbolIndex index(binary->data(), binary->size(), false); - index.locateSymbols(offset, symbols); - }); - - if(symbols.empty()) { - co_return true; - } - - /// If we found the symbol, we just return it. And break other tasks. - for(auto& symbol: symbols) { - ids.emplace_back(symbol.id(), symbol.name().str()); - } - co_return false; - }); - - co_return ids; -} - -async::Task<> IncludeGraph::lookup(llvm::ArrayRef targets, - llvm::ArrayRef files, - LookupCallback callback) { - co_await async::gather(files, [&](const std::string& indexPath) -> async::Task { - auto binary = co_await async::fs::read(indexPath + ".sidx"); - if(!binary) { - co_return false; - } - - llvm::SmallVector symbols; - index::SymbolIndex index(binary->data(), binary->size(), false); - co_await async::submit([&] { - for(auto& target: targets) { - if(auto symbol = index.locateSymbol(target.hash, target.name)) { - symbols.emplace_back(*symbol); - } - } - }); - - if(symbols.empty()) { - co_return true; - } - - auto srcPath = index.path().str(); - auto content = co_await async::fs::read(srcPath); - if(!content) { - co_return true; - } - - for(auto& symbol: symbols) { - if(!callback(srcPath, *content, symbol)) { - co_return false; - } - } - - co_return true; - }); -} - -async::Task IncludeGraph::lookup(const proto::ReferenceParams& params, - RelationKind kind) { - auto ids = co_await resolve(params); - /// FIXME: If the size of ids equal to one and it is not an external symbol, we should - /// just search the symbol in the current translation unit. - - proto::ReferenceResult result; - - std::vector files = indices(); - co_await lookup(ids, - files, - [&](llvm::StringRef path, - llvm::StringRef content, - const index::SymbolIndex::Symbol& symbol) { - /// FIXME: We may should collect and sort the ranges in one file. - /// So that we can cut off the context to speed up the process. - for(auto relation: symbol.relations()) { - if(relation.kind() & kind) { - result.emplace_back(proto::Location{ - .uri = SourceConverter::toURI(path), - .range = SC.toRange(*relation.range(), content), - }); - } - } - return true; - }); - - co_return result; -} - -async::Task - IncludeGraph::prepareHierarchy(const proto::HierarchyPrepareParams& params) { - auto ids = co_await resolve(params); - - proto::HierarchyPrepareResult result; - std::vector files = indices(); - co_return result; -} - std::string IncludeGraph::getIndexPath(llvm::StringRef file) { auto now = std::chrono::system_clock::now(); auto ms = std::chrono::duration_cast(now.time_since_epoch()).count(); @@ -438,40 +176,47 @@ async::Task IncludeGraph::check(llvm::StringRef file) { uint32_t IncludeGraph::addIncludeChain(std::vector& locations, llvm::DenseMap& files, clang::SourceManager& SM, - clang::FileID fid) { + clang::FileID fid, + ASTInfo& AST) { auto [iter, success] = files.try_emplace(fid, locations.size()); if(!success) { return iter->second; } auto index = iter->second; - locations.emplace_back(); - auto entry = SM.getFileEntryRefForID(fid); - assert(entry && "Invalid file entry"); - { - auto path = path::real_path(entry->getName()); - auto [iter, success] = pathIndices.try_emplace(path, pathPool.size()); - locations[index].filename = iter->second; - } - - if(auto presumed = SM.getPresumedLoc(SM.getIncludeLoc(fid), false); presumed.isValid()) { + auto includeLoc = SM.getIncludeLoc(fid); + if(includeLoc.isValid()) { + auto presumed = SM.getPresumedLoc(includeLoc, false); + locations.emplace_back(); locations[index].line = presumed.getLine(); - auto include = addIncludeChain(locations, files, SM, presumed.getFileID()); + + auto path = AST.getFilePath(presumed.getFileID()); + auto [iter, success] = pathIndices.try_emplace(path, pathPool.size()); + if(success) { + pathPool.emplace_back(path); + } + locations[index].file = iter->second; + + uint32_t include = -1; + if(presumed.getIncludeLoc().isValid()) { + include = + addIncludeChain(locations, files, SM, SM.getFileID(presumed.getIncludeLoc()), AST); + } locations[index].include = include; } return index; } -void IncludeGraph::addContexts(ASTInfo& info, +void IncludeGraph::addContexts(ASTInfo& AST, TranslationUnit* tu, llvm::DenseMap& files) { - auto& SM = info.srcMgr(); + auto& SM = AST.srcMgr(); std::vector locations; - for(auto& [fid, directive]: info.directives()) { + for(auto& [fid, directive]: AST.directives()) { for(auto& include: directive.includes) { /// If the include is invalid, it indicates that the file is skipped because of /// include guard, or `#pragma once`. Such file cannot provide header context. @@ -481,7 +226,7 @@ void IncludeGraph::addContexts(ASTInfo& info, } /// Add all include chains. - addIncludeChain(locations, files, SM, include.fid); + addIncludeChain(locations, files, SM, include.fid, AST); } } @@ -496,17 +241,15 @@ void IncludeGraph::addContexts(ASTInfo& info, continue; } - auto entry = SM.getFileEntryRefForID(fid); - assert(entry && "Invalid file entry"); - auto name = path::real_path(entry->getName()); + auto path = AST.getFilePath(fid); Header* header = nullptr; - if(auto iter = headers.find(name); iter != headers.end()) { + if(auto iter = headers.find(path); iter != headers.end()) { header = iter->second; } else { header = new Header; - header->srcPath = name; - headers.try_emplace(name, header); + header->srcPath = path; + headers.try_emplace(path, header); } /// Add new header context. @@ -526,43 +269,7 @@ void IncludeGraph::addContexts(ASTInfo& info, async::Task<> IncludeGraph::updateIndices(ASTInfo& info, TranslationUnit* tu, llvm::DenseMap& files) { - struct Index { - llvm::XXH128_hash_t symbolHash = {0, 0}; - std::optional symbol; - - llvm::XXH128_hash_t featureHash = {0, 0}; - std::optional feature; - }; - - auto indices = co_await async::submit([&info] { - llvm::DenseMap indices; - - auto symbolIndices = index::index(info); - for(auto& [fid, index]: symbolIndices) { - indices[fid].symbol.emplace(std::move(index)); - } - - auto featureIndices = index::indexFeature(info); - for(auto& [fid, index]: featureIndices) { - indices[fid].feature.emplace(std::move(index)); - } - - for(auto& [fid, index]: indices) { - if(index.symbol) { - auto data = llvm::ArrayRef(reinterpret_cast(index.symbol->base), - index.symbol->size); - index.symbolHash = llvm::xxh3_128bits(data); - } - - if(index.feature) { - auto data = llvm::ArrayRef(reinterpret_cast(index.feature->base), - index.feature->size); - index.featureHash = llvm::xxh3_128bits(data); - } - } - - return indices; - }); + auto indices = co_await async::submit([&info] { return index::Index2::build(info); }); auto& SM = info.srcMgr(); @@ -572,28 +279,12 @@ async::Task<> IncludeGraph::updateIndices(ASTInfo& info, tu->indexPath = getIndexPath(tu->srcPath); } - if(index.symbol) { - co_await async::fs::write(tu->indexPath + ".sidx", - index.symbol->base, - index.symbol->size); - } - - if(index.feature) { - co_await async::fs::write(tu->indexPath + ".fidx", - index.feature->base, - index.feature->size); - } + co_await index.write(tu->indexPath); continue; } - auto entry = SM.getFileEntryRefForID(fid); - if(!entry) { - log::info("Invalid file entry for file id: {}", fid.getHashValue()); - } - assert(entry && "Invalid file entry"); - - auto name = path::real_path(entry->getName()); + auto name = info.getFilePath(fid); assert(headers.contains(name) && "Invalid header name"); auto header = headers[name]; @@ -629,35 +320,7 @@ async::Task<> IncludeGraph::updateIndices(ASTInfo& info, .featureHash = index.featureHash, }); - // if(header->srcPath == "/home/ykiko/C++/clice/include/Support/JSON.h") { - // if(index.symbol) { - // auto json = index.symbol->toJSON(); - // llvm::SmallString<128> path; - // llvm::raw_svector_ostream stream(path); - // stream << json; - // - // co_await async::fs::write(indices.back().path + ".json", path.data(), - // path.size()); - // } - // - // // if(index.feature) { - // // println("{{ hash: {}, index: {} }}", - // // json::serialize(index.featureHash), - // // json::serialize(index.feature->semanticTokens())); - // // } - //} - - if(index.symbol) { - co_await async::fs::write(indices.back().path + ".sidx", - index.symbol->base, - index.symbol->size); - } - - if(index.feature) { - co_await async::fs::write(indices.back().path + ".fidx", - index.feature->base, - index.feature->size); - } + co_await index.write(indices.back().path); } } diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index 170aded2..140b1fe7 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -113,4 +113,326 @@ async::Task<> Indexer::index(std::string file) { tasks.try_emplace(file, std::move(next)); } +Header* Indexer::getHeader(llvm::StringRef file) const { + auto it = headers.find(file); + if(it == headers.end()) { + return nullptr; + } + return it->second; +} + +TranslationUnit* Indexer::getTranslationUnit(llvm::StringRef file) const { + auto it = tus.find(file); + if(it == tus.end()) { + return nullptr; + } + return it->second; +} + +std::optional Indexer::currentContext(llvm::StringRef file) const { + auto header = getHeader(file); + if(!header || !header->active.valid()) { + return std::nullopt; + } + + auto& active = header->active; + return proto::HeaderContext{ + .file = active.tu->srcPath, + .version = active.tu->version, + .include = active.context.include, + }; +} + +bool Indexer::switchContext(llvm::StringRef headerFile, proto::HeaderContext context) { + Header* header = getHeader(headerFile); + if(!header) { + return false; + } + + TranslationUnit* tu = getTranslationUnit(context.file); + if(!tu || tu->version != context.version) { + return false; + } + + auto index = header->getIndex(tu, context.include); + if(!index) { + return false; + } + + header->active = HeaderContext{ + .tu = tu, + .context = {.index = *index, .include = context.include}, + }; + + return true; +} + +std::vector Indexer::resolveContext(proto::HeaderContext context) const { + auto tu = getTranslationUnit(context.file); + if(!tu || tu->version != context.version) { + return {}; + } + + auto include = context.include; + auto& locations = tu->locations; + if(locations.size() <= include) [[unlikely]] { + /// FIXME: This should never occur, we should log. + return {}; + } + + std::vector result; + while(include != -1) { + auto& location = locations[include]; + result.emplace_back(proto::IncludeLocation{ + .line = location.line - 1, + .file = pathPool[location.file], + }); + include = location.include; + } + return result; +} + +std::vector Indexer::allContexts(llvm::StringRef headerFile, + uint32_t limit, + llvm::StringRef contextFile) const { + auto header = getHeader(headerFile); + if(!header) { + return {}; + } + + llvm::DenseMap> groups; + + if(auto tu = getTranslationUnit(contextFile)) { + for(auto context: header->contexts[tu]) { + groups[context.index].emplace_back(proto::HeaderContext{ + .file = tu->srcPath, + .version = tu->version, + .include = context.include, + }); + } + } else { + for(auto&& [tu, contexts]: header->contexts) { + for(auto& context: contexts) { + auto& group = groups[context.index]; + if(group.size() >= 10) { + continue; + } + + group.emplace_back(proto::HeaderContext{ + .file = tu->srcPath, + .version = tu->version, + .include = context.include, + }); + } + } + } + + std::vector result; + for(auto& [index, group]: groups) { + result.emplace_back(header->indices[index].path, std::move(group)); + } + return result; +} + +std::vector Indexer::indices(TranslationUnit* tu) { + std::vector indices; + if(tu) { + indices.emplace_back(tu->indexPath); + for(auto& header: tu->headers) { + for(auto& context: header->contexts[tu]) { + indices.emplace_back(header->indices[context.index].path); + } + } + } else { + for(auto& [_, tu]: tus) { + indices.emplace_back(tu->indexPath); + } + + for(auto& [_, header]: headers) { + for(auto& index: header->indices) { + indices.emplace_back(index.path); + } + } + } + return indices; +} + +async::Task> + Indexer::resolve(const proto::TextDocumentPositionParams& params) { + std::vector ids; + auto path = SourceConverter::toPath(params.textDocument.uri); + + std::uint32_t offset = 0; + { + auto content = co_await async::fs::read(path); + if(!content) { + co_return ids; + } + offset = SC.toOffset(*content, params.position); + } + + std::vector indices; + if(auto iter = tus.find(path); iter != tus.end()) { + indices.emplace_back(iter->second->indexPath); + } else if(auto iter = headers.find(path); iter != headers.end()) { + /// FIXME: What is the expected result of resolving a symbol that may refers different + /// declaration in different header contexts? Currently, we just return the first one. + for(auto& index: iter->second->indices) { + indices.emplace_back(index.path); + } + } + + co_await async::gather(indices, [&](const std::string& path) -> async::Task { + auto binary = co_await async::fs::read(path + ".sidx"); + if(!binary) { + co_return true; + } + + llvm::SmallVector symbols; + co_await async::submit([&] { + index::SymbolIndex index(binary->data(), binary->size(), false); + index.locateSymbols(offset, symbols); + }); + + if(symbols.empty()) { + co_return true; + } + + /// If we found the symbol, we just return it. And break other tasks. + for(auto& symbol: symbols) { + ids.emplace_back(symbol.id(), symbol.name().str()); + } + co_return false; + }); + + co_return ids; +} + +async::Task<> Indexer::lookup(llvm::ArrayRef targets, + llvm::ArrayRef files, + LookupCallback callback) { + co_await async::gather(files, [&](const std::string& indexPath) -> async::Task { + auto binary = co_await async::fs::read(indexPath + ".sidx"); + if(!binary) { + co_return false; + } + + llvm::SmallVector symbols; + index::SymbolIndex index(binary->data(), binary->size(), false); + co_await async::submit([&] { + for(auto& target: targets) { + if(auto symbol = index.locateSymbol(target.hash, target.name)) { + symbols.emplace_back(*symbol); + } + } + }); + + if(symbols.empty()) { + co_return true; + } + + auto srcPath = index.path().str(); + auto content = co_await async::fs::read(srcPath); + if(!content) { + co_return true; + } + + for(auto& symbol: symbols) { + if(!callback(srcPath, *content, symbol)) { + co_return false; + } + } + + co_return true; + }); +} + +async::Task Indexer::lookup(const proto::ReferenceParams& params, + RelationKind kind) { + auto ids = co_await resolve(params); + /// FIXME: If the size of ids equal to one and it is not an external symbol, we should + /// just search the symbol in the current translation unit. + + proto::ReferenceResult result; + + std::vector files = indices(); + co_await lookup(ids, + files, + [&](llvm::StringRef path, + llvm::StringRef content, + const index::SymbolIndex::Symbol& symbol) { + /// FIXME: We may should collect and sort the ranges in one file. + /// So that we can cut off the context to speed up the process. + for(auto relation: symbol.relations()) { + if(relation.kind() & kind) { + result.emplace_back(proto::Location{ + .uri = SourceConverter::toURI(path), + .range = SC.toRange(*relation.range(), content), + }); + } + } + return true; + }); + + co_return result; +} + +async::Task + Indexer::prepareHierarchy(const proto::HierarchyPrepareParams& params) { + auto ids = co_await resolve(params); + + proto::HierarchyPrepareResult result; + std::vector files = indices(); + co_return result; +} + +async::Task> + Indexer::getFeatureIndex(std::string& buffer, llvm::StringRef file) const { + std::string path; + if(auto tu = tus.find(file); tu != tus.end()) { + path = tu->second->indexPath; + } + + if(auto header = headers.find(file); header != headers.end()) { + for(auto&& index: header->second->indices) { + path = index.path; + break; + } + } + + auto content = co_await async::fs::read(path + ".fidx"); + if(!content) { + co_return std::nullopt; + } + + buffer = std::move(*content); + + co_return index::FeatureIndex(buffer.data(), buffer.size(), false); +} + +async::Task> + Indexer::semanticTokens(llvm::StringRef file) const { + std::vector result; + + std::string buffer; + auto index = co_await getFeatureIndex(buffer, file); + if(!index) { + co_return result; + } + + co_return index->semanticTokens(); +} + +async::Task> Indexer::foldingRanges(llvm::StringRef file) const { + std::vector result; + + std::string buffer; + auto index = co_await getFeatureIndex(buffer, file); + if(!index) { + co_return result; + } + + co_return index->foldingRanges(); +} + } // namespace clice diff --git a/src/Server/LSPConverter.cpp b/src/Server/LSPConverter.cpp index bf6b5d16..d7af9bdb 100644 --- a/src/Server/LSPConverter.cpp +++ b/src/Server/LSPConverter.cpp @@ -1,4 +1,5 @@ #include "Server/LSPConverter.h" +#include "Basic/SourceConverter.h" namespace clice { @@ -136,8 +137,8 @@ std::uint32_t remeasure(llvm::StringRef content, proto::PositionEncodingKind kin class PositionConverter { public: - PositionConverter(llvm::StringRef content, proto::PositionEncodingKind kind) : - content(content), kind(kind) {} + PositionConverter(llvm::StringRef content, proto::PositionEncodingKind encoding) : + content(content), encoding(encoding) {} /// Convert a offset to a proto::Position with given encoding. /// The input offset must be UTF-8 encoded and in order. @@ -167,7 +168,7 @@ public: auto lineContent = content.substr(lastLineOffset, lineLength); auto position = proto::Position{ .line = line, - .character = remeasure(lineContent, kind), + .character = remeasure(lineContent, encoding), }; /// Cache the result. @@ -187,10 +188,11 @@ public: } ranges::sort(offsets); - ranges::unique(offsets); for(auto&& offset: offsets) { - toPosition(offset); + if(auto it = cache.find(offset); it == cache.end()) { + cache.try_emplace(offset, toPosition(offset)); + } } } @@ -212,42 +214,54 @@ private: llvm::DenseMap cache; llvm::StringRef content; - proto::PositionEncodingKind kind; + proto::PositionEncodingKind encoding; }; } // namespace -LSPConverter::Result LSPConverter::convert(llvm::StringRef path, - llvm::ArrayRef tokens) { - /// FIXME: Use a better way to handle file content. - auto file = co_await async::fs::read(path.str()); - if(!file) { - co_return json::Value(nullptr); +proto::InitializeResult LSPConverter::initialize(json::Value value) { + params = json::deserialize(value); + + proto::InitializeResult result = {}; + result.serverInfo.name = "clice"; + result.serverInfo.version = "0.0.1"; + + auto& semantictokens = result.capabilities.semanticTokensProvider; + for(auto& name: SymbolKind::all()) { + std::string type{name}; + type[0] = std::tolower(type[0]); + semantictokens.legend.tokenTypes.emplace_back(std::move(type)); } - llvm::StringRef content = *file; - struct SemanticTokens { - /// The actual tokens. - std::vector data; + return result; +} - void add(std::uint32_t line, - std::uint32_t character, - std::uint32_t length, - SymbolKind kind, - SymbolModifiers modifiers) { - /// FIXME: Add a map between lsp kinds and our kinds. - /// [line, character, length, tokenType, tokenModifiers] - data.emplace_back(line); - data.emplace_back(character); - data.emplace_back(length); - data.emplace_back(kind.value()); - data.emplace_back(modifiers.value()); - } +llvm::StringRef LSPConverter::workspace() { + if(workspacePath.empty()) { + workspacePath = SourceConverter::toPath(params.workspaceFolders[0].uri); + } + return workspacePath; +} + +proto::SemanticTokens LSPConverter::transform(llvm::StringRef content, + llvm::ArrayRef tokens) { + proto::SemanticTokens result; + + auto addGroup = [&](uint32_t line, + uint32_t character, + uint32_t length, + SymbolKind kind, + SymbolModifiers modifiers) { + result.data.emplace_back(line); + result.data.emplace_back(character); + result.data.emplace_back(length); + result.data.emplace_back(kind.value()); + + /// FIXME: + result.data.emplace_back(0); }; - SemanticTokens result; - - PositionConverter converter(content, kind); + PositionConverter converter(content, encoding()); std::uint32_t lastLine = 0; std::uint32_t lastChar = 0; @@ -260,7 +274,7 @@ LSPConverter::Result LSPConverter::convert(llvm::StringRef path, std::uint32_t line = beginLine - lastLine; std::uint32_t character = (line == 0 ? beginChar - lastChar : beginChar); std::uint32_t length = endChar - beginChar; - result.add(line, character, length, token.kind, token.modifiers); + addGroup(line, character, length, token.kind, token.modifiers); } else { /// If the token spans multiple lines, split it into multiple tokens. auto subContent = content.substr(beginOffset, endOffset - beginOffset); @@ -288,8 +302,8 @@ LSPConverter::Result LSPConverter::convert(llvm::StringRef path, } std::uint32_t length = - remeasure(subContent.substr(lastLineOffset, lineLength), kind); - result.add(line, character, length, token.kind, token.modifiers); + remeasure(subContent.substr(lastLineOffset, lineLength), encoding()); + addGroup(line, character, length, token.kind, token.modifiers); lastLineOffset += lineLength; lineLength = 0; @@ -298,81 +312,24 @@ LSPConverter::Result LSPConverter::convert(llvm::StringRef path, /// Process the last line if it's not empty. if(lineLength > 0) { - std::uint32_t length = remeasure(subContent.substr(lastLineOffset), kind); - result.add(1, 0, length, token.kind, token.modifiers); + std::uint32_t length = remeasure(subContent.substr(lastLineOffset), encoding()); + addGroup(1, 0, length, token.kind, token.modifiers); } } lastLine = endLine; - lastChar = endChar; + lastChar = beginChar; } - co_return json::serialize(result); + return result; } -namespace proto { - -/// A set of predefined range kinds. -enum class FoldingRangeKind { - /// Folding range for a comment. - Comment, - - /// Folding range for imports or includes. - Imports, - - /// Folding range for a region. - Region, -}; - -/// Represents a folding range. To be valid, start and end line must be bigger -/// than zero and smaller than the number of lines in the document. Clients -/// are free to ignore invalid ranges. -struct FoldingRange { - /// The zero-based start line of the range to fold. The folded area starts - /// after the line's last character. To be valid, the end must be zero or - /// larger and smaller than the number of lines in the document. - uint32_t startLine; - - /// The zero-based character offset from where the folded range starts. If - /// not defined, defaults to the length of the start line. - std::optional startCharacter; - - /// The zero-based end line of the range to fold. The folded area ends with - /// the line's last character. To be valid, the end must be zero or larger - /// and smaller than the number of lines in the document. - uint32_t endLine; - - /// The zero-based character offset before the folded range ends. If not - /// defined, defaults to the length of the end line. - std::optional endCharacter; - - /// Describes the kind of the folding range such as `comment` or `region`. - /// The kind is used to categorize folding ranges and used by commands like - /// 'Fold all comments'. See [FoldingRangeKind](#FoldingRangeKind) for an - /// enumeration of standardized kinds. - FoldingRangeKind kind; - - /// The text that the client should show when the specified range is - /// collapsed. If not defined or not supported by the client, a default - /// will be chosen by the client. - /// - /// @since 3.17.0 - proposed - std::optional collapsedText; -}; - -} // namespace proto - -LSPConverter::Result LSPConverter::convert(llvm::StringRef path, - llvm::ArrayRef foldings) { - auto file = co_await async::fs::read(path.str()); - if(!file) { - co_return json::Value(nullptr); - } - llvm::StringRef content = *file; - +std::vector + LSPConverter::transform(llvm::StringRef content, + llvm::ArrayRef foldings) { std::vector result; - PositionConverter converter(content, kind); + PositionConverter converter(content, encoding()); converter.toPositions(foldings, [](auto&& folding) { return folding.range; }); for(auto&& folding: foldings) { @@ -384,13 +341,34 @@ LSPConverter::Result LSPConverter::convert(llvm::StringRef path, .startLine = beginLine, .startCharacter = beginChar, .endLine = endLine, - .endCharacter = endChar, + /// FIXME: Figure out how to handle end character. + .endCharacter = endChar - 1, .kind = proto::FoldingRangeKind::Region, .collapsedText = folding.text, }); } - co_return json::serialize(result); + return result; +} + +LSPConverter::Result LSPConverter::convert(llvm::StringRef path, + llvm::ArrayRef tokens) { + auto file = co_await async::fs::read(path.str()); + if(!file) { + co_return json::Value(nullptr); + } + llvm::StringRef content = *file; + co_return json::serialize(transform(content, tokens)); +} + +LSPConverter::Result LSPConverter::convert(llvm::StringRef path, + llvm::ArrayRef foldings) { + auto file = co_await async::fs::read(path.str()); + if(!file) { + co_return json::Value(nullptr); + } + llvm::StringRef content = *file; + co_return json::serialize(transform(content, foldings)); } LSPConverter::Result LSPConverter::convert(const feature::Hover& hover) { diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index e32066c4..3a22b2f9 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -4,44 +4,57 @@ namespace clice { -async::Task<> Server::onInitialize(json::Value id, const proto::InitializeParams& params) { +async::Task<> Server::initialize(json::Value value) { + converter.initialize(std::move(value)); + proto::InitializeResult result = {}; + + /// Initialize the server info. result.serverInfo.name = "clice"; result.serverInfo.version = "0.0.1"; - /// Set `SemanticTokensOptions` - for(auto kind: SymbolKind::all()) { - std::string name{kind}; - name[0] = std::tolower(name[0]); - result.capabilities.semanticTokensProvider.legend.tokenTypes.emplace_back(std::move(name)); - } - - result.capabilities.semanticTokensProvider.legend.tokenModifiers = { - - }; - co_await response(std::move(id), json::serialize(result)); - - auto workplace = SourceConverter::toPath(params.workspaceFolders[0].uri); - config::init(workplace); - - for(auto& dir: config::server.compile_commands_dirs) { - llvm::SmallString<128> path = {dir}; - path::append(path, "compile_commands.json"); - database.updateCommands(path); - } -} - -async::Task<> Server::onInitialized(const proto::InitializedParams& params) { co_return; } -async::Task<> Server::onExit(const proto::None&) { - co_return; -} +// async::Task<> Server::onInitialize(json::Value id, const proto::InitializeParams& params) { +// proto::InitializeResult result = {}; +// result.serverInfo.name = "clice"; +// result.serverInfo.version = "0.0.1"; +// +// /// Set `SemanticTokensOptions` +// for(auto kind: SymbolKind::all()) { +// std::string name{kind}; +// name[0] = std::tolower(name[0]); +// result.capabilities.semanticTokensProvider.legend.tokenTypes.emplace_back(std::move(name)); +// } +// +// result.capabilities.semanticTokensProvider.legend.tokenModifiers = { +// +// }; +// +// co_await response(std::move(id), json::serialize(result)); +// +// auto workplace = SourceConverter::toPath(params.workspaceFolders[0].uri); +// config::init(workplace); +// +// for(auto& dir: config::server.compile_commands_dirs) { +// llvm::SmallString<128> path = {dir}; +// path::append(path, "compile_commands.json"); +// database.updateCommands(path); +// } +// } -async::Task<> Server::onShutdown(json::Value id, const proto::None&) { - co_return; -} +/// async::Task<> Server::onInitialized(const proto::InitializedParams& params) { +/// co_return; +/// } +/// +/// async::Task<> Server::onExit(const proto::None&) { +/// co_return; +/// } +/// +/// async::Task<> Server::onShutdown(json::Value id, const proto::None&) { +/// co_return; +/// } } // namespace clice diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp deleted file mode 100644 index 1e840dd5..00000000 --- a/src/Server/Scheduler.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "Server/Scheduler.h" - -namespace clice { - -async::Task<> Scheduler::open(llvm::StringRef file) { - /// 首先根据 rule 来对 file 进行判别 - - /// 查看一下这 file 属于什么模式 - - /// 1. 正常的源文件或者模块文件 - - /// 预处理一遍源文件，获取一些相关的信息 - /// 是不是 module file? - /// 获取模块依赖关系 - /// 或者计算 Preamble 的位置 - - auto command = database.getCommand(file); - - /// 如果不是 readonly 模式 - /// 调用 CacheController 里面对应的函数更新这个文件的 cache - /// PCH or PCM - - /// 准备 CompilationParams 进行 AST 编译 - - /// 调度索引任务 ... - - /// 2. 头文件 - - /// 先检查一下该头文件是否有选中的上下文... [一组 include 位置] - /// - /// 对上下文源文件进行上述的处理 - - /// - /// ... - - /// 重新索引依赖这个头文件的源文件 - - co_return; -} - -} // namespace clice diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index aa930a16..78aee909 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -3,78 +3,128 @@ namespace clice { -Server::Server() : indexer(database, config::index), scheduler(database, {}) { - addMethod("initialize", &Server::onInitialize); - addMethod("initialized", &Server::onInitialized); - addMethod("shutdown", &Server::onShutdown); - addMethod("exit", &Server::onExit); - - addMethod("textDocument/didOpen", &Server::onDidOpen); - addMethod("textDocument/didChange", &Server::onDidChange); - addMethod("textDocument/didSave", &Server::onDidSave); - addMethod("textDocument/didClose", &Server::onDidClose); - - // addMethod("textDocument/declaration", &Server::onGotoDeclaration); - // addMethod("textDocument/definition", &Server::onGotoDefinition); - // addMethod("textDocument/typeDefinition", &Server::onGotoTypeDefinition); - // addMethod("textDocument/implementation", &Server::onGotoImplementation); - // addMethod("textDocument/references", &Server::onFindReferences); - // addMethod("textDocument/callHierarchy/prepare", &Server::onPrepareCallHierarchy); - // addMethod("textDocument/callHierarchy/incomingCalls", &Server::onIncomingCall); - // addMethod("textDocument/callHierarchy/outgoingCalls", &Server::onOutgoingCall); - // addMethod("textDocument/typeHierarchy/prepare", &Server::onPrepareTypeHierarchy); - // addMethod("textDocument/typeHierarchy/supertypes", &Server::onSupertypes); - // addMethod("textDocument/typeHierarchy/subtypes", &Server::onSubtypes); - // addMethod("textDocument/documentHighlight", &Server::onDocumentHighlight); - // addMethod("textDocument/documentLink", &Server::onDocumentLink); - // addMethod("textDocument/hover", &Server::onHover); - // addMethod("textDocument/codeLens", &Server::onCodeLens); - // addMethod("textDocument/foldingRange", &Server::onFoldingRange); - // addMethod("textDocument/documentSymbol", &Server::onDocumentSymbol); - // addMethod("textDocument/semanticTokens/full", &Server::onSemanticTokens); - // addMethod("textDocument/inlayHint", &Server::onInlayHint); - // addMethod("textDocument/completion", &Server::onCodeCompletion); - // addMethod("textDocument/signatureHelp", &Server::onSignatureHelp); - // addMethod("textDocument/codeAction", &Server::onCodeAction); - // addMethod("textDocument/formatting", &Server::onFormatting); - // addMethod("textDocument/rangeFormatting", &Server::onRangeFormatting); - - addMethod("workspace/didChangeWatchedFiles", &Server::onDidChangeWatchedFiles); - - addMethod("index/current", &Server::onIndexCurrent); - addMethod("index/all", &Server::onIndexAll); - addMethod("context/current", &Server::onContextCurrent); - addMethod("context/switch", &Server::onContextSwitch); - addMethod("context/all", &Server::onContextAll); -} +Server::Server() : indexer(database, config::index) {} async::Task<> Server::onReceive(json::Value value) { - assert(value.kind() == json::Value::Object); auto object = value.getAsObject(); - assert(object && "value is not an object"); - if(auto method = object->get("method")) { - auto name = *method->getAsString(); - auto params = object->get("params"); - if(auto id = object->get("id")) { - if(auto iter = requests.find(name); iter != requests.end()) { - /// auto tracer = Tracer(); - log::info("Receive request: {0}", name); - co_await iter->second(std::move(*id), - params ? std::move(*params) : json::Value(nullptr)); - log::info("Request {0} is done, elapsed {1}", name, 0); + if(!object) [[unlikely]] { + log::fatal("Invalid LSP message, not an object: {}", value); + } - } else { - log::warn("Unknown request: {0}", name); - } - } else { - if(auto iter = notifications.find(name); iter != notifications.end()) { - log::info("Notification: {0}", name); - co_await iter->second(params ? std::move(*params) : json::Value(nullptr)); - } else { - log::warn("Unknown notification: {0}", name); - } + /// If the json object has an `id`, it's a request, + /// which needs a response. Otherwise, it's a notification. + auto id = object->get("id"); + + llvm::StringRef method; + if(auto result = object->getString("method")) { + method = *result; + } else [[unlikely]] { + log::warn("Invalid LSP message, method not found: {}", value); + if(id) { + co_await response(std::move(*id), + proto::ErrorCodes::InvalidRequest, + "Method not found"); + } + co_return; + } + + json::Value params = json::Value(nullptr); + if(auto result = object->get("params")) { + params = std::move(*result); + } + + /// Handle request and notification separately. + /// TODO: Record the time of handling request and notification. + if(id) { + log::info("Handling request: {}", method); + auto result = co_await onRequest(method, std::move(params)); + co_await response(std::move(*id), std::move(result)); + log::info("Handled request: {}", method); + } else { + log::info("Handling notification: {}", method); + co_await onNotification(method, std::move(params)); + log::info("Handled notification: {}", method); + } + + co_return; +} + +async::Task Server::onRequest(llvm::StringRef method, json::Value value) { + if(method == "initialize") { + auto result = converter.initialize(std::move(value)); + config::init(converter.workspace()); + + /// FIXME: Use a better way to handle compile commands. + for(auto&& dir: config::server.compile_commands_dirs) { + database.updateCommands(dir + "/compile_commands.json"); + } + + indexer.load(); + + co_return json::serialize(result); + } else if(method == "shutdown") { + indexer.save(); + } else if(method.consume_front("textDocument/")) { + co_return co_await onTextDocument(method, std::move(value)); + } else if(method.consume_front("context/")) { + co_return co_await onContext(method, std::move(value)); + } else if(method.consume_front("index/")) { + co_return co_await onIndex(method, std::move(value)); + } + + co_return json::Value(nullptr); +} + +async::Task Server::onTextDocument(llvm::StringRef method, json::Value value) { + if(method == "semanticTokens/full") { + auto params2 = json::deserialize(value); + auto path = SourceConverter::toPath(params2.textDocument.uri); + auto tokens = co_await indexer.semanticTokens(path); + co_return co_await converter.convert(path, tokens); + } else if(method == "foldingRange") { + auto params2 = json::deserialize(value); + auto path = SourceConverter::toPath(params2.textDocument.uri); + auto foldings = co_await indexer.foldingRanges(path); + co_return co_await converter.convert(path, foldings); + } + + co_return json::Value(nullptr); +} + +async::Task Server::onContext(llvm::StringRef method, json::Value value) { + if(method == "current") { + auto param2 = json::deserialize(value); + auto path = SourceConverter::toPath(param2.textDocument.uri); + auto result = indexer.currentContext(path); + co_return result ? json::serialize(*result) : json::Value(nullptr); + } else if(method == "switch") { + auto params = json::deserialize(value); + auto header = SourceConverter::toPath(params.header); + indexer.switchContext(header, params.context); + } else if(method == "all") { + auto param2 = json::deserialize(value); + auto path = SourceConverter::toPath(param2.textDocument.uri); + auto result = indexer.allContexts(path); + co_return json::serialize(result); + } else if(method == "resolve") { + co_return json::serialize( + indexer.resolveContext(json::deserialize(value))); + } + + co_return json::Value(nullptr); +} + +async::Task Server::onIndex(llvm::StringRef method, json::Value value) { + co_return json::Value(nullptr); +} + +async::Task<> Server::onNotification(llvm::StringRef method, json::Value value) { + if(method.consume_front("index/")) { + if(method == "all") { + indexer.indexAll(); } } + co_return; } @@ -98,11 +148,24 @@ async::Task<> Server::notify(llvm::StringRef method, json::Value params) { async::Task<> Server::response(json::Value id, json::Value result) { co_await async::net::write(json::Object{ {"jsonrpc", "2.0" }, - {"id", id }, + {"id", std::move(id) }, {"result", std::move(result)}, }); } +async::Task<> Server::response(json::Value id, proto::ErrorCodes code, llvm::StringRef message) { + json::Object error{ + {"code", static_cast(code)}, + {"message", message }, + }; + + co_await async::net::write(json::Object{ + {"jsonrpc", "2.0" }, + {"id", std::move(id) }, + {"error", std::move(error)}, + }); +} + async::Task<> Server::registerCapacity(llvm::StringRef id, llvm::StringRef method, json::Value registerOptions) { diff --git a/src/Server/Workspace.cpp b/src/Server/Workspace.cpp deleted file mode 100644 index d5fca396..00000000 --- a/src/Server/Workspace.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "Basic/SourceConverter.h" -#include "Server/Server.h" - -namespace clice { - -async::Task<> Server::onDidChangeWatchedFiles(const proto::DidChangeWatchedFilesParams& params) { - co_return; -} - -} // namespace clice diff --git a/unittests/Compiler/Command.cpp b/unittests/Compiler/Command.cpp index 992a349f..160be781 100644 --- a/unittests/Compiler/Command.cpp +++ b/unittests/Compiler/Command.cpp @@ -5,31 +5,9 @@ namespace clice::testing { namespace { -TEST(clice, Command) { - llvm::SmallString<1024> buffer; - llvm::SmallVector args; +TEST(CompilationDatabase, Command) {} - auto command = - "/usr/local/bin/clang++" - " -Dclice_core_EXPORTS" - " -I/home/ykiko/C++/clice2/deps/llvm/build-install/include" - " -I/home/ykiko/C++/clice2/include" - " -I/home/ykiko/C++/clice2/deps/toml/include" - " -I/home/ykiko/C++/clice2/deps/libuv/include" - " -fno-rtti" - " -fno-exceptions" - " -g -O0 -fsanitize=address" - " -Wno-deprecated-declarations" - " -g -std=gnu++23 -fPIC -Winvalid-pch" - " -Xclang -include-pch" - " -Xclang /home/ykiko/C++/clice2/build/CMakeFiles/clice-core.dir/cmake_pch.hxx.pch" - " -Xclang -include -Xclang /home/ykiko/C++/clice2/build/CMakeFiles/clice-core.dir/cmake_pch.hxx" - " -o CMakeFiles/clice-core.dir/src/Basic/URI.cpp.o" - " -c /home/ykiko/C++/clice2/src/Basic/URI.cpp"; - - auto result = mangleCommand(command, args, buffer); - ASSERT_FALSE(bool(!result)); -} +TEST(CompilationDatabase, Module) {} } // namespace diff --git a/unittests/Server/Database.cpp b/unittests/Server/Database.cpp deleted file mode 100644 index 727daa07..00000000 --- a/unittests/Server/Database.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "Test/Test.h" -#include "Server/Database.h" - -namespace clice::testing { - -namespace { - -TEST(CompilationDatabase, Command) {} - -TEST(CompilationDatabase, Module) {} - -} // namespace - -} // namespace clice::testing diff --git a/unittests/Server/LSPConverter.cpp b/unittests/Server/LSPConverter.cpp new file mode 100644 index 00000000..0fd0b86c --- /dev/null +++ b/unittests/Server/LSPConverter.cpp @@ -0,0 +1,12 @@ +#include +#include + +namespace clice::testing { + +namespace { + +TEST(Server, SourceConverter) {} + +} // namespace + +} // namespace clice::testing