diff --git a/include/Index/IncludeGraph.h b/include/Index/IncludeGraph.h index 32233ded..8e5aee3d 100644 --- a/include/Index/IncludeGraph.h +++ b/include/Index/IncludeGraph.h @@ -37,7 +37,7 @@ struct IncludeGraph { static IncludeGraph from(CompilationUnit& unit); - std::string getPath(std::uint32_t path_ref) const { + llvm::StringRef path(std::uint32_t path_ref) const { assert(path_ref < paths.size()); return paths[path_ref]; } diff --git a/include/Index/Index.h b/include/Index/Index.h deleted file mode 100644 index e69de29b..00000000 diff --git a/include/Index/MergedIndex.h b/include/Index/MergedIndex.h index 85fe8fd2..78f36de5 100644 --- a/include/Index/MergedIndex.h +++ b/include/Index/MergedIndex.h @@ -93,12 +93,17 @@ namespace clice::index { struct HeaderContexts { std::uint32_t version = 0; - using Context = std::pair; + struct Context { + std::uint32_t include; + std::uint32_t canonical_id; + + friend bool operator== (const Context&, const Context&) = default; + }; /// A array of include location and its context id. llvm::SmallVector includes; - friend bool operator== (const HeaderContexts& lhs, const HeaderContexts& rhs) = default; + friend bool operator== (const HeaderContexts&, const HeaderContexts&) = default; }; struct MergedIndex { @@ -125,13 +130,21 @@ struct MergedIndex { /// All merged symbol relations. llvm::DenseMap> relations; + /// FIXME: The content of this file. + /// std::string content; + + /// Sorted occurrences cache for fast lookup. + std::vector cache_occurrences; + void remove(llvm::StringRef path); void merge(llvm::StringRef path, std::uint32_t include, FileIndex& index); + std::vector lookup(std::uint32_t offset); + void serialize(this MergedIndex& self, llvm::raw_ostream& out); - friend bool operator== (const MergedIndex& lhs, const MergedIndex& rhs) = default; + friend bool operator== (const MergedIndex&, const MergedIndex&) = default; }; struct MergedIndexView { diff --git a/include/Index/ProjectIndex.h b/include/Index/ProjectIndex.h new file mode 100644 index 00000000..a8fcbf65 --- /dev/null +++ b/include/Index/ProjectIndex.h @@ -0,0 +1,57 @@ +#pragma once + +#include "TUIndex.h" + +namespace clice::index { + +struct PathPool { + llvm::BumpPtrAllocator allocator; + + std::vector paths; + + llvm::DenseMap cache; + + llvm::StringRef save(llvm::StringRef s) { + auto data = allocator.Allocate(s.size() + 1); + std::ranges::copy(s, data); + data[s.size()] = '\0'; + return llvm::StringRef(data, s.size()); + } + + auto path_id(llvm::StringRef path) { + assert(!path.empty()); + auto [it, success] = cache.try_emplace(path, paths.size()); + if(!success) { + return it->second; + } + + auto& [k, v] = *it; + k = save(path); + paths.emplace_back(k); + return it->second; + } + + llvm::StringRef path(std::uint32_t id) { + return paths[id]; + } +}; + +struct FileInfo { + std::int64_t mtime; +}; + +struct ProjectIndex { + PathPool path_pool; + + llvm::DenseMap indices; + + SymbolTable symbols; + + void merge(this ProjectIndex& self, TUIndex& index); + + void serialize(this ProjectIndex& self, llvm::raw_ostream& os); + + static ProjectIndex from(const void* data); +}; + +} // namespace clice::index diff --git a/include/Index/TUIndex.h b/include/Index/TUIndex.h index a42c78e9..15af6b2c 100644 --- a/include/Index/TUIndex.h +++ b/include/Index/TUIndex.h @@ -20,11 +20,11 @@ struct Relation { SymbolHash target_symbol; - void set_definition_range(LocalSourceRange range) { + constexpr void set_definition_range(LocalSourceRange range) { target_symbol = std::bit_cast(range); } - auto definition_range() { + constexpr auto definition_range() { return std::bit_cast(target_symbol); } }; @@ -35,6 +35,8 @@ struct Occurrence { /// SymbolHash target; + + friend bool operator== (const Occurrence&, const Occurrence&) = default; }; struct FileIndex { @@ -50,12 +52,16 @@ struct Symbol { /// All files that referenced this symbol. Bitmap reference_files; + + friend bool operator== (const Symbol&, const Symbol&) = default; }; +using SymbolTable = llvm::DenseMap; + struct TUIndex { IncludeGraph graph; - llvm::DenseMap symbols; + SymbolTable symbols; llvm::DenseMap file_indices; diff --git a/include/Index/schema.fbs b/include/Index/schema.fbs index 1542a9e6..1ae268d8 100644 --- a/include/Index/schema.fbs +++ b/include/Index/schema.fbs @@ -52,6 +52,16 @@ table SymbolRelationsEntry { relations: [RelationEntry]; } +table Symbol { + kind: ubyte; + refs: [ubyte]; +} + +table SymbolEntry { + symbol_id: ulong; + symbol: Symbol; +} + table MergedIndex { max_canonical_id: uint; @@ -63,3 +73,19 @@ table MergedIndex { relations: [SymbolRelationsEntry]; } + +table PathEntry { + path: string; + id: uint; +} + +struct PathMapEntry { + source: uint; + index: uint; +} + +table ProjectIndex { + paths: [PathEntry]; + indices: [PathMapEntry]; + symbols: [SymbolEntry]; +} diff --git a/include/Protocol/Feature/Declaration.h b/include/Protocol/Feature/Declaration.h index cddf8c26..31b95cf5 100644 --- a/include/Protocol/Feature/Declaration.h +++ b/include/Protocol/Feature/Declaration.h @@ -8,4 +8,6 @@ struct DeclarationClientCapabilities {}; using DeclarationOptions = WorkDoneProgressOptions; +using DeclarationParams = TextDocumentPositionParams; + } // namespace clice::proto diff --git a/include/Protocol/Feature/Definition.h b/include/Protocol/Feature/Definition.h index 4a96b814..86d97366 100644 --- a/include/Protocol/Feature/Definition.h +++ b/include/Protocol/Feature/Definition.h @@ -8,4 +8,6 @@ struct DefinitionClientCapabilities {}; using DefinitionOptions = WorkDoneProgressOptions; +using DefinitionParams = TextDocumentPositionParams; + } // namespace clice::proto diff --git a/include/Protocol/Feature/Reference.h b/include/Protocol/Feature/Reference.h index 0f2895b8..4f19e8e8 100644 --- a/include/Protocol/Feature/Reference.h +++ b/include/Protocol/Feature/Reference.h @@ -8,4 +8,6 @@ struct ReferenceClientCapabilities {}; using ReferenceOptions = WorkDoneProgressOptions; +using ReferenceParams = TextDocumentPositionParams; + } // namespace clice::proto diff --git a/include/Protocol/Lifecycle.h b/include/Protocol/Lifecycle.h index 65df13cf..5acbe805 100644 --- a/include/Protocol/Lifecycle.h +++ b/include/Protocol/Lifecycle.h @@ -95,10 +95,10 @@ struct ServerCapabilities { SignatureHelpOptions signatureHelpProvider; /// The server provides go to declaration support. - /// FIXME: DeclarationOptions declarationProvider; + DeclarationOptions declarationProvider; /// The server provides goto definition support. - /// FIXME: DefinitionOptions definitionProvider; + DefinitionOptions definitionProvider; /// The server provides goto type definition support. /// FIXME: TypeDefinitionOptions typeDefinitionProvider; @@ -107,7 +107,7 @@ struct ServerCapabilities { /// FIXME: ImplementationOptions implementationProvider; /// The server provides find references support. - /// FIXME: ReferenceOptions referencesProvider; + ReferenceOptions referencesProvider; /// The server provides document highlight support. /// FIXME: DocumentHighlightOptions documentHighlightProvider; diff --git a/include/Server/Convert.h b/include/Server/Convert.h index 19fd22c6..c4eabd84 100644 --- a/include/Server/Convert.h +++ b/include/Server/Convert.h @@ -155,8 +155,8 @@ public: return position; } - template - void to_positions(Range&& range, Proj&& proj) { + template + void to_positions(Range&& range, const Proj&& proj = {}) { std::vector offsets; for(auto&& item: range) { auto [begin, end] = proj(item); diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h index 82eefb6c..d463f9d0 100644 --- a/include/Server/Indexer.h +++ b/include/Server/Indexer.h @@ -1,18 +1,62 @@ #pragma once +#include #include + #include "Async/Async.h" -#include "AST/SymbolID.h" +#include "Compiler/Command.h" +#include "Index/MergedIndex.h" +#include "Index/ProjectIndex.h" +#include "Protocol/Protocol.h" + #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringMap.h" -#include "Compiler/Command.h" -#include "Index/Index.h" namespace clice { class CompilationUnit; -class Indexer {}; +class Indexer { +public: + Indexer(CompilationDatabase& database) : database(database) {} + + async::Task<> index(llvm::StringRef path); + + async::Task<> index(llvm::StringRef path, llvm::StringRef content); + + async::Task<> schedule_next(); + + async::Task<> index_all(); + + using Result = async::Task>; + + auto lookup(llvm::StringRef path, std::uint32_t offset, RelationKind kind) -> Result; + + auto declaration(llvm::StringRef path, std::uint32_t offset) -> Result; + + auto definition(llvm::StringRef path, std::uint32_t offset) -> Result; + + auto references(llvm::StringRef path, std::uint32_t offset) -> Result; + + /// TODO: Calls ... + + /// TODO: Types ... + +private: + CompilationDatabase& database; + + index::ProjectIndex project_index; + + llvm::DenseMap in_memory_indices; + + /// Currently indexes tasks ... + std::vector> workings; + + /// FIXME: Use a LRU to make sure we won't index a file twice ... + std::deque waitings; + + async::Event update_event; +}; } // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index 1d9c6e59..608a51e0 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -202,6 +202,12 @@ private: auto on_signature_help(proto::SignatureHelpParams params) -> Result; + auto on_go_to_declaration(proto::DeclarationParams params) -> Result; + + auto on_go_to_definition(proto::DefinitionParams params) -> Result; + + auto on_find_references(proto::ReferenceParams params) -> Result; + auto on_document_symbol(proto::DocumentSymbolParams params) -> Result; auto on_document_link(proto::DocumentLinkParams params) -> Result; @@ -237,6 +243,8 @@ private: PathMapping mapping; config::Config config; + + Indexer indexer; }; } // namespace clice diff --git a/src/AST/Selection.cpp b/src/AST/Selection.cpp index ed1e6c0e..21dd7f2e 100644 --- a/src/AST/Selection.cpp +++ b/src/AST/Selection.cpp @@ -24,6 +24,12 @@ namespace clice { +#ifdef NDEBUG +#define LOGGING_DEBUG(...) +#else +#define LOGGING_DEBUG(...) logging::debug(__VA_ARGS__) +#endif + namespace { using Node = SelectionTree::Node; @@ -948,10 +954,10 @@ private: } if(!checker.may_hit(S)) { - logging::debug("{2}skip: {0} {1}", - print_node_to_string(N, print_policy), - S.printToString(SM), - indent()); + LOGGING_DEBUG("{2}skip: {0} {1}", + print_node_to_string(N, print_policy), + S.printToString(SM), + indent()); return true; } @@ -971,10 +977,10 @@ private: // Performs early hit detection for some nodes (on the earlySourceRange). void push(clang::DynTypedNode node) { clang::SourceRange Early = early_source_range(node); - logging::debug("{2}push: {0} {1}", - print_node_to_string(node, print_policy), - node.getSourceRange().printToString(SM), - indent()); + LOGGING_DEBUG("{2}push: {0} {1}", + print_node_to_string(node, print_policy), + node.getSourceRange().printToString(SM), + indent()); nodes.emplace_back(); nodes.back().data = std::move(node); nodes.back().parent = stack.top(); @@ -987,7 +993,7 @@ private: // Performs primary hit detection. void pop() { Node& N = *stack.top(); - logging::debug("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1)); + LOGGING_DEBUG("{1}pop: {0}", print_node_to_string(N.data, print_policy), indent(-1)); claim_tokens_for(N.data, N.selected); if(N.selected == no_tokens) { N.selected = SelectionTree::Unselected; @@ -1118,7 +1124,7 @@ private: } if(result && result != no_tokens) { - logging::debug("{1}hit selection: {0}", S.printToString(SM), indent()); + LOGGING_DEBUG("{1}hit selection: {0}", S.printToString(SM), indent()); } } @@ -1243,9 +1249,9 @@ SelectionTree::SelectionTree(CompilationUnit& unit, LocalSourceRange range) : print_policy.IncludeNewlines = false; auto [begin, end] = range; - logging::debug("Computing selection for {0}", - clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end)) - .printToString(SM)); + LOGGING_DEBUG("Computing selection for {0}", + clang::SourceRange(SM.getComposedLoc(fid, begin), SM.getComposedLoc(fid, end)) + .printToString(SM)); nodes = SelectionVisitor::collect(unit, print_policy, range, fid); m_root = nodes.empty() ? nullptr : &nodes.front(); diff --git a/src/Async/Async.cpp b/src/Async/Async.cpp index 0a347809..f8f9f307 100644 --- a/src/Async/Async.cpp +++ b/src/Async/Async.cpp @@ -53,7 +53,8 @@ void run() { init(); } - uv_check_result(uv_os_setenv("UV_THREADPOOL_SIZE", "20")); + auto pool_size = std::max(std::thread::hardware_concurrency(), 4u); + uv_check_result(uv_os_setenv("UV_THREADPOOL_SIZE", std::to_string(pool_size).c_str())); uv_check_result(uv_run(loop, UV_RUN_DEFAULT)); diff --git a/src/Compiler/Command.cpp b/src/Compiler/Command.cpp index 3d0d6929..bdbdaf82 100644 --- a/src/Compiler/Command.cpp +++ b/src/Compiler/Command.cpp @@ -277,12 +277,12 @@ auto CompilationDatabase::query_driver(this Self& self, llvm::StringRef driver) {output_path.str()}, }; +#ifdef _WIN32 /// If the env is `std::nullopt`, `ExecuteAndWait` will inherit env from parent process, /// which is very important for msvc and clang on windows. Thay depend on the environment /// variables to find correct standard library path. constexpr auto env = std::nullopt; -#ifdef _WIN32 llvm::SmallVector argv; if(driver_name.ends_with("cl") || driver_name.starts_with("clang-cl")) { /// FIXME: MSVC command:` cl /Bv`, should we support it? @@ -293,8 +293,10 @@ auto CompilationDatabase::query_driver(this Self& self, llvm::StringRef driver) } #else /// FIXME: We should find a better way to convert "LANG=C", this is important - /// for gcc with locality. Otherwise, it will output non-ASCII char. - llvm::SmallVector argv = {"LANG=C", driver, "-E", "-v", "-xc++", "/dev/null"}; + /// for gcc with locality. Otherwise, it will output non-ASCII char. We also + /// want to inherit the environment variables like windows. + llvm::SmallVector env = {"LANG=C"}; + llvm::SmallVector argv = {driver, "-E", "-v", "-xc++", "/dev/null"}; #endif std::string message; diff --git a/src/Index/MergedIndex.cpp b/src/Index/MergedIndex.cpp index aed13727..ad303881 100644 --- a/src/Index/MergedIndex.cpp +++ b/src/Index/MergedIndex.cpp @@ -1,3 +1,4 @@ +#include "Support/Compare.h" #include "schema_generated.h" #include "Index/MergedIndex.h" #include "llvm/Support/SHA256.h" @@ -84,119 +85,32 @@ void MergedIndex::merge(llvm::StringRef path, std::uint32_t include, FileIndex& max_canonical_id += 1; } -void MergedIndex::serialize(this MergedIndex& self, llvm::raw_ostream& out) { - namespace fbs = flatbuffers; - fbs::FlatBufferBuilder builder(1024); - - std::vector> canonical_cache; - canonical_cache.reserve(self.canonical_cache.size()); - for(auto& [hash, canonical_id]: self.canonical_cache) { - canonical_cache.emplace_back( - binary::CreateCacheEntry(builder, - builder.CreateString(hash.data(), hash.size()), - canonical_id)); - }; - - std::vector> header_contexts; - header_contexts.reserve(self.contexts.size()); - for(auto& [path, contexts]: self.contexts) { - header_contexts.emplace_back(binary::CreateHeaderContextsEntry( - builder, - builder.CreateString(path.data(), path.size()), - binary::CreateHeaderContexts( - builder, - contexts.version, - builder.CreateVectorOfStructs( - reinterpret_cast(contexts.includes.data()), - contexts.includes.size())))); - }; - - llvm::SmallVector buffer; - - std::vector> occurrences; - occurrences.reserve(self.occurrences.size()); - for(auto& [occurrence, bitmap]: self.occurrences) { - buffer.resize_for_overwrite(bitmap.getSizeInBytes(false)); - bitmap.write(buffer.data(), false); - occurrences.emplace_back(binary::CreateOccurrenceEntry( - builder, - reinterpret_cast(&occurrence), - builder.CreateVector(reinterpret_cast(buffer.data()), buffer.size()))); - buffer.clear(); +std::vector MergedIndex::lookup(std::uint32_t offset) { + if(cache_occurrences.size() != occurrences.size()) { + cache_occurrences.clear(); + for(auto& [occurrence, _]: occurrences) { + cache_occurrences.emplace_back(occurrence); + } + std::ranges::sort(cache_occurrences, refl::less); } - std::vector> relations; - relations.reserve(self.relations.size()); - for(auto& [symbold_id, symbol_relations]: self.relations) { - std::vector> entries; - entries.reserve(symbol_relations.size()); - for(auto& [relation, bitmap]: symbol_relations) { - buffer.resize_for_overwrite(bitmap.getSizeInBytes(false)); - bitmap.write(buffer.data(), false); - entries.emplace_back(binary::CreateRelationEntry( - builder, - reinterpret_cast(&relation), - builder.CreateVector(reinterpret_cast(buffer.data()), - buffer.size()))); - buffer.clear(); + auto it = + std::ranges::lower_bound(cache_occurrences, offset, {}, [](index::Occurrence& occurrence) { + return occurrence.range.end; + }); + + std::vector occurrences; + while(it != cache_occurrences.end()) { + if(it->range.contains(offset)) { + occurrences.emplace_back(*it); + it++; + continue; } - relations.emplace_back( - binary::CreateSymbolRelationsEntryDirect(builder, symbold_id, &entries)); + break; } - auto merged_index = binary::CreateMergedIndexDirect(builder, - self.max_canonical_id, - &canonical_cache, - &header_contexts, - &occurrences, - &relations); - builder.Finish(merged_index); - - out.write(reinterpret_cast(builder.GetBufferPointer()), builder.GetSize()); -} - -MergedIndex MergedIndexView::deserialize() { - namespace fbs = flatbuffers; - auto root = fbs::GetRoot(data); - - MergedIndex index; - index.max_canonical_id = root->max_canonical_id(); - - for(auto entry: *root->canonical_cache()) { - index.canonical_cache.try_emplace(entry->sha256()->string_view(), entry->canonical_id()); - } - - index.canonical_ref_counts.resize(index.max_canonical_id, 0); - - HeaderContexts contexts; - for(auto entry: *root->contexts()) { - auto path = entry->path()->string_view(); - contexts.version = entry->contexts()->version(); - for(auto include: *entry->contexts()->includes()) { - index.canonical_ref_counts[include->canonical_id()] += 1; - contexts.includes.emplace_back(include->include_(), include->canonical_id()); - } - index.contexts.try_emplace(path, std::move(contexts)); - } - - for(auto entry: *root->occurrences()) { - index.occurrences.try_emplace( - *reinterpret_cast(entry->occurrence()), - Bitmap::read(reinterpret_cast(entry->context()->data()), false)); - } - - for(auto entry: *root->relations()) { - auto& relations = index.relations[entry->symbol()]; - for(auto relation_entry: *entry->relations()) { - relations.try_emplace( - *reinterpret_cast(relation_entry->relation()), - Bitmap::read(reinterpret_cast(relation_entry->context()->data()), - false)); - } - } - - return index; + return occurrences; } } // namespace clice::index diff --git a/src/Index/ProjectIndex.cpp b/src/Index/ProjectIndex.cpp new file mode 100644 index 00000000..1f3cab05 --- /dev/null +++ b/src/Index/ProjectIndex.cpp @@ -0,0 +1,24 @@ +#include "schema_generated.h" +#include "Index/ProjectIndex.h" +#include "Support/Ranges.h" + +namespace clice::index { + +void ProjectIndex::merge(this ProjectIndex& self, TUIndex& index) { + auto& paths = index.graph.paths; + llvm::SmallVector file_ids_map; + file_ids_map.resize_for_overwrite(paths.size()); + + for(auto i = 0; i < paths.size(); i++) { + file_ids_map[i] = self.path_pool.path_id(paths[i]); + } + + for(auto& [symbol_id, symbol]: index.symbols) { + auto& target_symbol = self.symbols[symbol_id]; + for(auto ref: symbol.reference_files) { + target_symbol.reference_files.add(file_ids_map[ref]); + } + } +} + +} // namespace clice::index diff --git a/src/Index/Serialization.cpp b/src/Index/Serialization.cpp new file mode 100644 index 00000000..fd880968 --- /dev/null +++ b/src/Index/Serialization.cpp @@ -0,0 +1,227 @@ +#include "schema_generated.h" +#include "Index/MergedIndex.h" +#include "Index/ProjectIndex.h" +#include "Support/Ranges.h" + +namespace clice::index { + +namespace fbs = flatbuffers; + +namespace { + +template +using Offsets = llvm::SmallVector, 0>; + +template +const U* safe_cast(const V* v) { + static_assert(sizeof(U) == sizeof(V)); + assert((void(std::bit_cast(V{})), true)); + return reinterpret_cast(v); +} + +auto CreateString(fbs::FlatBufferBuilder& builder, llvm::StringRef string) { + return builder.CreateString(string.data(), string.size()); +} + +template +auto CreateVector(fbs::FlatBufferBuilder& builder, const Range& range) { + return builder.CreateVector(range.data(), range.size()); +} + +auto CreateVector(fbs::FlatBufferBuilder& builder, const llvm::SmallVector& range) { + return builder.CreateVector(reinterpret_cast(range.data()), range.size()); +} + +template +auto CreateStructVector(fbs::FlatBufferBuilder& builder, const Range& range) { + using V = ranges::range_value_t; + return builder.CreateVectorOfStructs(safe_cast(range.data()), range.size()); +} + +template +auto transform(const Range& range, const Functor& functor) { + using V = ranges::range_value_t; + using R = std::invoke_result_t; + + llvm::SmallVector result; + result.resize_for_overwrite(ranges::size(range)); + + auto i = 0; + for(auto&& v: range) { + result[i] = functor(v); + i += 1; + } + return result; +} + +Bitmap read_bitmap(const fbs::Vector* buffer) { + return Bitmap::read(reinterpret_cast(buffer->data()), false); +} + +} // namespace + +void MergedIndex::serialize(this MergedIndex& self, llvm::raw_ostream& out) { + fbs::FlatBufferBuilder builder(1024); + + llvm::SmallVector buffer; + + auto canonical_cache = transform(self.canonical_cache, [&](auto&& value) { + auto&& [hash, canonical_id] = value; + return binary::CreateCacheEntry(builder, CreateString(builder, hash), canonical_id); + }); + + auto header_contexts = transform(self.contexts, [&](auto&& value) { + auto& [path, contexts] = value; + return binary::CreateHeaderContextsEntry( + builder, + CreateString(builder, path), + binary::CreateHeaderContexts( + builder, + contexts.version, + CreateStructVector(builder, contexts.includes))); + }); + + auto occurrences = transform(self.occurrences, [&](auto&& value) { + auto&& [occurrence, bitmap] = value; + buffer.clear(); + buffer.resize_for_overwrite(bitmap.getSizeInBytes(false)); + bitmap.write(buffer.data(), false); + return binary::CreateOccurrenceEntry(builder, + safe_cast(&occurrence), + CreateVector(builder, buffer)); + }); + + auto relations = transform(self.relations, [&](auto&& value) { + auto&& [symbold_id, symbol_relations] = value; + auto relations = transform(symbol_relations, [&](auto&& value) { + auto&& [relation, bitmap] = value; + buffer.clear(); + buffer.resize_for_overwrite(bitmap.getSizeInBytes(false)); + bitmap.write(buffer.data(), false); + return binary::CreateRelationEntry(builder, + safe_cast(&relation), + CreateVector(builder, buffer)); + }); + return binary::CreateSymbolRelationsEntry(builder, + symbold_id, + CreateVector(builder, relations)); + }); + + auto merged_index = binary::CreateMergedIndex(builder, + self.max_canonical_id, + CreateVector(builder, canonical_cache), + CreateVector(builder, header_contexts), + CreateVector(builder, occurrences), + CreateVector(builder, relations)); + builder.Finish(merged_index); + + out.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); +} + +MergedIndex MergedIndexView::deserialize() { + auto root = fbs::GetRoot(data); + + MergedIndex index; + index.max_canonical_id = root->max_canonical_id(); + + for(auto entry: *root->canonical_cache()) { + index.canonical_cache.try_emplace(entry->sha256()->string_view(), entry->canonical_id()); + } + + index.canonical_ref_counts.resize(index.max_canonical_id, 0); + + HeaderContexts contexts; + for(auto entry: *root->contexts()) { + auto path = entry->path()->string_view(); + contexts.version = entry->contexts()->version(); + for(auto include: *entry->contexts()->includes()) { + index.canonical_ref_counts[include->canonical_id()] += 1; + contexts.includes.emplace_back(include->include_(), include->canonical_id()); + } + index.contexts.try_emplace(path, std::move(contexts)); + } + + for(auto entry: *root->occurrences()) { + index.occurrences.try_emplace(*safe_cast(entry->occurrence()), + read_bitmap(entry->context())); + } + + for(auto entry: *root->relations()) { + auto& relations = index.relations[entry->symbol()]; + for(auto relation_entry: *entry->relations()) { + relations.try_emplace(*safe_cast(relation_entry->relation()), + read_bitmap(relation_entry->context())); + } + } + + return index; +} + +void ProjectIndex::serialize(this ProjectIndex& self, llvm::raw_ostream& os) { + fbs::FlatBufferBuilder builder(1024); + + llvm::SmallVector buffer; + + auto i = 0; + auto paths = transform(self.path_pool.paths, [&](llvm::StringRef path) { + auto enrty = + binary::CreatePathEntry(builder, CreateString(builder, self.path_pool.paths[i]), i); + i += 1; + return enrty; + }); + + auto indices = transform(self.indices, [&](auto&& value) { + auto&& [source, index] = value; + return binary::PathMapEntry(source, index); + }); + + auto symbols = transform(self.symbols, [&](auto&& value) { + auto& [symbol_id, symbol] = value; + + buffer.clear(); + buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false)); + symbol.reference_files.write(buffer.data(), false); + + return binary::CreateSymbolEntry( + builder, + symbol_id, + binary::CreateSymbol(builder, symbol.kind.value(), CreateVector(builder, buffer))); + }); + + auto project_index = + binary::CreateProjectIndex(builder, + CreateVector(builder, paths), + CreateStructVector(builder, indices), + CreateVector(builder, symbols)); + + builder.Finish(project_index); + os.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); +} + +ProjectIndex ProjectIndex::from(const void* data) { + auto root = fbs::GetRoot(data); + + ProjectIndex index; + + auto& pool = index.path_pool; + pool.paths.resize(root->paths()->size()); + for(auto entry: *root->paths()) { + auto k = pool.save(entry->path()->string_view()); + pool.paths[entry->id()] = k; + pool.cache.try_emplace(k, entry->id()); + } + + for(auto entry: *root->indices()) { + index.indices.try_emplace(entry->source(), entry->index()); + } + + for(auto entry: *root->symbols()) { + auto& symbol = index.symbols[entry->symbol_id()]; + symbol.kind = SymbolKind(entry->symbol()->kind()); + symbol.reference_files = read_bitmap(entry->symbol()->refs()); + } + + return index; +} + +} // namespace clice::index diff --git a/src/Server/Feature.cpp b/src/Server/Feature.cpp index c593240c..c0f0ee02 100644 --- a/src/Server/Feature.cpp +++ b/src/Server/Feature.cpp @@ -100,6 +100,27 @@ async::Task Server::on_signature_help(proto::SignatureHelpParams pa } } +auto Server::on_go_to_declaration(proto::DeclarationParams params) -> Result { + auto path = mapping.to_path(params.textDocument.uri); + auto opening_file = opening_files.get_or_add(path); + auto offset = to_offset(kind, opening_file->content, params.position); + co_return json::serialize(co_await indexer.declaration(path, offset)); +} + +auto Server::on_go_to_definition(proto::DefinitionParams params) -> Result { + auto path = mapping.to_path(params.textDocument.uri); + auto opening_file = opening_files.get_or_add(path); + auto offset = to_offset(kind, opening_file->content, params.position); + co_return json::serialize(co_await indexer.definition(path, offset)); +} + +auto Server::on_find_references(proto::ReferenceParams params) -> Result { + auto path = mapping.to_path(params.textDocument.uri); + auto opening_file = opening_files.get_or_add(path); + auto offset = to_offset(kind, opening_file->content, params.position); + co_return json::serialize(co_await indexer.references(path, offset)); +} + auto Server::on_document_symbol(proto::DocumentSymbolParams params) -> Result { auto path = mapping.to_path(params.textDocument.uri); auto opening_file = opening_files.get_or_add(path); diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index 00fb6331..51d844bc 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -1,7 +1,154 @@ -#include "Compiler/CompilationUnit.h" + #include "Compiler/Compilation.h" -#include "Index/Index.h" #include "Server/Indexer.h" +#include "Server/Convert.h" +#include "Support/Compare.h" #include "Support/Logging.h" -namespace clice {} // namespace clice +namespace clice { + +async::Task<> Indexer::index(llvm::StringRef path) { + CompilationParams params; + params.kind = CompilationUnit::Indexing; + params.arguments = database.get_command(path).arguments; + + /// FIXME: We may want to stop the task in the future. + /// params.stop; + + /// Check update? + + auto tu_index = co_await async::submit([&]() -> std::optional { + auto unit = compile(params); + if(!unit) { + logging::info("Fail to index for {}, because: {}", path, unit.error()); + return std::nullopt; + } + + return index::TUIndex::build(*unit); + }); + + if(!tu_index) { + co_return; + } + + project_index.merge(*tu_index); + + /// FIXME: Currently, we merge index eagerly, I would like to improve + /// this in the future. + for(auto& [fid, index]: tu_index->file_indices) { + auto path = tu_index->graph.path(tu_index->graph.path_id(fid)); + auto& merged_index = in_memory_indices[project_index.path_pool.path_id(path)]; + + merged_index.merge(path, tu_index->graph.include_location_id(fid), index); + } + + logging::info("Successfully index {}", path); +} + +async::Task<> Indexer::schedule_next() { + while(true) { + while(waitings.empty()) { + co_await update_event; + } + + auto file_id = waitings.front(); + waitings.pop_front(); + + auto file = project_index.path_pool.path(file_id); + + auto i = 0; + for(; i < workings.size(); i++) { + if(workings[i].empty()) { + workings[i] = index(file); + break; + } + } + + co_await workings[i]; + workings[i].release().destroy(); + } +} + +async::Task<> Indexer::index_all() { + for(auto& [file, cmd]: database) { + waitings.push_back(project_index.path_pool.path_id(file)); + } + + auto max_count = std::max(std::thread::hardware_concurrency(), 4u); + + /// FIXME: Currently, we just reserve two thread for other kind of tasks, + /// there may be a better way to handle this in the future ... + workings.resize(max_count - 2); + + for(auto i = 0; i < max_count - 2; i++) { + auto task = schedule_next(); + task.schedule(); + task.dispose(); + } + + co_return; +} + +auto Indexer::lookup(llvm::StringRef path, std::uint32_t offset, RelationKind kind) -> Result { + std::vector locations; + + auto path_id = project_index.path_pool.path_id(path); + auto index = in_memory_indices[path_id]; + auto occurrences = index.lookup(offset); + if(occurrences.empty()) { + co_return locations; + } + + /// FIXME: We only handle first element now ... + auto symbol_id = occurrences.front().target; + auto refs = project_index.symbols[symbol_id].reference_files; + + /// FIXME: We may want to parallelize this ... + for(auto file: refs) { + auto& relations = in_memory_indices[file].relations[symbol_id]; + + std::vector results; + for(auto& [relation, _]: relations) { + if(relation.kind & kind) { + results.emplace_back(relation.range); + } + } + + llvm::StringRef path = project_index.path_pool.path(file); + auto content = fs::read(path); + if(!content) { + continue; + } + + /// FIXME: User server's encoding kind. + ranges::sort(results, refl::less); + PositionConverter converter(*content, PositionEncodingKind::UTF16); + + for(auto result: results) { + auto begin = converter.toPosition(result.begin); + auto end = converter.toPosition(result.end); + locations.emplace_back(path.str(), proto::Range(begin, end)); + } + } + + co_return locations; +} + +auto Indexer::declaration(llvm::StringRef path, std::uint32_t offset) -> Result { + co_return co_await lookup(path, + offset, + RelationKind(RelationKind::Declaration, RelationKind::Definition)); +} + +auto Indexer::definition(llvm::StringRef path, std::uint32_t offset) -> Result { + co_return co_await lookup(path, offset, RelationKind::Definition); +} + +auto Indexer::references(llvm::StringRef path, std::uint32_t offset) -> Result { + co_return co_await lookup( + path, + offset, + RelationKind(RelationKind::Declaration, RelationKind::Definition, RelationKind::Reference)); +} + +} // namespace clice diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index 07b82178..a45f9b03 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -61,6 +61,11 @@ async::Task Server::on_initialize(proto::InitializeParams params) { /// SignatureHelp capabilities.signatureHelpProvider.triggerCharacters = {"(", ")", "{", "}", "<", ">", ","}; + /// FIXME: In the future, we would support work done progress. + capabilities.declarationProvider.workDoneProgress = false; + capabilities.definitionProvider.workDoneProgress = false; + capabilities.referencesProvider.workDoneProgress = false; + /// DocumentSymbol capabilities.documentSymbolProvider = {}; @@ -91,6 +96,7 @@ async::Task Server::on_initialize(proto::InitializeParams params) { } async::Task<> Server::on_initialized(proto::InitializedParams) { + co_await indexer.index_all(); co_return; } diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index ee6f4d70..2bdae212 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -95,7 +95,7 @@ async::Task<> Server::registerCapacity(llvm::StringRef id, }); } -Server::Server() { +Server::Server() : indexer(database) { register_callback<&Server::on_initialize>("initialize"); register_callback<&Server::on_initialized>("initialized"); register_callback<&Server::on_shutdown>("shutdown"); @@ -109,6 +109,9 @@ Server::Server() { register_callback<&Server::on_completion>("textDocument/completion"); register_callback<&Server::on_hover>("textDocument/hover"); register_callback<&Server::on_signature_help>("textDocument/signatureHelp"); + register_callback<&Server::on_go_to_declaration>("textDocument/declaration"); + register_callback<&Server::on_go_to_definition>("textDocument/definition"); + register_callback<&Server::on_find_references>("textDocument/references"); register_callback<&Server::on_document_symbol>("textDocument/documentSymbol"); register_callback<&Server::on_document_link>("textDocument/documentLink"); register_callback<&Server::on_document_format>("textDocument/formatting");