diff --git a/.gitattributes b/.gitattributes index 997504b4..d2eb3c4e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,6 @@ # SCM syntax highlighting & preventing 3-way merges pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff + +# Force LF line endings for test data so that byte offsets from clang +# (which reads from disk) match the content sent by didOpen in tests. +tests/data/** text eol=lf diff --git a/src/index/include_graph.h b/src/index/include_graph.h index de55e12b..8e9da218 100644 --- a/src/index/include_graph.h +++ b/src/index/include_graph.h @@ -57,7 +57,7 @@ struct IncludeGraph { return it->second; } - std::uint32_t path_id(clang::FileID fid) { + std::uint32_t path_id(clang::FileID fid) const { auto include = include_location_id(fid); if(include != -1) { return locations[include].path_id; diff --git a/src/index/merged_index.cpp b/src/index/merged_index.cpp index d170bad9..b8c2dbbf 100644 --- a/src/index/merged_index.cpp +++ b/src/index/merged_index.cpp @@ -230,6 +230,16 @@ void MergedIndex::load_in_memory(this Self& self) { index.compilation_contexts.try_emplace(path, std::move(context)); } + // Count ref counts from compilation contexts. + for(auto entry: *root->compilation_contexts()) { + index.canonical_ref_counts[entry->canonical_id()] += 1; + } + + // Deserialize removed bitmap. + if(root->removed() && root->removed()->size() > 0) { + index.removed = read_bitmap(root->removed()); + } + for(auto entry: *root->occurrences()) { index.occurrences.try_emplace(*safe_cast(entry->occurrence()), read_bitmap(entry->context())); @@ -243,6 +253,10 @@ void MergedIndex::load_in_memory(this Self& self) { } } + if(root->content()) { + index.content = root->content()->str(); + } + self.buffer.reset(); } @@ -337,13 +351,25 @@ void MergedIndex::serialize(this const Self& self, llvm::raw_ostream& out) { return std::get<0>(e); }); + // Serialize removed bitmap. + buffer.clear(); + if(!index->removed.isEmpty()) { + buffer.resize_for_overwrite(index->removed.getSizeInBytes(false)); + index->removed.write(buffer.data(), false); + } + auto removed = CreateVector(builder, buffer); + + auto content_offset = CreateString(builder, index->content); + auto merged_index = binary::CreateMergedIndex(builder, index->max_canonical_id, CreateVector(builder, canonical_cache), CreateVector(builder, header_contexts), CreateVector(builder, compilation_contexts), CreateVector(builder, occurrences), - CreateVector(builder, relations)); + CreateVector(builder, relations), + removed, + content_offset); builder.Finish(merged_index); out.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); @@ -371,6 +397,18 @@ void MergedIndex::lookup(this const Self& self, while(it != occurrences.end()) { if(it->range.contains(offset)) { + // Skip occurrences whose canonical_ids are all removed. + if(!index.removed.isEmpty()) { + auto bitmap_it = index.occurrences.find(*it); + if(bitmap_it != index.occurrences.end()) { + auto remaining = bitmap_it->second - index.removed; + if(remaining.isEmpty()) { + it++; + continue; + } + } + } + if(!callback(*it)) { break; } @@ -416,8 +454,16 @@ void MergedIndex::lookup(this const Self& self, } auto& relations = it->second; - for(auto& [relation, _]: relations) { + for(auto& [relation, bitmap]: relations) { if(relation.kind & kind) { + // Skip relations whose canonical_ids are all removed. + if(!self.impl->removed.isEmpty()) { + auto remaining = bitmap - self.impl->removed; + if(remaining.isEmpty()) { + continue; + } + } + if(!callback(relation)) { break; } @@ -504,43 +550,78 @@ void MergedIndex::remove(this Self& self, std::uint32_t path_id) { self.load_in_memory(); auto& index = *self.impl; - auto& includes = index.header_contexts[path_id].includes; + // Handle header context removal. + auto hc_it = index.header_contexts.find(path_id); + if(hc_it != index.header_contexts.end()) { + for(auto& [_, canonical_id]: hc_it->second.includes) { + auto& ref_counts = index.canonical_ref_counts[canonical_id]; + ref_counts -= 1; + if(ref_counts == 0) { + index.removed.add(canonical_id); + } + } + index.header_contexts.erase(hc_it); + } - for(auto& [_, canonical_id]: includes) { + // Handle compilation context removal. + auto cc_it = index.compilation_contexts.find(path_id); + if(cc_it != index.compilation_contexts.end()) { + auto canonical_id = cc_it->second.canonical_id; auto& ref_counts = index.canonical_ref_counts[canonical_id]; ref_counts -= 1; - if(ref_counts == 0) { index.removed.add(canonical_id); } + index.compilation_contexts.erase(cc_it); } - includes.clear(); + // Invalidate cached occurrences. + index.occurrences_cache.clear(); } void MergedIndex::merge(this Self& self, std::uint32_t path_id, std::chrono::milliseconds build_at, std::vector include_locations, - FileIndex& index) { + FileIndex& index, + llvm::StringRef content) { self.load_in_memory(); + self.impl->content = content.str(); self.impl->merge(path_id, index, [&](Impl& self, std::uint32_t canonical_id) { auto& context = self.compilation_contexts[path_id]; context.canonical_id = canonical_id; context.build_at = build_at.count(); context.include_locations = std::move(include_locations); }); + self.impl->occurrences_cache.clear(); } void MergedIndex::merge(this Self& self, std::uint32_t path_id, std::uint32_t include_id, - FileIndex& index) { + FileIndex& index, + llvm::StringRef content) { self.load_in_memory(); + if(self.impl->content.empty() && !content.empty()) { + self.impl->content = content.str(); + } self.impl->merge(path_id, index, [&](Impl& self, std::uint32_t canonical_id) { auto& context = self.header_contexts[path_id]; context.includes.emplace_back(include_id, canonical_id); }); + self.impl->occurrences_cache.clear(); +} + +llvm::StringRef MergedIndex::content(this const Self& self) { + if(self.impl) { + return self.impl->content; + } else if(self.buffer) { + auto root = fbs::GetRoot(self.buffer->getBufferStart()); + if(root->content()) { + return root->content()->string_view(); + } + } + return {}; } bool operator==(MergedIndex& lhs, MergedIndex& rhs) { diff --git a/src/index/merged_index.h b/src/index/merged_index.h index 4f540843..2e4bc375 100644 --- a/src/index/merged_index.h +++ b/src/index/merged_index.h @@ -64,15 +64,23 @@ public: /// Remove the index of specific path id. void remove(this Self& self, std::uint32_t path_id); + /// Get the stored source content for position mapping. + llvm::StringRef content(this const Self& self); + /// Merge the index with given compilation context. void merge(this Self& self, std::uint32_t path_id, std::chrono::milliseconds build_at, std::vector include_locations, - FileIndex& index); + FileIndex& index, + llvm::StringRef content); /// Merge the index with given header context. - void merge(this Self& self, std::uint32_t path_id, std::uint32_t include_id, FileIndex& index); + void merge(this Self& self, + std::uint32_t path_id, + std::uint32_t include_id, + FileIndex& index, + llvm::StringRef content); friend bool operator==(MergedIndex& lhs, MergedIndex& rhs); diff --git a/src/index/project_index.cpp b/src/index/project_index.cpp index 28bc4751..08741383 100644 --- a/src/index/project_index.cpp +++ b/src/index/project_index.cpp @@ -9,12 +9,16 @@ llvm::SmallVector ProjectIndex::merge(this ProjectIndex& self, TU llvm::SmallVector file_ids_map; file_ids_map.resize_for_overwrite(paths.size()); - for(auto i = 0; i < paths.size(); i++) { + for(std::uint32_t i = 0; i < paths.size(); i++) { file_ids_map[i] = self.path_pool.path_id(paths[i]); } for(auto& [symbol_id, symbol]: index.symbols) { auto& target_symbol = self.symbols[symbol_id]; + if(target_symbol.name.empty()) { + target_symbol.name = symbol.name; + target_symbol.kind = symbol.kind; + } for(auto ref: symbol.reference_files) { target_symbol.reference_files.add(file_ids_map[ref]); } @@ -48,10 +52,12 @@ void ProjectIndex::serialize(this ProjectIndex& self, llvm::raw_ostream& os) { buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false)); symbol.reference_files.write(buffer.data(), false); - return binary::CreateSymbolEntry( - builder, - symbol_id, - binary::CreateSymbol(builder, symbol.kind.value(), CreateVector(builder, buffer))); + return binary::CreateSymbolEntry(builder, + symbol_id, + binary::CreateSymbol(builder, + CreateString(builder, symbol.name), + symbol.kind.value(), + CreateVector(builder, buffer))); }); auto project_index = @@ -72,7 +78,11 @@ ProjectIndex ProjectIndex::from(const void* data) { auto& pool = index.path_pool; pool.paths.resize(root->paths()->size()); for(auto entry: *root->paths()) { - auto k = pool.save(entry->path()->string_view()); + // Normalize backslashes to forward slashes for cross-platform consistency + // (persisted index may contain native-separator paths from Windows). + llvm::SmallString<256> normalized(entry->path()->string_view()); + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + auto k = pool.save(normalized.str()); pool.paths[entry->id()] = k; pool.cache.try_emplace(k, entry->id()); } @@ -83,8 +93,12 @@ ProjectIndex ProjectIndex::from(const void* data) { for(auto entry: *root->symbols()) { auto& symbol = index.symbols[entry->symbol_id()]; - symbol.kind = SymbolKind(static_cast(entry->symbol()->kind())); - symbol.reference_files = read_bitmap(entry->symbol()->refs()); + auto* fb_symbol = entry->symbol(); + if(auto* name = fb_symbol->name()) { + symbol.name = name->str(); + } + symbol.kind = SymbolKind(static_cast(fb_symbol->kind())); + symbol.reference_files = read_bitmap(fb_symbol->refs()); } return index; diff --git a/src/index/project_index.h b/src/index/project_index.h index 09940bff..12bbebe0 100644 --- a/src/index/project_index.h +++ b/src/index/project_index.h @@ -7,6 +7,7 @@ #include "index/tu_index.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Allocator.h" @@ -29,6 +30,17 @@ struct PathPool { auto path_id(llvm::StringRef path) { assert(!path.empty()); + + // Normalize backslashes to forward slashes so that paths from different + // sources (URI decoding, CDB, clang FileManager) compare equal on + // Windows where native separators are backslashes. + llvm::SmallString<256> normalized; + if(path.contains('\\')) { + normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + path = normalized; + } + auto [it, success] = cache.try_emplace(path, paths.size()); if(!success) { return it->second; @@ -43,6 +55,18 @@ struct PathPool { llvm::StringRef path(std::uint32_t id) { return paths[id]; } + + /// Look up a path in the cache, normalizing backslashes first. + /// Returns cache.end() if the path is not interned. + auto find(llvm::StringRef path) { + llvm::SmallString<256> normalized; + if(path.contains('\\')) { + normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + path = normalized; + } + return cache.find(path); + } }; struct FileInfo { diff --git a/src/index/schema.fbs b/src/index/schema.fbs index b9e6cbf9..e25e1f29 100644 --- a/src/index/schema.fbs +++ b/src/index/schema.fbs @@ -1,103 +1,173 @@ namespace clice.index.binary; struct Range { - begin: uint; - end: uint; + begin : uint; + end : uint; } struct Occurrence { - range: Range; - target: ulong; + range : Range; + target : ulong; } struct Relation { - kind: uint; - padding: uint; - range: Range; - target_symbol: ulong; + kind : uint; + padding : uint; + range : Range; + target_symbol : ulong; } table CacheEntry { - sha256: string; - canonical_id: uint; +sha256: + string; +canonical_id: + uint; } struct IncludeContext { - include_id: uint; - canonical_id: uint; + include_id : uint; + canonical_id : uint; } table HeaderContextEntry { - path_id: uint; - version: uint; - includes: [IncludeContext]; +path_id: + uint; +version: + uint; +includes: + [IncludeContext]; } struct IncludeLocation { - path_id: uint; - line: uint; - include_id: uint; + path_id : uint; + line : uint; + include_id : uint; } table CompilationContextEntry { - path_id: uint; - version: uint; - canonical_id: uint; - build_at: ulong; - include_locations: [IncludeLocation]; +path_id: + uint; +version: + uint; +canonical_id: + uint; +build_at: + ulong; +include_locations: + [IncludeLocation]; } table OccurrenceEntry { - occurrence: Occurrence; - context: [ubyte]; +occurrence: + Occurrence; +context: + [ubyte]; } table RelationEntry { - relation: Relation; - context: [ubyte]; +relation: + Relation; +context: + [ubyte]; } table SymbolRelationsEntry { - symbol: ulong; - relations: [RelationEntry]; +symbol: + ulong; +relations: + [RelationEntry]; } table Symbol { - kind: ubyte; - refs: [ubyte]; +name: + string; +kind: + ubyte; +refs: + [ubyte]; } table SymbolEntry { - symbol_id: ulong; - symbol: Symbol; +symbol_id: + ulong; +symbol: + Symbol; } table MergedIndex { - max_canonical_id: uint; +max_canonical_id: + uint; - canonical_cache: [CacheEntry]; +canonical_cache: + [CacheEntry]; - header_contexts: [HeaderContextEntry]; +header_contexts: + [HeaderContextEntry]; - compilation_contexts: [CompilationContextEntry]; +compilation_contexts: + [CompilationContextEntry]; - occurrences: [OccurrenceEntry]; +occurrences: + [OccurrenceEntry]; - relations: [SymbolRelationsEntry]; +relations: + [SymbolRelationsEntry]; + +removed: + [ubyte]; + +content: + string; +} + +table TUFileRelationsEntry { +symbol: + ulong; +relations: + [Relation]; +} + +table TUFileIndexEntry { +file_id: + uint; +occurrences: + [Occurrence]; +relations: + [TUFileRelationsEntry]; +} + +table TUIndex { +built_at: + ulong; +paths: + [string]; +locations: + [IncludeLocation]; +symbols: + [SymbolEntry]; +file_indices: + [TUFileIndexEntry]; +main_file_index: + TUFileIndexEntry; } table PathEntry { - path: string; - id: uint; +path: + string; +id: + uint; } struct PathMapEntry { - source: uint; - index: uint; + source : uint; + index : uint; } table ProjectIndex { - paths: [PathEntry]; - indices: [PathMapEntry]; - symbols: [SymbolEntry]; +paths: + [PathEntry]; +indices: + [PathMapEntry]; +symbols: + [SymbolEntry]; } diff --git a/src/index/tu_index.cpp b/src/index/tu_index.cpp index 7f27120a..67e4dd69 100644 --- a/src/index/tu_index.cpp +++ b/src/index/tu_index.cpp @@ -2,6 +2,7 @@ #include +#include "index/serialization.h" #include "semantic/ast_utility.h" #include "semantic/semantic_visitor.h" @@ -65,7 +66,7 @@ public: index.occurrences.emplace_back(range, symbol_id.hash); Relation relation{ - .kind = RelationKind::Definition, + .kind = kind, .range = range, .target_symbol = 0, }; @@ -197,4 +198,120 @@ TUIndex TUIndex::build(CompilationUnitRef unit) { return index; } +void TUIndex::serialize(llvm::raw_ostream& os) const { + fbs::FlatBufferBuilder builder(4096); + + llvm::SmallVector buffer; + + auto paths = + transform(graph.paths, [&](const std::string& p) { return builder.CreateString(p); }); + + auto syms = transform(symbols, [&](auto&& value) { + auto& [symbol_id, symbol] = value; + buffer.clear(); + buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false)); + symbol.reference_files.write(buffer.data(), false); + return binary::CreateSymbolEntry(builder, + symbol_id, + binary::CreateSymbol(builder, + CreateString(builder, symbol.name), + symbol.kind.value(), + CreateVector(builder, buffer))); + }); + + /// Serialize a single FileIndex into a TUFileIndexEntry. + auto serialize_file_index = [&](std::uint32_t fid, const FileIndex& index) { + auto occs = CreateStructVector(builder, index.occurrences); + auto rels = transform(index.relations, [&](auto&& value) { + auto& [symbol_id, relations] = value; + return binary::CreateTUFileRelationsEntry( + builder, + symbol_id, + CreateStructVector(builder, relations)); + }); + return binary::CreateTUFileIndexEntry(builder, fid, occs, CreateVector(builder, rels)); + }; + + /// Convert FileID-keyed file_indices to path_id-keyed entries. + llvm::SmallVector> file_idx_vec; + for(auto& [fid, index]: file_indices) { + auto pid = graph.path_id(fid); + file_idx_vec.push_back(serialize_file_index(pid, index)); + } + + /// Main file is the last path in graph.paths (convention from IncludeGraph). + auto main_idx = + serialize_file_index(static_cast(graph.paths.size() - 1), main_file_index); + + auto tu_index = + binary::CreateTUIndex(builder, + static_cast(built_at.count()), + CreateVector(builder, paths), + CreateStructVector(builder, graph.locations), + CreateVector(builder, syms), + builder.CreateVector(file_idx_vec.data(), file_idx_vec.size()), + main_idx); + + builder.Finish(tu_index); + os.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); +} + +TUIndex TUIndex::from(const void* data) { + auto root = fbs::GetRoot(data); + + TUIndex index; + index.built_at = std::chrono::milliseconds(root->built_at()); + + for(auto p: *root->paths()) { + index.graph.paths.emplace_back(p->str()); + } + + for(auto loc: *root->locations()) { + index.graph.locations.emplace_back(*safe_cast(loc)); + } + + for(auto entry: *root->symbols()) { + auto& symbol = index.symbols[entry->symbol_id()]; + symbol.name = entry->symbol()->name()->str(); + symbol.kind = SymbolKind(static_cast(entry->symbol()->kind())); + symbol.reference_files = read_bitmap(entry->symbol()->refs()); + } + + /// Helper to deserialize a TUFileIndexEntry into a FileIndex. + auto deserialize_file_index = [](const binary::TUFileIndexEntry* entry) -> FileIndex { + FileIndex fi; + if(entry->occurrences()) { + fi.occurrences.reserve(entry->occurrences()->size()); + for(auto o: *entry->occurrences()) { + fi.occurrences.emplace_back(*safe_cast(o)); + } + } + if(entry->relations()) { + for(auto rel_entry: *entry->relations()) { + auto& rels = fi.relations[rel_entry->symbol()]; + if(rel_entry->relations()) { + rels.reserve(rel_entry->relations()->size()); + for(auto r: *rel_entry->relations()) { + rels.emplace_back(*safe_cast(r)); + } + } + } + } + return fi; + }; + + /// Populate path_file_indices keyed by path_id (no clang::FileID needed). + if(root->file_indices()) { + for(auto entry: *root->file_indices()) { + index.path_file_indices[entry->file_id()] = deserialize_file_index(entry); + } + } + + if(root->main_file_index()) { + index.main_file_index = deserialize_file_index(root->main_file_index()); + } + + return index; +} + } // namespace clice::index diff --git a/src/index/tu_index.h b/src/index/tu_index.h index 93318323..308a2a3d 100644 --- a/src/index/tu_index.h +++ b/src/index/tu_index.h @@ -12,6 +12,8 @@ #include "semantic/symbol_kind.h" #include "support/bitmap.h" +#include "llvm/Support/raw_ostream.h" + namespace clice::index { using Range = LocalSourceRange; @@ -77,9 +79,17 @@ struct TUIndex { llvm::DenseMap file_indices; + /// File indices keyed by path_id, populated by from() for deserialized data. + /// When built from AST, this is empty and file_indices (keyed by FileID) is used. + llvm::DenseMap path_file_indices; + FileIndex main_file_index; static TUIndex build(CompilationUnitRef unit); + + void serialize(llvm::raw_ostream& os) const; + + static TUIndex from(const void* data); }; } // namespace clice::index diff --git a/src/semantic/ast_utility.cpp b/src/semantic/ast_utility.cpp index c0549924..0be1b7d4 100644 --- a/src/semantic/ast_utility.cpp +++ b/src/semantic/ast_utility.cpp @@ -308,6 +308,12 @@ const clang::NamedDecl* decl_of_impl(const void* T) { } auto decl_of(clang::QualType type) -> const clang::NamedDecl* { + // Strip type-sugar that wraps the underlying type without adding a decl + // (e.g. ElaboratedType for "struct Foo" vs plain "Foo"). + if(auto ET = type->getAs()) { + type = ET->getNamedType(); + } + if(auto TST = type->getAs()) { auto decl = TST->getTemplateName().getAsTemplateDecl(); if(type->isDependentType()) { diff --git a/src/server/config.cpp b/src/server/config.cpp index 89d2aa6c..68078fcc 100644 --- a/src/server/config.cpp +++ b/src/server/config.cpp @@ -37,9 +37,14 @@ void CliceConfig::apply_defaults(const std::string& workspace_root) { cache_dir = path::join(workspace_root, ".clice"); } + if(index_dir.empty() && !cache_dir.empty()) { + index_dir = path::join(cache_dir, "index"); + } + // Apply variable substitution to string fields substitute_workspace(compile_commands_path, workspace_root); substitute_workspace(cache_dir, workspace_root); + substitute_workspace(index_dir, workspace_root); } std::optional CliceConfig::load(const std::string& path, diff --git a/src/server/config.h b/src/server/config.h index 087c351f..4e103cf2 100644 --- a/src/server/config.h +++ b/src/server/config.h @@ -19,6 +19,9 @@ struct CliceConfig { // Cache directory (empty = default: /.clice/) std::string cache_dir; + // Index storage directory (default: /index/) + std::string index_dir; + // Debounce interval for re-compilation after edits (milliseconds) int debounce_ms = 200; diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index 1187da16..a8a7d6da 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -6,11 +6,13 @@ #include #include +#include "eventide/ipc/json_codec.h" #include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/uri.h" #include "eventide/reflection/enum.h" #include "eventide/serde/json/json.h" #include "eventide/serde/serde/raw_value.h" +#include "index/tu_index.h" #include "semantic/symbol_kind.h" #include "server/protocol.h" #include "support/filesystem.h" @@ -18,6 +20,7 @@ #include "syntax/dependency_graph.h" #include "syntax/scan.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Support/xxhash.h" namespace clice { @@ -225,6 +228,8 @@ et::task<> MasterServer::run_build_drain(std::uint32_t path_id, std::string uri) if(!doc2.build_requested) { doc2.build_running = false; doc2.drain_scheduled = false; + // Trigger background indexing after successful compile settles. + schedule_indexing(); co_return; } // Loop continues for the next build @@ -306,6 +311,26 @@ et::task<> MasterServer::load_workspace() { } } + // Load persisted index from disk. + load_index(); + + // Build index queue from CDB entries (all source files). + // CDB entries use the CDB's internal path_ids; convert to server path_ids. + if(config.enable_indexing) { + for(auto& entry: cdb.get_entries()) { + auto file = cdb.resolve_path(entry.file); + auto server_id = path_pool.intern(file); + index_queue.push_back(server_id); + } + if(!index_queue.empty()) { + LOG_INFO("Queued {} files for background indexing", index_queue.size()); + for(auto sid: index_queue) { + LOG_INFO(" queue entry: server_path_id={} path='{}'", sid, path_pool.resolve(sid)); + } + schedule_indexing(); + } + } + if(path_to_module.empty()) { LOG_INFO("No C++20 modules detected, skipping CompileGraph"); co_return; @@ -475,6 +500,270 @@ et::task MasterServer::ensure_compiled(std::uint32_t path_id, const std::s co_return true; } +// ========================================================================= +// Index integration +// ========================================================================= + +void MasterServer::merge_index_result(const void* tu_index_data, std::size_t size) { + auto tu_index = index::TUIndex::from(tu_index_data); + + // Merge symbols into ProjectIndex, get TU-local path_id -> global path_id mapping. + auto file_ids_map = project_index.merge(tu_index); + + auto main_tu_path_id = static_cast(tu_index.graph.paths.size() - 1); + + // Merge a single file's index into the corresponding MergedIndex shard. + auto merge_file_index = [&](std::uint32_t tu_path_id, index::FileIndex& file_idx) { + auto global_path_id = file_ids_map[tu_path_id]; + auto& merged = merged_indices[global_path_id]; + + if(tu_path_id == main_tu_path_id) { + // Main file (source file) gets a compilation context with include locations. + // Collect ALL include locations with path_ids remapped to project-level ids. + std::vector include_locs; + for(auto& loc: tu_index.graph.locations) { + index::IncludeLocation remapped = loc; + remapped.path_id = file_ids_map[loc.path_id]; + include_locs.push_back(remapped); + } + // Read the file content from disk for position mapping in queries. + auto file_path = project_index.path_pool.path(global_path_id); + llvm::StringRef file_content; + std::string file_content_storage; + auto buf = llvm::MemoryBuffer::getFile(file_path); + if(buf) { + file_content_storage = (*buf)->getBuffer().str(); + file_content = file_content_storage; + } + merged.merge(global_path_id, + tu_index.built_at, + std::move(include_locs), + file_idx, + file_content); + } else { + // Header files get a header context keyed by include location. + std::uint32_t include_id = 0; + for(std::uint32_t i = 0; i < tu_index.graph.locations.size(); ++i) { + if(tu_index.graph.locations[i].path_id == tu_path_id) { + include_id = i; + break; + } + } + // Read header file content for position mapping in queries. + auto header_path = project_index.path_pool.path(global_path_id); + llvm::StringRef header_content; + std::string header_content_storage; + auto header_buf = llvm::MemoryBuffer::getFile(header_path); + if(header_buf) { + header_content_storage = (*header_buf)->getBuffer().str(); + header_content = header_content_storage; + } + merged.merge(global_path_id, include_id, file_idx, header_content); + } + }; + + // Merge from path_file_indices (deserialized TUIndex from IPC). + for(auto& [tu_path_id, file_idx]: tu_index.path_file_indices) { + merge_file_index(tu_path_id, file_idx); + } + + // Merge main file index. + merge_file_index(main_tu_path_id, tu_index.main_file_index); + + LOG_INFO("Merged TUIndex: {} paths, {} symbols, {} merged_shards", + tu_index.graph.paths.size(), + tu_index.symbols.size(), + merged_indices.size()); + for(auto& [pid, _]: merged_indices) { + LOG_INFO(" shard proj_path_id={} path='{}'", pid, project_index.path_pool.path(pid)); + } +} + +void MasterServer::save_index() { + if(config.index_dir.empty()) + return; + + auto ec = llvm::sys::fs::create_directories(config.index_dir); + if(ec) { + LOG_WARN("Failed to create index directory {}: {}", config.index_dir, ec.message()); + return; + } + + // Save ProjectIndex. + auto project_path = path::join(config.index_dir, "project.idx"); + { + std::error_code write_ec; + llvm::raw_fd_ostream os(project_path, write_ec); + if(!write_ec) { + project_index.serialize(os); + LOG_INFO("Saved ProjectIndex to {}", project_path); + } else { + LOG_WARN("Failed to save ProjectIndex: {}", write_ec.message()); + } + } + + // Save MergedIndex shards. + auto shards_dir = path::join(config.index_dir, "shards"); + ec = llvm::sys::fs::create_directories(shards_dir); + if(ec) { + LOG_WARN("Failed to create shards directory: {}", ec.message()); + return; + } + + std::size_t saved = 0; + for(auto& [path_id, merged]: merged_indices) { + if(!merged.need_rewrite()) + continue; + auto shard_path = path::join(shards_dir, std::to_string(path_id) + ".idx"); + std::error_code write_ec; + llvm::raw_fd_ostream os(shard_path, write_ec); + if(!write_ec) { + merged.serialize(os); + ++saved; + } + } + LOG_INFO("Saved {} MergedIndex shards (of {} total)", saved, merged_indices.size()); +} + +void MasterServer::load_index() { + if(config.index_dir.empty()) + return; + + // Load ProjectIndex. + auto project_path = path::join(config.index_dir, "project.idx"); + auto buf = llvm::MemoryBuffer::getFile(project_path); + if(buf) { + project_index = index::ProjectIndex::from((*buf)->getBufferStart()); + LOG_INFO("Loaded ProjectIndex: {} symbols", project_index.symbols.size()); + } + + // Load MergedIndex shards. + auto shards_dir = path::join(config.index_dir, "shards"); + std::error_code ec; + for(auto it = llvm::sys::fs::directory_iterator(shards_dir, ec); + !ec && it != llvm::sys::fs::directory_iterator(); + it.increment(ec)) { + auto filename = llvm::sys::path::filename(it->path()); + if(!filename.ends_with(".idx")) + continue; + + auto stem = filename.drop_back(4); // remove ".idx" + std::uint32_t path_id = 0; + if(stem.getAsInteger(10, path_id)) + continue; + + merged_indices[path_id] = index::MergedIndex::load(it->path()); + } + + if(!merged_indices.empty()) { + LOG_INFO("Loaded {} MergedIndex shards", merged_indices.size()); + } +} + +void MasterServer::schedule_indexing() { + LOG_INFO( + "schedule_indexing called: enable={} active={} scheduled={} queue_size={} queue_pos={}", + config.enable_indexing, + indexing_active, + indexing_scheduled, + index_queue.size(), + index_queue_pos); + if(!config.enable_indexing || indexing_active || indexing_scheduled) + return; + indexing_scheduled = true; + + // Create or reset idle timer. + if(!index_idle_timer) { + index_idle_timer = std::make_shared(et::timer::create(loop)); + } + index_idle_timer->start(std::chrono::milliseconds(config.idle_timeout_ms)); + loop.schedule(run_background_indexing()); +} + +et::task<> MasterServer::run_background_indexing() { + // Wait for idle timeout before starting. + if(index_idle_timer) { + co_await index_idle_timer->wait(); + } + indexing_scheduled = false; + + if(index_queue_pos >= index_queue.size()) { + LOG_DEBUG("Background indexing: queue exhausted"); + co_return; + } + + indexing_active = true; + std::size_t processed = 0; + + while(index_queue_pos < index_queue.size()) { + auto server_path_id = index_queue[index_queue_pos]; + index_queue_pos++; + + auto file_path = std::string(path_pool.resolve(server_path_id)); + + // Note: we do NOT skip open documents here. Index data is needed for + // cross-file features (references, call hierarchy, type hierarchy, etc.) + // regardless of whether the file is open. + + // Check if the index needs update by checking mtime against existing shard. + // If the file is not yet in the project_index path pool, it has never been + // indexed — always proceed. Only skip when we already have a shard that is + // still fresh. + auto cache_it = project_index.path_pool.find(file_path); + if(cache_it != project_index.path_pool.cache.end()) { + auto proj_path_id = cache_it->second; + auto merged_it = merged_indices.find(proj_path_id); + if(merged_it != merged_indices.end()) { + // Build path mapping for need_update check. + llvm::SmallVector path_mapping; + for(auto& p: project_index.path_pool.paths) { + path_mapping.push_back(p); + } + if(!merged_it->second.need_update(path_mapping)) + continue; + } + } + + // Prepare IndexParams for the stateless worker. + worker::IndexParams params; + params.file = file_path; + if(!fill_compile_args(file_path, params.directory, params.arguments)) + continue; + + // Fill PCM deps for module-aware indexing. + for(auto& [pid, pcm_path]: pcm_paths) { + auto mod_it = path_to_module.find(pid); + if(mod_it != path_to_module.end()) { + params.pcms[mod_it->second] = pcm_path; + } + } + + LOG_INFO("Background indexing: {}", file_path); + + auto result = co_await pool.send_stateless(params); + if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { + LOG_INFO("Background indexing got TUIndex for {}: {} bytes", + file_path, + result.value().tu_index_data.size()); + merge_index_result(result.value().tu_index_data.data(), + result.value().tu_index_data.size()); + ++processed; + } else if(result.has_value() && !result.value().success) { + LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); + } else if(result.has_value() && result.value().tu_index_data.empty()) { + LOG_WARN("Background index returned empty TUIndex for {}", file_path); + } else { + LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); + } + } + + indexing_active = false; + LOG_INFO("Background indexing complete: {} files processed", processed); + + // Persist index to disk after a full pass. + save_index(); +} + // ========================================================================= // Forwarding helpers // ========================================================================= @@ -573,6 +862,288 @@ MasterServer::RawResult MasterServer::forward_stateless(const std::string& uri, co_return std::move(result.value()); } +// Serialize a value to a JSON RawValue using LSP config. +template +static serde_raw to_raw(const T& value) { + auto json = et::serde::json::to_json(value); + return serde_raw{json ? std::move(*json) : "null"}; +} + +MasterServer::RawResult MasterServer::query_index_relations(const std::string& uri, + const protocol::Position& position, + RelationKind kind) { + auto path = uri_to_path(uri); + auto server_path_id = path_pool.intern(path); + + // Need document text for position-to-offset conversion. + auto doc_it = documents.find(server_path_id); + if(doc_it == documents.end()) + co_return serde_raw{"null"}; + + lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); + auto offset_opt = mapper.to_offset(position); + if(!offset_opt) + co_return serde_raw{"null"}; + auto offset = *offset_opt; + + // Find the project-level path_id for this file. + auto proj_cache_it = project_index.path_pool.find(path); + if(proj_cache_it == project_index.path_pool.cache.end()) { + LOG_DEBUG("query_index_relations: path '{}' not in project_index", path); + co_return serde_raw{"null"}; + } + auto proj_path_id = proj_cache_it->second; + + // Lookup occurrence at offset in this file's MergedIndex. + auto merged_it = merged_indices.find(proj_path_id); + if(merged_it == merged_indices.end()) { + LOG_DEBUG("query_index_relations: no MergedIndex for proj_path_id={}", proj_path_id); + co_return serde_raw{"null"}; + } + + index::SymbolHash symbol_hash = 0; + merged_it->second.lookup(offset, [&](const index::Occurrence& o) { + symbol_hash = o.target; + return false; // stop after first match + }); + + if(symbol_hash == 0) { + LOG_DEBUG("query_index_relations: no occurrence at offset {} in '{}'", offset, path); + co_return serde_raw{"null"}; + } + + // Get reference files from ProjectIndex. + auto sym_it = project_index.symbols.find(symbol_hash); + if(sym_it == project_index.symbols.end()) { + LOG_DEBUG("query_index_relations: symbol {} not in project_index", symbol_hash); + co_return serde_raw{"null"}; + } + + // Query each referenced file's MergedIndex for relations of the requested kind. + std::vector locations; + + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + auto file_path = project_index.path_pool.path(file_id); + auto file_uri = lsp::URI::from_file_path(file_path); + if(!file_uri) + continue; + auto file_uri_str = file_uri->str(); + + // Use stored content from MergedIndex for offset-to-position conversion. + auto file_content = file_merged_it->second.content(); + if(file_content.empty()) + continue; + + lsp::PositionMapper file_mapper(file_content, lsp::PositionEncoding::UTF16); + + file_merged_it->second.lookup(symbol_hash, kind, [&](const index::Relation& r) { + auto start = file_mapper.to_position(r.range.begin); + auto end = file_mapper.to_position(r.range.end); + if(start && end) { + protocol::Location loc; + loc.uri = file_uri_str; + loc.range = protocol::Range{*start, *end}; + locations.push_back(std::move(loc)); + } + return true; // continue collecting + }); + } + + if(locations.empty()) + co_return serde_raw{"null"}; + + co_return to_raw(locations); +} + +protocol::SymbolKind MasterServer::to_lsp_symbol_kind(SymbolKind kind) { + switch(kind) { + case SymbolKind::Namespace: return protocol::SymbolKind::Namespace; + case SymbolKind::Class: return protocol::SymbolKind::Class; + case SymbolKind::Struct: return protocol::SymbolKind::Struct; + case SymbolKind::Union: return protocol::SymbolKind::Class; + case SymbolKind::Enum: return protocol::SymbolKind::Enum; + case SymbolKind::Type: return protocol::SymbolKind::TypeParameter; + case SymbolKind::Field: return protocol::SymbolKind::Field; + case SymbolKind::EnumMember: return protocol::SymbolKind::EnumMember; + case SymbolKind::Function: return protocol::SymbolKind::Function; + case SymbolKind::Method: return protocol::SymbolKind::Method; + case SymbolKind::Variable: return protocol::SymbolKind::Variable; + case SymbolKind::Parameter: return protocol::SymbolKind::Variable; + case SymbolKind::Macro: return protocol::SymbolKind::Function; + case SymbolKind::Concept: return protocol::SymbolKind::Interface; + case SymbolKind::Module: return protocol::SymbolKind::Module; + case SymbolKind::Operator: return protocol::SymbolKind::Operator; + default: return protocol::SymbolKind::Variable; + } +} + +et::task> + MasterServer::lookup_symbol_at_position(const std::string& uri, + const protocol::Position& position) { + auto path = uri_to_path(uri); + auto server_path_id = path_pool.intern(path); + + // Need document text for position-to-offset conversion. + auto doc_it = documents.find(server_path_id); + if(doc_it == documents.end()) + co_return std::nullopt; + + lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); + auto offset_opt = mapper.to_offset(position); + if(!offset_opt) + co_return std::nullopt; + auto offset = *offset_opt; + + // Find the project-level path_id for this file. + auto proj_cache_it = project_index.path_pool.find(path); + if(proj_cache_it == project_index.path_pool.cache.end()) { + LOG_WARN("lookup_symbol: path '{}' not in project_index (pool has {} entries)", + path, + project_index.path_pool.paths.size()); + co_return std::nullopt; + } + auto proj_path_id = proj_cache_it->second; + + // Lookup occurrence at offset in this file's MergedIndex. + auto merged_it = merged_indices.find(proj_path_id); + if(merged_it == merged_indices.end()) { + LOG_WARN("lookup_symbol: no MergedIndex for proj_path_id={} (have {} shards)", + proj_path_id, + merged_indices.size()); + co_return std::nullopt; + } + + index::SymbolHash symbol_hash = 0; + index::Range occ_range{}; + merged_it->second.lookup(offset, [&](const index::Occurrence& o) { + symbol_hash = o.target; + occ_range = o.range; + return false; // stop after first match + }); + + if(symbol_hash == 0) { + LOG_WARN("lookup_symbol: no occurrence at offset {} in '{}'", offset, path); + co_return std::nullopt; + } + + // Get symbol info from ProjectIndex. + auto sym_it = project_index.symbols.find(symbol_hash); + if(sym_it == project_index.symbols.end()) + co_return std::nullopt; + + // Convert occurrence range to LSP Range. + auto start = mapper.to_position(occ_range.begin); + auto end = mapper.to_position(occ_range.end); + if(!start || !end) + co_return std::nullopt; + + SymbolInfo info; + info.hash = symbol_hash; + info.name = sym_it->second.name; + info.kind = sym_it->second.kind; + info.uri = uri; + info.range = protocol::Range{*start, *end}; + co_return info; +} + +std::optional + MasterServer::find_symbol_definition_location(index::SymbolHash hash) { + auto sym_it = project_index.symbols.find(hash); + if(sym_it == project_index.symbols.end()) + return std::nullopt; + + // Search reference files for a Definition relation. + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + auto file_path = project_index.path_pool.path(file_id); + auto file_uri = lsp::URI::from_file_path(file_path); + if(!file_uri) + continue; + + // Use stored content from MergedIndex for offset-to-position conversion. + auto file_content = file_merged_it->second.content(); + if(file_content.empty()) + continue; + lsp::PositionMapper file_mapper(file_content, lsp::PositionEncoding::UTF16); + + std::optional result; + file_merged_it->second.lookup(hash, + RelationKind::Definition, + [&](const index::Relation& r) { + auto start = file_mapper.to_position(r.range.begin); + auto end = file_mapper.to_position(r.range.end); + if(start && end) { + protocol::Location loc; + loc.uri = file_uri->str(); + loc.range = protocol::Range{*start, *end}; + result = std::move(loc); + return false; // found it, stop + } + return true; + }); + + if(result) + return result; + } + + return std::nullopt; +} + +protocol::CallHierarchyItem MasterServer::build_call_hierarchy_item(const SymbolInfo& info) { + protocol::CallHierarchyItem item; + item.name = info.name; + item.kind = to_lsp_symbol_kind(info.kind); + item.uri = info.uri; + item.range = info.range; + item.selection_range = info.range; + // Store the symbol hash in data for later use in incoming/outgoing calls. + item.data = protocol::LSPAny(static_cast(info.hash)); + return item; +} + +protocol::TypeHierarchyItem MasterServer::build_type_hierarchy_item(const SymbolInfo& info) { + protocol::TypeHierarchyItem item; + item.name = info.name; + item.kind = to_lsp_symbol_kind(info.kind); + item.uri = info.uri; + item.range = info.range; + item.selection_range = info.range; + item.data = protocol::LSPAny(static_cast(info.hash)); + return item; +} + +et::task> + MasterServer::resolve_hierarchy_item(const std::string& uri, + const protocol::Range& range, + const std::optional& data) { + // Try to extract symbol hash from the stored data field first. + if(data) { + if(auto* int_val = std::get_if(&*data)) { + auto hash = static_cast(*int_val); + auto sym_it = project_index.symbols.find(hash); + if(sym_it != project_index.symbols.end()) { + SymbolInfo info; + info.hash = hash; + info.name = sym_it->second.name; + info.kind = sym_it->second.kind; + info.uri = uri; + info.range = range; + co_return info; + } + } + } + + // Fallback: re-lookup from position (requires document to be open). + co_return co_await lookup_symbol_at_position(uri, range.start); +} + void MasterServer::register_handlers() { // === initialize === peer.on_request([this](RequestContext& ctx, const protocol::InitializeParams& params) @@ -606,11 +1177,15 @@ void MasterServer::register_handlers() { result.capabilities.completion_provider = protocol::CompletionOptions{}; result.capabilities.signature_help_provider = protocol::SignatureHelpOptions{}; result.capabilities.definition_provider = true; + result.capabilities.references_provider = true; result.capabilities.document_symbol_provider = true; result.capabilities.document_link_provider = protocol::DocumentLinkOptions{}; result.capabilities.code_action_provider = true; result.capabilities.folding_range_provider = true; result.capabilities.inlay_hint_provider = true; + result.capabilities.call_hierarchy_provider = true; + result.capabilities.type_hierarchy_provider = true; + result.capabilities.workspace_symbol_provider = true; // Semantic tokens protocol::SemanticTokensOptions sem_opts; @@ -686,6 +1261,9 @@ void MasterServer::register_handlers() { lifecycle = ServerLifecycle::Exited; LOG_INFO("Exit notification received"); + // Persist index state before stopping. + save_index(); + // Graceful shutdown: cancel compilations, stop workers, then stop loop loop.schedule([this]() -> et::task<> { co_await pool.stop(); @@ -873,9 +1451,50 @@ void MasterServer::register_handlers() { // --- textDocument/definition --- peer.on_request( [this](RequestContext& ctx, const protocol::DefinitionParams& params) -> RawResult { - co_return co_await forward_stateful( - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); + auto& uri = params.text_document_position_params.text_document.uri; + auto& pos = params.text_document_position_params.position; + + // Try index-based lookup first. + auto result = co_await query_index_relations(uri, pos, RelationKind::Definition); + if(result.has_value() && !result.value().empty() && result.value().data != "null") { + co_return std::move(result).value(); + } + + // Fall back to stateful worker AST query. + co_return co_await forward_stateful(uri, pos); + }); + + // --- textDocument/references --- + peer.on_request( + [this](RequestContext& ctx, const protocol::ReferenceParams& params) -> RawResult { + auto& uri = params.text_document_position_params.text_document.uri; + auto& pos = params.text_document_position_params.position; + + auto refs = co_await query_index_relations(uri, pos, RelationKind::Reference); + + if(params.context.include_declaration) { + // Also include Definition locations when the client requests the declaration. + auto defs = co_await query_index_relations(uri, pos, RelationKind::Definition); + if(defs.has_value() && !defs.value().empty() && defs.value().data != "null") { + if(!refs.has_value() || refs.value().empty() || refs.value().data == "null") { + co_return std::move(defs).value(); + } + // Merge: parse both JSON arrays and concatenate. + auto& ref_json = refs.value().data; + auto& def_json = defs.value().data; + // Both are JSON arrays like "[...]". Merge them. + if(ref_json.size() > 2 && def_json.size() > 2) { + // Remove trailing ']' from refs, add comma, add def content without leading + // '[' + std::string merged = ref_json.substr(0, ref_json.size() - 1); + merged += ','; + merged += def_json.substr(1); + co_return serde_raw{std::move(merged)}; + } + } + } + + co_return refs; }); // ========================================================================= @@ -897,6 +1516,343 @@ void MasterServer::register_handlers() { params.text_document_position_params.text_document.uri, params.text_document_position_params.position); }); + + // ========================================================================= + // Hierarchy and workspace symbol handlers (index-based) + // ========================================================================= + + // --- textDocument/prepareCallHierarchy --- + peer.on_request([this](RequestContext& ctx, + const protocol::CallHierarchyPrepareParams& params) -> RawResult { + auto info = co_await lookup_symbol_at_position( + params.text_document_position_params.text_document.uri, + params.text_document_position_params.position); + if(!info) + co_return serde_raw{"null"}; + + // Only functions/methods are valid for call hierarchy. + if(!(info->kind == SymbolKind::Function || info->kind == SymbolKind::Method)) + co_return serde_raw{"null"}; + + std::vector items; + items.push_back(build_call_hierarchy_item(*info)); + co_return to_raw(items); + }); + + // --- callHierarchy/incomingCalls --- + peer.on_request([this](RequestContext& ctx, + const protocol::CallHierarchyIncomingCallsParams& params) -> RawResult { + // Re-lookup the symbol from the item. + auto info = + co_await resolve_hierarchy_item(params.item.uri, params.item.range, params.item.data); + if(!info) + co_return serde_raw{"null"}; + + auto sym_it = project_index.symbols.find(info->hash); + if(sym_it == project_index.symbols.end()) + co_return serde_raw{"null"}; + + // Collect all Caller relations across reference files. + // Caller relation: on the callee symbol, target_symbol is the caller's hash. + std::vector results; + + // Group call sites by caller symbol hash. + llvm::DenseMap> caller_ranges; + + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + // Use stored content from MergedIndex for offset-to-position conversion. + auto file_content = file_merged_it->second.content(); + if(file_content.empty()) + continue; + + lsp::PositionMapper file_mapper(file_content, lsp::PositionEncoding::UTF16); + + file_merged_it->second.lookup(info->hash, + RelationKind::Caller, + [&](const index::Relation& r) { + auto start = file_mapper.to_position(r.range.begin); + auto end = file_mapper.to_position(r.range.end); + if(start && end) { + caller_ranges[r.target_symbol].push_back( + protocol::Range{*start, *end}); + } + return true; + }); + } + + // Build incoming call items from grouped caller symbols. + for(auto& [caller_hash, ranges]: caller_ranges) { + auto def_loc = find_symbol_definition_location(caller_hash); + auto caller_sym_it = project_index.symbols.find(caller_hash); + if(caller_sym_it == project_index.symbols.end()) + continue; + + if(!def_loc) + continue; + + protocol::CallHierarchyItem caller_item; + caller_item.name = caller_sym_it->second.name; + caller_item.kind = to_lsp_symbol_kind(caller_sym_it->second.kind); + caller_item.uri = def_loc->uri; + caller_item.range = def_loc->range; + caller_item.selection_range = def_loc->range; + caller_item.data = protocol::LSPAny(static_cast(caller_hash)); + + protocol::CallHierarchyIncomingCall call; + call.from = std::move(caller_item); + call.from_ranges = std::move(ranges); + results.push_back(std::move(call)); + } + + if(results.empty()) + co_return serde_raw{"null"}; + co_return to_raw(results); + }); + + // --- callHierarchy/outgoingCalls --- + peer.on_request([this](RequestContext& ctx, + const protocol::CallHierarchyOutgoingCallsParams& params) -> RawResult { + auto info = + co_await resolve_hierarchy_item(params.item.uri, params.item.range, params.item.data); + if(!info) + co_return serde_raw{"null"}; + + auto sym_it = project_index.symbols.find(info->hash); + if(sym_it == project_index.symbols.end()) + co_return serde_raw{"null"}; + + // Collect Callee relations (outgoing calls from this function). + // Group call sites by callee symbol hash. + llvm::DenseMap> callee_ranges; + + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + // Use stored content from MergedIndex for offset-to-position conversion. + auto file_content = file_merged_it->second.content(); + if(file_content.empty()) + continue; + + lsp::PositionMapper file_mapper(file_content, lsp::PositionEncoding::UTF16); + + file_merged_it->second.lookup(info->hash, + RelationKind::Callee, + [&](const index::Relation& r) { + auto start = file_mapper.to_position(r.range.begin); + auto end = file_mapper.to_position(r.range.end); + if(start && end) { + callee_ranges[r.target_symbol].push_back( + protocol::Range{*start, *end}); + } + return true; + }); + } + + std::vector results; + for(auto& [callee_hash, ranges]: callee_ranges) { + auto def_loc = find_symbol_definition_location(callee_hash); + auto callee_sym_it = project_index.symbols.find(callee_hash); + if(callee_sym_it == project_index.symbols.end()) + continue; + + if(!def_loc) + continue; + + protocol::CallHierarchyItem callee_item; + callee_item.name = callee_sym_it->second.name; + callee_item.kind = to_lsp_symbol_kind(callee_sym_it->second.kind); + callee_item.uri = def_loc->uri; + callee_item.range = def_loc->range; + callee_item.selection_range = def_loc->range; + callee_item.data = protocol::LSPAny(static_cast(callee_hash)); + + protocol::CallHierarchyOutgoingCall call; + call.to = std::move(callee_item); + call.from_ranges = std::move(ranges); + results.push_back(std::move(call)); + } + + if(results.empty()) + co_return serde_raw{"null"}; + co_return to_raw(results); + }); + + // --- textDocument/prepareTypeHierarchy --- + peer.on_request([this](RequestContext& ctx, + const protocol::TypeHierarchyPrepareParams& params) -> RawResult { + auto info = co_await lookup_symbol_at_position( + params.text_document_position_params.text_document.uri, + params.text_document_position_params.position); + if(!info) + co_return serde_raw{"null"}; + + // Only class-like types are valid for type hierarchy. + if(!(info->kind == SymbolKind::Class || info->kind == SymbolKind::Struct || + info->kind == SymbolKind::Enum || info->kind == SymbolKind::Union)) + co_return serde_raw{"null"}; + + std::vector items; + items.push_back(build_type_hierarchy_item(*info)); + co_return to_raw(items); + }); + + // --- typeHierarchy/supertypes --- + peer.on_request([this](RequestContext& ctx, + const protocol::TypeHierarchySupertypesParams& params) -> RawResult { + auto info = + co_await resolve_hierarchy_item(params.item.uri, params.item.range, params.item.data); + if(!info) + co_return serde_raw{"null"}; + + auto sym_it = project_index.symbols.find(info->hash); + if(sym_it == project_index.symbols.end()) + co_return serde_raw{"null"}; + + // Find Base relations: supertypes of this class. + std::vector results; + + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + file_merged_it->second.lookup( + info->hash, + RelationKind::Base, + [&](const index::Relation& r) { + auto base_hash = r.target_symbol; + auto base_sym_it = project_index.symbols.find(base_hash); + if(base_sym_it == project_index.symbols.end()) + return true; + + // Find the definition location of the base class. + auto def_loc = find_symbol_definition_location(base_hash); + if(!def_loc) + return true; + + protocol::TypeHierarchyItem item; + item.name = base_sym_it->second.name; + item.kind = to_lsp_symbol_kind(base_sym_it->second.kind); + item.uri = def_loc->uri; + item.range = def_loc->range; + item.selection_range = def_loc->range; + item.data = protocol::LSPAny(static_cast(base_hash)); + results.push_back(std::move(item)); + return true; + }); + } + + if(results.empty()) + co_return serde_raw{"null"}; + co_return to_raw(results); + }); + + // --- typeHierarchy/subtypes --- + peer.on_request([this](RequestContext& ctx, + const protocol::TypeHierarchySubtypesParams& params) -> RawResult { + auto info = + co_await resolve_hierarchy_item(params.item.uri, params.item.range, params.item.data); + if(!info) + co_return serde_raw{"null"}; + + auto sym_it = project_index.symbols.find(info->hash); + if(sym_it == project_index.symbols.end()) + co_return serde_raw{"null"}; + + // Find Derived relations across all reference files: subtypes of this class. + std::vector results; + + for(auto file_id: sym_it->second.reference_files) { + auto file_merged_it = merged_indices.find(file_id); + if(file_merged_it == merged_indices.end()) + continue; + + file_merged_it->second.lookup( + info->hash, + RelationKind::Derived, + [&](const index::Relation& r) { + auto derived_hash = r.target_symbol; + auto derived_sym_it = project_index.symbols.find(derived_hash); + if(derived_sym_it == project_index.symbols.end()) + return true; + + auto def_loc = find_symbol_definition_location(derived_hash); + if(!def_loc) + return true; + + protocol::TypeHierarchyItem item; + item.name = derived_sym_it->second.name; + item.kind = to_lsp_symbol_kind(derived_sym_it->second.kind); + item.uri = def_loc->uri; + item.range = def_loc->range; + item.selection_range = def_loc->range; + item.data = protocol::LSPAny(static_cast(derived_hash)); + results.push_back(std::move(item)); + return true; + }); + } + + if(results.empty()) + co_return serde_raw{"null"}; + co_return to_raw(results); + }); + + // --- workspace/symbol --- + peer.on_request( + [this](RequestContext& ctx, const protocol::WorkspaceSymbolParams& params) -> RawResult { + auto query = llvm::StringRef(params.query); + std::vector results; + + // Case-insensitive substring match on symbol names. + std::string query_lower = query.lower(); + + for(auto& [hash, symbol]: project_index.symbols) { + if(results.size() >= 100) + break; + + // Skip non-indexable symbol kinds (punctuation, literals, etc.) + // Skip non-indexable symbol kinds using a positive check instead. + auto sk = symbol.kind; + if(!(sk == SymbolKind::Namespace || sk == SymbolKind::Class || + sk == SymbolKind::Struct || sk == SymbolKind::Union || + sk == SymbolKind::Enum || sk == SymbolKind::Type || sk == SymbolKind::Field || + sk == SymbolKind::EnumMember || sk == SymbolKind::Function || + sk == SymbolKind::Method || sk == SymbolKind::Variable || + sk == SymbolKind::Parameter || sk == SymbolKind::Macro || + sk == SymbolKind::Concept || sk == SymbolKind::Module || + sk == SymbolKind::Operator || sk == SymbolKind::MacroParameter || + sk == SymbolKind::Label || sk == SymbolKind::Attribute)) + continue; + + if(symbol.name.empty()) + continue; + + // Check case-insensitive substring match. + std::string name_lower = llvm::StringRef(symbol.name).lower(); + if(!query_lower.empty() && name_lower.find(query_lower) == std::string::npos) + continue; + + auto def_loc = find_symbol_definition_location(hash); + if(!def_loc) + continue; + + protocol::SymbolInformation info; + info.name = symbol.name; + info.kind = to_lsp_symbol_kind(symbol.kind); + info.location = std::move(*def_loc); + results.push_back(std::move(info)); + } + + if(results.empty()) + co_return serde_raw{"null"}; + co_return to_raw(results); + }); } } // namespace clice diff --git a/src/server/master_server.h b/src/server/master_server.h index 89f24f00..c8a1aaf7 100644 --- a/src/server/master_server.h +++ b/src/server/master_server.h @@ -9,6 +9,9 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/peer.h" #include "eventide/serde/serde/raw_value.h" +#include "index/merged_index.h" +#include "index/project_index.h" +#include "semantic/relation_kind.h" #include "server/compile_graph.h" #include "server/config.h" #include "server/worker_pool.h" @@ -84,6 +87,29 @@ private: // path_id -> in-flight PCH build event (later arrivals co_await the same build). llvm::DenseMap> pch_building; + // === Index state === + + // Global symbol table and path mapping for the project. + index::ProjectIndex project_index; + + // Per-file merged index shards (keyed by project-level path_id). + llvm::DenseMap merged_indices; + + // Files queued for background indexing (server-level path_ids from CDB). + std::vector index_queue; + + // Index of next file to process in index_queue. + std::size_t index_queue_pos = 0; + + // Whether background indexing is currently in progress. + bool indexing_active = false; + + // Whether a background indexing coroutine has been scheduled (waiting on timer). + bool indexing_scheduled = false; + + // Timer for idle-triggered background indexing. + std::shared_ptr index_idle_timer; + // Document state: path_id -> DocumentState llvm::DenseMap documents; @@ -123,6 +149,21 @@ private: const std::string& directory, const std::vector& arguments); + // Schedule background indexing when idle. + void schedule_indexing(); + + // Background indexing coroutine: picks files from queue and dispatches to workers. + et::task<> run_background_indexing(); + + // Merge a TUIndex result into ProjectIndex and MergedIndex shards. + void merge_index_result(const void* tu_index_data, std::size_t size); + + // Persist index state to disk. + void save_index(); + + // Load index state from disk. + void load_index(); + // Forwarding helpers for feature requests (RawValue passthrough) using RawResult = et::task; @@ -137,6 +178,44 @@ private: /// Forward a stateless request with document content and compile args. template RawResult forward_stateless(const std::string& uri, const protocol::Position& position); + + /// Query index for symbol relations (GoToDefinition, FindReferences, etc.). + /// Returns LSP Location array as RawValue. + RawResult query_index_relations(const std::string& uri, + const protocol::Position& position, + RelationKind kind); + + /// Information about a symbol at a given position. + struct SymbolInfo { + index::SymbolHash hash = 0; + std::string name; + SymbolKind kind; + std::string uri; + protocol::Range range; + }; + + /// Look up a symbol at a position, returning its hash, name, kind, and range. + et::task> + lookup_symbol_at_position(const std::string& uri, const protocol::Position& position); + + /// Find the definition location (uri + range) of a symbol by its hash. + std::optional find_symbol_definition_location(index::SymbolHash hash); + + /// Convert clice::SymbolKind to LSP protocol::SymbolKind. + static protocol::SymbolKind to_lsp_symbol_kind(SymbolKind kind); + + /// Build a CallHierarchyItem from a SymbolInfo. + protocol::CallHierarchyItem build_call_hierarchy_item(const SymbolInfo& info); + + /// Build a TypeHierarchyItem from a SymbolInfo. + protocol::TypeHierarchyItem build_type_hierarchy_item(const SymbolInfo& info); + + /// Resolve SymbolInfo from a hierarchy item's stored data (symbol hash). + /// Falls back to position-based lookup if data is missing. + et::task> + resolve_hierarchy_item(const std::string& uri, + const protocol::Range& range, + const std::optional& data); }; } // namespace clice diff --git a/src/server/stateless_worker.cpp b/src/server/stateless_worker.cpp index ce2a3263..f11dded7 100644 --- a/src/server/stateless_worker.cpp +++ b/src/server/stateless_worker.cpp @@ -10,9 +10,12 @@ #include "eventide/serde/json/serializer.h" #include "eventide/serde/serde/raw_value.h" #include "feature/feature.h" +#include "index/tu_index.h" #include "server/protocol.h" #include "support/logging.h" +#include "llvm/Support/raw_ostream.h" + namespace clice { namespace et = eventide; @@ -212,9 +215,17 @@ int run_stateless_worker_mode() { return {false, "Index compilation failed", ""}; } - LOG_INFO("Index done: file={}, {}ms", params.file, timer.ms()); - // TODO: Generate TUIndex from the compilation unit - return {true, "", ""}; + auto tu_index = index::TUIndex::build(unit); + + std::string serialized; + llvm::raw_string_ostream os(serialized); + tu_index.serialize(os); + + LOG_INFO("Index done: file={}, {} symbols, {}ms", + params.file, + tu_index.symbols.size(), + timer.ms()); + return {true, "", std::move(serialized)}; }); co_return result.value(); }); diff --git a/src/support/path_pool.h b/src/support/path_pool.h index 81f6c887..0e8a5ec2 100644 --- a/src/support/path_pool.h +++ b/src/support/path_pool.h @@ -4,6 +4,7 @@ #include #include +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" @@ -18,6 +19,17 @@ struct PathPool { llvm::StringMap cache; std::uint32_t intern(llvm::StringRef path) { + // Normalize backslashes to forward slashes so that paths from different + // sources (URI decoding, CDB, include resolution) compare equal on + // Windows where native separators are backslashes. + llvm::SmallString<256> normalized; + bool needs_normalize = path.contains('\\'); + if(needs_normalize) { + normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + path = normalized; + } + auto [it, inserted] = cache.try_emplace(path, paths.size()); if(inserted) { // Allocate with null terminator so that resolve().data() is safe diff --git a/tests/conftest.py b/tests/conftest.py index 45db7c5b..0afd9a5c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,7 +108,8 @@ class CliceClient(BaseLanguageClient): def open(self, filepath: Path, version: int = 0) -> tuple[str, str]: """Open a text document and return (uri, content).""" - content = filepath.read_text(encoding="utf-8") + # Read in binary mode to preserve CRLF on Windows, matching real LSP clients. + content = filepath.read_bytes().decode("utf-8") uri = filepath.as_uri() self.text_document_did_open( DidOpenTextDocumentParams( @@ -227,6 +228,10 @@ def workspace(request: pytest.FixtureRequest, test_data_dir: Path) -> Path | Non path = test_data_dir / marker.args[0] if (path / "CMakeLists.txt").exists(): generate_cdb(path) + # Clean up persisted index/cache so each test starts fresh. + clice_dir = path / ".clice" + if clice_dir.exists(): + shutil.rmtree(clice_dir) return path @@ -267,6 +272,18 @@ async def client( if hasattr(c, "_server") and c._server is not None and c._server.returncode is None: c._server.kill() + # Dump server stderr warnings for diagnostics. + try: + server = getattr(c, "_server", None) + if server and server.stderr: + stderr_data = await asyncio.wait_for(server.stderr.read(), timeout=2.0) + if stderr_data: + for line in stderr_data.decode("utf-8", errors="replace").splitlines(): + if "[warn]" in line or "[error]" in line: + print(f"[server] {line}", flush=True) + except Exception: + pass + # Stop pygls client (with timeout to avoid hanging) try: c._stop_event.set() diff --git a/tests/data/index_features/CMakeLists.txt b/tests/data/index_features/CMakeLists.txt new file mode 100644 index 00000000..538e335d --- /dev/null +++ b/tests/data/index_features/CMakeLists.txt @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.20) +project(index_features CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +add_executable(main main.cpp) diff --git a/tests/data/index_features/main.cpp b/tests/data/index_features/main.cpp new file mode 100644 index 00000000..963cb83f --- /dev/null +++ b/tests/data/index_features/main.cpp @@ -0,0 +1,47 @@ +// Base class for type hierarchy +struct Animal { + virtual void speak() {} + + virtual ~Animal() = default; +}; + +// Derived class for type hierarchy +struct Dog : public Animal { + void speak() override {} +}; + +// Another derived class +struct Cat : public Animal { + void speak() override {} +}; + +// Free function for call hierarchy +int add(int a, int b) { + return a + b; +} + +// Caller function +int compute() { + int x = add(1, 2); + int y = add(3, 4); + return x + y; +} + +// For find references +int global_var = 42; + +int use_global() { + return global_var + 1; +} + +int use_global_again() { + return global_var * 2; +} + +int main() { + Dog d; + d.speak(); + Cat c; + c.speak(); + return compute() + use_global() + use_global_again(); +} diff --git a/tests/integration/test_index.py b/tests/integration/test_index.py new file mode 100644 index 00000000..50953a41 --- /dev/null +++ b/tests/integration/test_index.py @@ -0,0 +1,282 @@ +"""Integration tests for index-based LSP features: GoToDefinition, FindReferences, +CallHierarchy, TypeHierarchy, and WorkspaceSymbol.""" + +import asyncio + +import pytest +from lsprotocol.types import ( + CallHierarchyIncomingCallsParams, + CallHierarchyOutgoingCallsParams, + CallHierarchyPrepareParams, + DefinitionParams, + DidCloseTextDocumentParams, + Position, + ReferenceContext, + ReferenceParams, + TextDocumentIdentifier, + TypeHierarchyPrepareParams, + TypeHierarchySubtypesParams, + TypeHierarchySupertypesParams, + WorkspaceSymbolParams, +) + + +def _doc(uri: str) -> TextDocumentIdentifier: + return TextDocumentIdentifier(uri=uri) + + +async def _wait_for_index(client, timeout=30): + """Poll workspace/symbol until indexing is ready (symbols appear).""" + for _ in range(timeout): + result = await client.workspace_symbol_async(WorkspaceSymbolParams(query="add")) + if result and any(s.name == "add" for s in result): + return True + await asyncio.sleep(1) + return False + + +# --------------------------------------------------------------------------- +# GoToDefinition +# --------------------------------------------------------------------------- + + +@pytest.mark.workspace("index_features") +async def test_goto_definition(client, workspace): + """Test GoToDefinition navigates from a call site to the function definition.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # 'add' call on line 24 (0-indexed), column 12 + result = await client.text_document_definition_async( + DefinitionParams( + text_document=_doc(uri), + position=Position(line=24, character=12), + ) + ) + assert result is not None + locs = result if isinstance(result, list) else [result] + assert len(locs) > 0, f"GoToDefinition returned empty list, result={result}" + # Definition should point to line 18 where 'int add(...)' is declared + assert any(loc.range.start.line == 18 for loc in locs), ( + f"Expected line 18, got locations:" + f" {[(loc.uri, loc.range.start.line, loc.range.start.character) for loc in locs]}" + ) + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +# --------------------------------------------------------------------------- +# FindReferences +# --------------------------------------------------------------------------- + + +@pytest.mark.workspace("index_features") +async def test_find_references(client, workspace): + """Test FindReferences returns all usages of global_var.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # global_var definition on line 30 (0-indexed), column 4 + result = await client.text_document_references_async( + ReferenceParams( + text_document=_doc(uri), + position=Position(line=30, character=4), + context=ReferenceContext(include_declaration=True), + ) + ) + assert result is not None, "FindReferences returned None" + # global_var is declared on line 30 and used on lines 33 and 37 + assert len(result) >= 3, ( + f"Expected >=3 refs, got {len(result)}:" + f" {[(r.uri, r.range.start.line) for r in result]}" + ) + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +# --------------------------------------------------------------------------- +# CallHierarchy +# --------------------------------------------------------------------------- + + +@pytest.mark.workspace("index_features") +async def test_call_hierarchy_prepare(client, workspace): + """Test prepareCallHierarchy returns a CallHierarchyItem for 'add'.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # 'add' definition at line 18 (0-indexed), column 4 + result = await client.text_document_prepare_call_hierarchy_async( + CallHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=18, character=4), + ) + ) + assert result is not None, "prepareCallHierarchy returned None" + assert len(result) > 0, f"prepareCallHierarchy returned empty, result={result}" + assert result[0].name == "add", f"Expected 'add', got '{result[0].name}'" + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +@pytest.mark.workspace("index_features") +async def test_call_hierarchy_incoming(client, workspace): + """Test incomingCalls shows compute() calls add().""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # Prepare call hierarchy for 'add' at line 18 (0-indexed), column 4 + items = await client.text_document_prepare_call_hierarchy_async( + CallHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=18, character=4), + ) + ) + assert items and len(items) > 0, f"prepareCallHierarchy returned {items}" + + incoming = await client.call_hierarchy_incoming_calls_async( + CallHierarchyIncomingCallsParams(item=items[0]) + ) + assert incoming is not None, "incomingCalls returned None" + caller_names = [call.from_.name for call in incoming] + assert "compute" in caller_names, ( + f"Expected 'compute' in callers, got {caller_names}" + ) + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +@pytest.mark.workspace("index_features") +async def test_call_hierarchy_outgoing(client, workspace): + """Test outgoingCalls shows compute() calls add().""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # Prepare call hierarchy for 'compute' at line 23 (0-indexed), column 4 + items = await client.text_document_prepare_call_hierarchy_async( + CallHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=23, character=4), + ) + ) + assert items and len(items) > 0, f"prepareCallHierarchy returned {items}" + + outgoing = await client.call_hierarchy_outgoing_calls_async( + CallHierarchyOutgoingCallsParams(item=items[0]) + ) + assert outgoing is not None, "outgoingCalls returned None" + callee_names = [call.to.name for call in outgoing] + assert "add" in callee_names, f"Expected 'add' in callees, got {callee_names}" + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +# --------------------------------------------------------------------------- +# TypeHierarchy +# --------------------------------------------------------------------------- + + +@pytest.mark.workspace("index_features") +async def test_type_hierarchy_prepare(client, workspace): + """Test prepareTypeHierarchy returns a TypeHierarchyItem for 'Dog'.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # 'Dog' at line 8 (0-indexed), column 7 + result = await client.text_document_prepare_type_hierarchy_async( + TypeHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=8, character=7), + ) + ) + assert result is not None, "prepareTypeHierarchy returned None" + assert len(result) > 0, f"prepareTypeHierarchy returned empty" + assert result[0].name == "Dog", f"Expected 'Dog', got '{result[0].name}'" + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +@pytest.mark.workspace("index_features") +async def test_type_hierarchy_supertypes(client, workspace): + """Test supertypes of Dog includes Animal.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # 'Dog' at line 8 (0-indexed), column 7 + items = await client.text_document_prepare_type_hierarchy_async( + TypeHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=8, character=7), + ) + ) + assert items and len(items) > 0, f"prepareTypeHierarchy returned {items}" + + supertypes = await client.type_hierarchy_supertypes_async( + TypeHierarchySupertypesParams(item=items[0]) + ) + assert supertypes is not None, "supertypes returned None" + supertype_names = [t.name for t in supertypes] + assert "Animal" in supertype_names, ( + f"Expected 'Animal' in supertypes, got {supertype_names}" + ) + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +@pytest.mark.workspace("index_features") +async def test_type_hierarchy_subtypes(client, workspace): + """Test subtypes of Animal includes Dog and Cat.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + # 'Animal' at line 1, column 7 + items = await client.text_document_prepare_type_hierarchy_async( + TypeHierarchyPrepareParams( + text_document=_doc(uri), + position=Position(line=1, character=7), + ) + ) + assert items and len(items) > 0, f"prepareTypeHierarchy returned {items}" + + subtypes = await client.type_hierarchy_subtypes_async( + TypeHierarchySubtypesParams(item=items[0]) + ) + assert subtypes is not None, "subtypes returned None" + subtype_names = [t.name for t in subtypes] + assert "Dog" in subtype_names, f"Expected 'Dog' in subtypes, got {subtype_names}" + assert "Cat" in subtype_names, f"Expected 'Cat' in subtypes, got {subtype_names}" + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +# --------------------------------------------------------------------------- +# WorkspaceSymbol +# --------------------------------------------------------------------------- + + +@pytest.mark.workspace("index_features") +async def test_workspace_symbol(client, workspace): + """Test workspace/symbol finds symbols by query string.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + result = await client.workspace_symbol_async(WorkspaceSymbolParams(query="add")) + assert result is not None + names = [s.name for s in result] + assert "add" in names + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) + + +@pytest.mark.workspace("index_features") +async def test_workspace_symbol_class(client, workspace): + """Test workspace/symbol finds class symbols.""" + uri, _ = await client.open_and_wait(workspace / "main.cpp") + assert await _wait_for_index(client), "Index not ready after 30s" + + result = await client.workspace_symbol_async(WorkspaceSymbolParams(query="Animal")) + assert result is not None + names = [s.name for s in result] + assert "Animal" in names + + client.text_document_did_close(DidCloseTextDocumentParams(text_document=_doc(uri))) diff --git a/tests/unit/index/index_query_tests.cpp b/tests/unit/index/index_query_tests.cpp new file mode 100644 index 00000000..703a6851 --- /dev/null +++ b/tests/unit/index/index_query_tests.cpp @@ -0,0 +1,370 @@ +#include "test/test.h" +#include "test/tester.h" +#include "index/merged_index.h" +#include "index/project_index.h" +#include "index/tu_index.h" + +namespace clice::testing { +namespace { + +TEST_SUITE(IndexQuery, Tester) { + +index::ProjectIndex project_index; +llvm::DenseMap merged_indices; + +/// Build TUIndex from code and merge into ProjectIndex + MergedIndex shards. +void build_and_merge(llvm::StringRef code, + std::source_location location = std::source_location::current()) { + add_main("main.cpp", code); + ASSERT_TRUE(compile()); + + auto tu_index = index::TUIndex::build(*unit); + auto file_ids_map = project_index.merge(tu_index); + + // Merge main file index as compilation context. + auto main_tu_path_id = static_cast(tu_index.graph.paths.size() - 1); + auto main_global_id = file_ids_map[main_tu_path_id]; + + std::vector include_locs; + for(auto& loc: tu_index.graph.locations) { + index::IncludeLocation remapped = loc; + remapped.path_id = file_ids_map[loc.path_id]; + include_locs.push_back(remapped); + } + + merged_indices[main_global_id].merge(main_global_id, + tu_index.built_at, + std::move(include_locs), + tu_index.main_file_index, + {}); + + // Merge header file indices. + for(auto& [fid, file_idx]: tu_index.file_indices) { + auto tu_pid = tu_index.graph.path_id(fid); + auto global_pid = file_ids_map[tu_pid]; + auto include_id = tu_index.graph.include_location_id(fid); + merged_indices[global_pid].merge(global_pid, include_id, file_idx, {}); + } +} + +/// Reset index state between test cases. +void reset() { + project_index = index::ProjectIndex(); + merged_indices.clear(); + clear(); +} + +/// Lookup the symbol hash at a given annotation offset in any merged index. +index::SymbolHash lookup_symbol(llvm::StringRef pos) { + auto offset = point(pos); + index::SymbolHash result = 0; + for(auto& [path_id, merged]: merged_indices) { + merged.lookup(offset, [&](const index::Occurrence& o) { + if(o.range.contains(offset)) { + result = o.target; + return false; + } + return true; + }); + if(result != 0) + break; + } + return result; +} + +/// Find all relations of a given kind for a symbol across all merged indices. +std::vector find_relations(index::SymbolHash symbol, RelationKind kind) { + std::vector results; + + auto sym_it = project_index.symbols.find(symbol); + if(sym_it == project_index.symbols.end()) + return results; + + // Search every shard that references this symbol. + for(auto file_id: sym_it->second.reference_files) { + auto it = merged_indices.find(file_id); + if(it == merged_indices.end()) + continue; + + it->second.lookup(symbol, kind, [&](const index::Relation& r) { + results.push_back(r); + return true; + }); + } + + // Also search all shards (symbol may appear in files not tracked by reference_files). + if(results.empty()) { + for(auto& [pid, merged]: merged_indices) { + merged.lookup(symbol, kind, [&](const index::Relation& r) { + results.push_back(r); + return true; + }); + } + } + + return results; +} + +// ============================================================ +// Test cases +// ============================================================ + +TEST_CASE(GoToDefinition) { + reset(); + build_and_merge(R"( + int $(decl)foo(); + + int @def[$(def)foo]() { return 42; } + + int main() { + return $(use)foo(); + } + )"); + + auto hash = lookup_symbol("use"); + ASSERT_NE(hash, 0UL); + + auto defs = find_relations(hash, RelationKind::Definition); + ASSERT_FALSE(defs.empty()); + + auto expected = range("def"); + ASSERT_EQ(dump(defs.front().range), dump(expected)); +} + +TEST_CASE(FindReferences) { + reset(); + build_and_merge(R"( + int $(decl)foo(); + + int $(def)foo() { return 42; } + + int bar() { + return $(ref1)foo() + $(ref2)foo(); + } + )"); + + auto hash = lookup_symbol("decl"); + ASSERT_NE(hash, 0UL); + + auto refs = find_relations(hash, RelationKind::Reference); + ASSERT_GE(refs.size(), 2U); +} + +TEST_CASE(DeclAndDef) { + reset(); + build_and_merge(R"( + int $(decl)foo(); + int @def[$(def)foo]() { return 42; } + )"); + + auto hash = lookup_symbol("decl"); + ASSERT_NE(hash, 0UL); + + auto decls = find_relations(hash, RelationKind::Declaration); + ASSERT_FALSE(decls.empty()); + + auto defs = find_relations(hash, RelationKind::Definition); + ASSERT_FALSE(defs.empty()); + + auto expected_def = range("def"); + ASSERT_EQ(dump(defs.front().range), dump(expected_def)); +} + +TEST_CASE(CallerCallee) { + reset(); + build_and_merge(R"( + void $(callee_def)callee() {} + + void $(caller_def)caller() { + $(call_site)callee(); + } + )"); + + auto caller_hash = lookup_symbol("caller_def"); + ASSERT_NE(caller_hash, 0UL); + + auto callees = find_relations(caller_hash, RelationKind::Callee); + ASSERT_FALSE(callees.empty()); + + auto callee_hash = lookup_symbol("callee_def"); + ASSERT_NE(callee_hash, 0UL); + + auto callers = find_relations(callee_hash, RelationKind::Caller); + ASSERT_FALSE(callers.empty()); +} + +TEST_CASE(OverrideRelation) { + reset(); + build_and_merge(R"( + struct Base { + virtual void $(base_method)method() {} + }; + + struct Derived : Base { + void $(derived_method)method() override {} + }; + )"); + + // Derived::method should have Interface relation to Base::method. + auto derived_hash = lookup_symbol("derived_method"); + ASSERT_NE(derived_hash, 0UL); + + auto interfaces = find_relations(derived_hash, RelationKind::Interface); + ASSERT_FALSE(interfaces.empty()); + + // Base::method should have Implementation relation. + auto base_hash = lookup_symbol("base_method"); + ASSERT_NE(base_hash, 0UL); + + auto impls = find_relations(base_hash, RelationKind::Implementation); + ASSERT_FALSE(impls.empty()); +} + +TEST_CASE(BaseAndDerived) { + reset(); + build_and_merge(R"( + struct $(base_cls)Animal { + virtual void speak() {} + }; + + struct $(derived_cls)Dog : $(base_ref)Animal { + void speak() override {} + }; + )"); + + auto derived_hash = lookup_symbol("derived_cls"); + ASSERT_NE(derived_hash, 0UL); + + // Look for any Base relation in any shard. + bool found_base = false; + for(auto& [pid, merged]: merged_indices) { + merged.lookup(derived_hash, RelationKind::Base, [&](const index::Relation& r) { + found_base = true; + return false; + }); + } + ASSERT_TRUE(found_base); +} + +TEST_CASE(ClassTemplate) { + reset(); + build_and_merge(R"( + template + struct @primary[$(primary)foo] {}; + + $(use)foo x; + )"); + + auto hash = lookup_symbol("use"); + ASSERT_NE(hash, 0UL); + + auto defs = find_relations(hash, RelationKind::Definition); + ASSERT_FALSE(defs.empty()); +} + +TEST_CASE(SymbolKinds) { + reset(); + build_and_merge(R"( + struct $(cls)MyClass {}; + void $(func)myFunc() {} + int $(var)myVar = 0; + )"); + + auto cls_hash = lookup_symbol("cls"); + ASSERT_NE(cls_hash, 0UL); + ASSERT_TRUE(project_index.symbols.contains(cls_hash)); + ASSERT_EQ(project_index.symbols[cls_hash].kind.value(), SymbolKind(SymbolKind::Struct).value()); + + auto func_hash = lookup_symbol("func"); + ASSERT_NE(func_hash, 0UL); + ASSERT_TRUE(project_index.symbols.contains(func_hash)); + ASSERT_EQ(project_index.symbols[func_hash].kind.value(), + SymbolKind(SymbolKind::Function).value()); + + auto var_hash = lookup_symbol("var"); + ASSERT_NE(var_hash, 0UL); + ASSERT_TRUE(project_index.symbols.contains(var_hash)); + ASSERT_EQ(project_index.symbols[var_hash].kind.value(), + SymbolKind(SymbolKind::Variable).value()); +} + +TEST_CASE(ReferenceFiles) { + reset(); + build_and_merge(R"( + int $(target)target = 42; + int a = $(ref)target + 1; + )"); + + auto hash = lookup_symbol("target"); + ASSERT_NE(hash, 0UL); + + auto sym_it = project_index.symbols.find(hash); + ASSERT_TRUE(sym_it != project_index.symbols.end()); + + // reference_files should contain at least the main file. + ASSERT_FALSE(sym_it->second.reference_files.isEmpty()); +} + +TEST_CASE(CrossFileQuery) { + reset(); + + add_file("header.h", R"( + #pragma once + int $(hdr_decl)helper(); + )"); + add_main("main.cpp", R"( + #include "header.h" + + int main() { + return $(use_helper)helper(); + } + )"); + ASSERT_TRUE(compile()); + + auto tu_index = index::TUIndex::build(*unit); + auto file_ids_map = project_index.merge(tu_index); + + // Merge main file. + auto main_tu_path_id = static_cast(tu_index.graph.paths.size() - 1); + auto main_global_id = file_ids_map[main_tu_path_id]; + + std::vector include_locs; + for(auto& loc: tu_index.graph.locations) { + index::IncludeLocation remapped = loc; + remapped.path_id = file_ids_map[loc.path_id]; + include_locs.push_back(remapped); + } + merged_indices[main_global_id].merge(main_global_id, + tu_index.built_at, + std::move(include_locs), + tu_index.main_file_index, + {}); + + // Merge header file indices. + for(auto& [fid, file_idx]: tu_index.file_indices) { + auto tu_pid = tu_index.graph.path_id(fid); + auto global_pid = file_ids_map[tu_pid]; + auto include_id = tu_index.graph.include_location_id(fid); + merged_indices[global_pid].merge(global_pid, include_id, file_idx, {}); + } + + // Query: from usage in main.cpp, find the symbol via merged index. + auto use_offset = point("use_helper"); + index::SymbolHash helper_hash = 0; + merged_indices[main_global_id].lookup(use_offset, [&](const index::Occurrence& o) { + if(o.range.contains(use_offset)) { + helper_hash = o.target; + return false; + } + return true; + }); + ASSERT_NE(helper_hash, 0UL); + + // Find declaration across all shards -- should find it in header shard. + auto decls = find_relations(helper_hash, RelationKind::Declaration); + ASSERT_FALSE(decls.empty()); +} + +}; // TEST_SUITE(IndexQuery) +} // namespace +} // namespace clice::testing diff --git a/tests/unit/index/merged_index_tests.cpp b/tests/unit/index/merged_index_tests.cpp index 3fd13c3a..88bc1a5b 100644 --- a/tests/unit/index/merged_index_tests.cpp +++ b/tests/unit/index/merged_index_tests.cpp @@ -54,7 +54,7 @@ TEST_CASE(Serialization) { auto& graph = tu_index.graph; for(auto& [fid, index]: tu_index.file_indices) { llvm::StringRef path = graph.paths[graph.path_id(fid)]; - merged_indices[path].merge(0, graph.include_location_id(fid), index); + merged_indices[path].merge(0, graph.include_location_id(fid), index, {}); } for(auto& [path, merged]: merged_indices) { @@ -77,7 +77,7 @@ TEST_CASE(LookupByOffset) { // Merge the main file index into a MergedIndex. index::MergedIndex merged; auto fid = unit->interested_file(); - merged.merge(0, tu_index.graph.include_location_id(fid), tu_index.main_file_index); + merged.merge(0, tu_index.graph.include_location_id(fid), tu_index.main_file_index, {}); // Lookup at the reference offset should find an occurrence. auto ref_offset = point("ref"); @@ -99,7 +99,7 @@ TEST_CASE(LookupBySymbolAndKind) { index::MergedIndex merged; auto fid = unit->interested_file(); - merged.merge(0, tu_index.graph.include_location_id(fid), tu_index.main_file_index); + merged.merge(0, tu_index.graph.include_location_id(fid), tu_index.main_file_index, {}); // Find the target_func symbol hash via occurrence lookup. auto target_offset = point("target"); @@ -148,10 +148,10 @@ TEST_CASE(MultipleMergesDedup) { // Merge header indices from both TUs into same MergedIndex. index::MergedIndex merged_header; for(auto& [fid, file_index]: tu_a.file_indices) { - merged_header.merge(0, tu_a.graph.include_location_id(fid), file_index); + merged_header.merge(0, tu_a.graph.include_location_id(fid), file_index, {}); } for(auto& [fid, file_index]: tu_b.file_indices) { - merged_header.merge(1, tu_b.graph.include_location_id(fid), file_index); + merged_header.merge(1, tu_b.graph.include_location_id(fid), file_index, {}); } // Serialize and deserialize to verify dedup survives round-trip. @@ -173,7 +173,7 @@ TEST_CASE(SerializationRoundTripInMemory) { index::MergedIndex merged; auto fid = unit->interested_file(); auto include_id = tu_index.graph.include_location_id(fid); - merged.merge(0, include_id, tu_index.main_file_index); + merged.merge(0, include_id, tu_index.main_file_index, {}); // Serialize. llvm::SmallString<4096> buf; @@ -199,6 +199,164 @@ TEST_CASE(SerializationRoundTripInMemory) { ASSERT_TRUE(found); } +TEST_CASE(RemoveCompilationContext) { + build_index(R"( + int foo() { return 42; } + int bar() { return foo(); } + )"); + + // Merge as a compilation context (using the build_at overload). + index::MergedIndex merged; + auto fid = unit->interested_file(); + std::vector locations; + merged.merge(0, tu_index.built_at, std::move(locations), tu_index.main_file_index, {}); + + // Verify occurrence lookup works before remove. + bool found_before = false; + for(auto& occ: tu_index.main_file_index.occurrences) { + merged.lookup(occ.range.begin, [&](const index::Occurrence& o) { + found_before = true; + return false; + }); + if(found_before) + break; + } + ASSERT_TRUE(found_before); + + // Remove the compilation context. + merged.remove(0); + + // Serialize and verify the removed data round-trips. + llvm::SmallString<4096> buf; + llvm::raw_svector_ostream os(buf); + merged.serialize(os); + // Should not crash. + auto restored = index::MergedIndex(buf); +} + +TEST_CASE(RemoveHeaderContext) { + add_file("header.h", R"( + #pragma once + inline int shared() { return 1; } + )"); + add_main("main.cpp", R"( + #include "header.h" + int use() { return shared(); } + )"); + ASSERT_TRUE(compile()); + tu_index = index::TUIndex::build(*unit); + + // Merge header index as header context. + index::MergedIndex merged_header; + for(auto& [fid, file_index]: tu_index.file_indices) { + merged_header.merge(0, tu_index.graph.include_location_id(fid), file_index, {}); + } + + // Remove should not crash. + merged_header.remove(0); + + // Serialize after remove should work. + llvm::SmallString<4096> buf; + llvm::raw_svector_ostream os(buf); + merged_header.serialize(os); +} + +TEST_CASE(RemovedBitmapRoundTrip) { + build_index(R"( + int foo() { return 42; } + )"); + + // Merge as compilation context. + index::MergedIndex merged; + std::vector locations; + merged.merge(0, tu_index.built_at, std::move(locations), tu_index.main_file_index, {}); + + // Remove to populate the removed bitmap. + merged.remove(0); + + // Serialize. + llvm::SmallString<4096> buf; + llvm::raw_svector_ostream os(buf); + merged.serialize(os); + + // Deserialize and compare. + auto restored = index::MergedIndex(buf); + ASSERT_TRUE(merged == restored); +} + +TEST_CASE(LookupFiltersRemoved) { + build_index(R"( + int $(target)foo() { return 42; } + )"); + + // Merge as compilation context. + index::MergedIndex merged; + std::vector locations; + merged.merge(0, tu_index.built_at, std::move(locations), tu_index.main_file_index, {}); + + // Verify lookup finds something before removal. + auto offset = point("target"); + bool found_before = false; + merged.lookup(offset, [&](const index::Occurrence& occ) { + if(occ.range.contains(offset)) + found_before = true; + return true; + }); + ASSERT_TRUE(found_before); + + // Remove the compilation context. + merged.remove(0); + + // Verify lookup finds nothing after removal. + bool found_after = false; + merged.lookup(offset, [&](const index::Occurrence& occ) { + if(occ.range.contains(offset)) + found_after = true; + return true; + }); + ASSERT_FALSE(found_after); +} + +TEST_CASE(CacheInvalidatedAfterMerge) { + build_index(R"( + int $(first)foo() { return 42; } + )"); + + // Merge first TU as header context. + index::MergedIndex merged; + auto fid = unit->interested_file(); + merged.merge(0, tu_index.graph.include_location_id(fid), tu_index.main_file_index, {}); + + // Trigger cache build by doing a lookup. + auto first_offset = point("first"); + bool found_first = false; + merged.lookup(first_offset, [&](const index::Occurrence& occ) { + if(occ.range.contains(first_offset)) + found_first = true; + return true; + }); + ASSERT_TRUE(found_first); + + // Build a second TU with different content. + build_index(R"( + int $(second)bar() { return 99; } + )"); + + // Merge second TU. + auto fid2 = unit->interested_file(); + merged.merge(1, tu_index.graph.include_location_id(fid2), tu_index.main_file_index, {}); + + // Verify lookup finds the new occurrence (cache was invalidated). + auto second_offset = point("second"); + bool found_second = false; + merged.lookup(second_offset, [&](const index::Occurrence& occ) { + if(occ.range.contains(second_offset)) + found_second = true; + return true; + }); + ASSERT_TRUE(found_second); +} + }; // TEST_SUITE(MergedIndex) } // namespace } // namespace clice::testing diff --git a/tests/unit/index/project_index_tests.cpp b/tests/unit/index/project_index_tests.cpp index d6f2172f..eb9f3666 100644 --- a/tests/unit/index/project_index_tests.cpp +++ b/tests/unit/index/project_index_tests.cpp @@ -163,6 +163,43 @@ TEST_CASE(FileIdsMapCorrectness) { } } +TEST_CASE(NameSurvivesRoundTrip) { + index::TUIndex tu; + ASSERT_TRUE(build_and_index(R"( + int my_variable = 42; + void my_function() {} + )", + tu)); + + index::ProjectIndex project; + project.merge(tu); + + // Verify names are populated after merge. + bool found_var = false; + bool found_func = false; + for(auto& [hash, symbol]: project.symbols) { + if(symbol.name == "my_variable") + found_var = true; + if(symbol.name == "my_function") + found_func = true; + } + ASSERT_TRUE(found_var); + ASSERT_TRUE(found_func); + + // Serialize and deserialize. + llvm::SmallString<4096> buf; + llvm::raw_svector_ostream os(buf); + project.serialize(os); + auto restored = index::ProjectIndex::from(buf.data()); + + // Verify names survive round-trip. + for(auto& [hash, symbol]: project.symbols) { + ASSERT_TRUE(restored.symbols.contains(hash)); + ASSERT_EQ(restored.symbols[hash].name, symbol.name); + ASSERT_EQ(restored.symbols[hash].kind.value(), symbol.kind.value()); + } +} + }; // TEST_SUITE(ProjectIndex) } // namespace } // namespace clice::testing