diff --git a/include/Index/Contextual.h b/include/Index/Contextual.h new file mode 100644 index 00000000..53d55ceb --- /dev/null +++ b/include/Index/Contextual.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/BitVector.h" + +namespace clice::index { + +struct Contextual { + /// The actual element id, + std::uint32_t element_id; + + constexpr inline static std::uint32_t FLAG = (1ull << 31); + + static Contextual from(bool is_dependent, std::uint32_t offset) { + Contextual ctx; + ctx.element_id = offset; + if(!is_dependent) { + ctx.element_id |= FLAG; + } + return ctx; + } + + bool is_dependent() { + return (element_id & FLAG) == 0; + } + + std::uint32_t offset() { + return element_id & ~FLAG; + } +}; + +} // namespace clice::index diff --git a/include/Index/Contexts.h b/include/Index/HeaderIndex.h similarity index 56% rename from include/Index/Contexts.h rename to include/Index/HeaderIndex.h index 67dfdcdf..b8000729 100644 --- a/include/Index/Contexts.h +++ b/include/Index/HeaderIndex.h @@ -1,45 +1,11 @@ #pragma once -#include -#include -#include -#include +#include "RawIndex.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/ADT/BitVector.h" +namespace clice::index::memory { -namespace clice::index { - -struct Contextual { - /// The actual element id, - std::uint32_t element_id; - - constexpr inline static std::uint32_t FLAG = (1ull << 31); - - static Contextual from(bool is_dependent, std::uint32_t offset) { - Contextual ctx; - ctx.element_id = offset; - if(!is_dependent) { - ctx.element_id |= FLAG; - } - return ctx; - } - - bool is_dependent() { - return (element_id & FLAG) == 0; - } - - std::uint32_t offset() { - return element_id & ~FLAG; - } -}; - -/// A header context could be represented by file:include. -/// In the following context, hctx means "header context" and cctx means -/// "canonical context". So hcid is header context id and ccid is -/// canonical context id. -class Contexts { +/// HeaderIndex store extra information to merge raw index from different header contexts. +class HeaderIndex : public RawIndex { public: std::uint32_t file_count() { return header_contexts.size(); @@ -69,8 +35,6 @@ public: return map; } - - /// Get a new header context id. std::uint32_t alloc_hctx_id(); @@ -89,9 +53,34 @@ public: return id; } - void remove(this Contexts& self, llvm::StringRef path); + struct HeaderContext { + /// The include location id of this header context. + std::uint32_t include; + + /// The header context id of this header context. + std::uint32_t hctx_id; + + /// The canonical context id of this header context. + std::uint32_t cctx_id; + }; + + void remove(this HeaderIndex& self, llvm::StringRef path); + + HeaderContext add_context(llvm::StringRef path, std::uint32_t include) { + assert(!merged && ""); + auto& context = header_contexts[path].emplace_back(); + context.include = include; + context.cctx_id = alloc_cctx_id(); + context.hctx_id = alloc_hctx_id(); + return context; + } + + HeaderContext + merge(this HeaderIndex& self, llvm::StringRef path, std::uint32_t include, RawIndex& raw); public: + bool merged = false; + /// The max header context id. std::uint32_t max_hctx_id = 0; @@ -105,17 +94,6 @@ public: /// Same as above but for canonical context id. std::deque erased_cctx_ids; - struct HeaderContext { - /// The include location id of this header context. - std::uint32_t include; - - /// The header context id of this header context. - std::uint32_t hctx_id; - - /// The canonical context id of this header context. - std::uint32_t cctx_id; - }; - /// A map between source file path and its header contexts. llvm::StringMap> header_contexts; @@ -139,4 +117,68 @@ public: std::vector> independent_elem_states; }; -} // namespace clice::index +} // namespace clice::index::memory + +namespace llvm { + +template +unsigned dense_hash(const Ts&... ts) { + return llvm::DenseMapInfo>::getHashValue(std::tuple{ts...}); +} + +template <> +struct DenseMapInfo { + using R = clice::LocalSourceRange; + + inline static R getEmptyKey() { + return R(0, -1); + } + + inline static R getTombstoneKey() { + return R(-1, 0); + } + + static auto getHashValue(const R& r) { + return dense_hash(r.begin, r.end); + } + + static bool isEqual(const R& lhs, const R& rhs) { + return lhs == rhs; + } +}; + +template <> +struct DenseMapInfo { + using R = clice::index::memory::Relation; + + inline static R getEmptyKey() { + return R{ + .kind = clice::RelationKind(), + .range = clice::LocalSourceRange(0, 0), + .target_symbol = 0, + }; + } + + inline static R getTombstoneKey() { + return R{ + .kind = clice::RelationKind(), + .range = clice::LocalSourceRange(-1, -1), + .target_symbol = 0, + }; + } + + /// Contextual doen't take part in hashing and equality. + static auto getHashValue(const R& relation) { + return dense_hash(relation.kind.value(), + relation.range.begin, + relation.range.end, + relation.target_symbol); + } + + static bool isEqual(const R& lhs, const R& rhs) { + return lhs.kind == rhs.kind && lhs.range == rhs.range && + lhs.target_symbol == rhs.target_symbol; + } +}; + +} // namespace llvm diff --git a/include/Index/Index.h b/include/Index/Index.h index 81382c90..5a6035ac 100644 --- a/include/Index/Index.h +++ b/include/Index/Index.h @@ -1,29 +1,22 @@ #pragma once -#include "SymbolIndex.h" -#include "FeatureIndex.h" -#include "Async/FileSystem.h" +#include +#include +#include +#include -namespace clice::index { +#include "TUIndex.h" +#include "RawIndex.h" +#include "HeaderIndex.h" +#include "IncludeGraph.h" -struct Index { - std::optional> symbol; - llvm::XXH128_hash_t symbolHash = {0, 0}; +namespace clice::index::memory { - std::optional> feature; - llvm::XXH128_hash_t featureHash = {0, 0}; - - static Shared build(ASTInfo& AST); - - async::Task<> write(std::string path) { - if(symbol) { - co_await async::fs::write(path + ".sidx", symbol->data(), symbol->size()); - } - - if(feature) { - co_await async::fs::write(path + ".fidx", feature->data(), feature->size()); - } - } +struct Indices { + std::unique_ptr tu_index; + llvm::DenseMap> header_indices; }; -} // namespace clice::index +Indices index(ASTInfo& AST); + +} // namespace clice::index::memory diff --git a/include/Index/Index2.h b/include/Index/Index2.h deleted file mode 100644 index 6983955d..00000000 --- a/include/Index/Index2.h +++ /dev/null @@ -1,176 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "Shared.h" -#include "Contexts.h" -#include "AST/SymbolID.h" -#include "AST/SymbolKind.h" -#include "AST/RelationKind.h" -#include "AST/SourceCode.h" - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallVector.h" - -namespace clice::index::memory2 { - -using SymbolID = std::uint64_t; - -using SourceRange = LocalSourceRange; - -struct Relation { - Contextual ctx; - RelationKind kind; - - /// The range of this relation. - SourceRange range; - - union { - SymbolID target_symbol; - SourceRange definition_range; - }; -}; - -struct Symbol { - /// The symbol id. - SymbolID id; - - /// The symbol kind. - SymbolKind kind; - - /// Whether this symbol is not visible to other tu. - bool is_tu_local = false; - - /// Whether this symbol is defined in function scope. - bool is_function_local = false; - - /// The symbol name. - std::string name; - - /// All relations of this symbol. - llvm::DenseSet relations; -}; - -struct Occurrence { - Contextual ctx; - SymbolID target_symbol; -}; - -/// For most of symbol occurrence, it has only one corresponding symbol. -using OccurrenceGroup = llvm::SmallVector; - -class SymbolIndex : public Contexts { -public: - static index::Shared> build(ASTInfo& AST); - - Symbol& getSymbol(std::uint64_t symbol_id); - - HeaderContext add_context(llvm::StringRef path, std::uint32_t include) { - assert(!merged && ""); - auto& context = header_contexts[path].emplace_back(); - context.include = include; - context.cctx_id = alloc_cctx_id(); - context.hctx_id = alloc_hctx_id(); - return context; - } - - void addRelation(Symbol& symbol, Relation relation, bool is_dependent = true); - - void addOccurrence(LocalSourceRange range, - std::int64_t target_symbol, - bool is_dependent = true); - - HeaderContext merge(this SymbolIndex& self, SymbolIndex& other); - -private: - /// Merge another index into this. Most of header file is actually - /// self contained file and has only one canonical context. This - /// is a fast path for it. - HeaderContext quick_merge(this SymbolIndex& self, SymbolIndex& other); - - /// Merge another index into this, this could handle even though - /// another has multiple canonical context. But of course slow than - /// the fast path. - /// TODO: This function hasn't been implemented. - void slow_merge(this SymbolIndex& self, SymbolIndex& other); - -public: - /// Whether this has been merged with other files. - bool merged = false; - - /// All symbols in this index. - llvm::DenseMap symbols; - - /// All occurrences in this index. - llvm::DenseMap occurrences; -}; - -} // namespace clice::index::memory2 - -namespace llvm { - -template -unsigned dense_hash(const Ts&... ts) { - return llvm::DenseMapInfo>::getHashValue(std::tuple{ts...}); -} - -template <> -struct DenseMapInfo { - using R = clice::LocalSourceRange; - - inline static R getEmptyKey() { - return R(0, -1); - } - - inline static R getTombstoneKey() { - return R(-1, 0); - } - - static auto getHashValue(const R& r) { - return dense_hash(r.begin, r.end); - } - - static bool isEqual(const R& lhs, const R& rhs) { - return lhs == rhs; - } -}; - -template <> -struct DenseMapInfo { - using R = clice::index::memory2::Relation; - - inline static R getEmptyKey() { - return R{ - .kind = clice::RelationKind(), - .range = clice::LocalSourceRange(0, 0), - .target_symbol = 0, - }; - } - - inline static R getTombstoneKey() { - return R{ - .kind = clice::RelationKind(), - .range = clice::LocalSourceRange(-1, -1), - .target_symbol = 0, - }; - } - - /// Contextual doen't take part in hashing and equality. - static auto getHashValue(const R& relation) { - return dense_hash(relation.kind.value(), - relation.range.begin, - relation.range.end, - relation.target_symbol); - } - - static bool isEqual(const R& lhs, const R& rhs) { - return lhs.kind == rhs.kind && lhs.range == rhs.range && - lhs.target_symbol == rhs.target_symbol; - } -}; - -} // namespace llvm diff --git a/include/Index/RawIndex.h b/include/Index/RawIndex.h new file mode 100644 index 00000000..e34be33a --- /dev/null +++ b/include/Index/RawIndex.h @@ -0,0 +1,95 @@ +#pragma once + +#include "Contextual.h" +#include "AST/SymbolID.h" +#include "AST/SymbolKind.h" +#include "AST/RelationKind.h" +#include "AST/SourceCode.h" + +namespace clice::index::memory { + +using SymbolID = std::uint64_t; + +struct Relation { + /// The context of this relation. + Contextual ctx; + + /// The relation kind of this relation. + RelationKind kind; + + /// The range of this relation. + LocalSourceRange range; + + /// A field to store extra information depend on kind. + union { + SymbolID target_symbol; + LocalSourceRange definition_range; + }; +}; + +struct Symbol { + /// The symbol id. + SymbolID id; + + /// The symbol kind. + SymbolKind kind; + + /// Whether this symbol is not visible to other tu. + bool is_tu_local = false; + + /// Whether this symbol is defined in function scope. + bool is_function_local = false; + + /// The symbol name. + std::string name; + + /// All relations of this symbol. + llvm::DenseSet relations; +}; + +struct Occurrence { + /// The context of this occurrence. + Contextual ctx; + + /// The target symbol of this occurrence. + SymbolID target_symbol; +}; + +/// The raw index directly generated by indexing the source file. +struct RawIndex { +public: + Symbol& get_symbol(SymbolID ID) { + if(auto it = symbols.find(ID); it != symbols.end()) { + return it->second; + } + + /// If not found, create a new symbol and return it. + auto symbol_ref = symbols.size(); + auto [it, _] = symbols.try_emplace(ID, Symbol{.id = ID}); + return it->second; + } + + void add_relation(Symbol& symbol, Relation relation, bool is_dependent = true) { + relation.ctx = Contextual::from(is_dependent, 0); + symbol.relations.insert(relation); + } + + void add_occurrence(LocalSourceRange range, SymbolID target_symbol, bool is_dependent = true) { + occurrences[range].emplace_back(Contextual::from(is_dependent, 0), target_symbol); + } + +public: + /// The path of source file. + std::string path; + + /// The content of source file. + std::string content; + + /// All symbols in this index. + llvm::DenseMap symbols; + + /// All occurrences in this index. + llvm::DenseMap> occurrences; +}; + +} // namespace clice::index::memory diff --git a/include/Index/TUIndex.h b/include/Index/TUIndex.h new file mode 100644 index 00000000..25b0d237 --- /dev/null +++ b/include/Index/TUIndex.h @@ -0,0 +1,19 @@ +#pragma once + +#include "RawIndex.h" +#include "IncludeGraph.h" + +namespace clice::index::memory { + +class TUIndex : public RawIndex { +public: + +public: + /// The time of building this index. + std::int64_t time; + + /// The include graph of this index. + IncludeGraph graph; +}; + +} // namespace clice::index::memory diff --git a/include/Server/Config.h b/include/Server/Config.h index 386fcfbc..7d53fc8e 100644 --- a/include/Server/Config.h +++ b/include/Server/Config.h @@ -16,17 +16,16 @@ std::expected load(llvm::StringRef execute, llvm::StringRef f void init(std::string_view workplace); struct ServerOptions { - std::vector compile_commands_dirs; + std::vector compile_commands_dirs = {"${workspace}/build"}; }; struct CacheOptions { - std::string dir; + std::string dir = "${workspace}/.clice/cache"; uint32_t limit = 0; }; struct IndexOptions { - std::string dir; - bool implicitInstantiation = true; + std::string dir = "${workspace}/.clice/index"; }; struct Rule { diff --git a/include/Server/DocumentManager.h b/include/Server/DocumentManager.h deleted file mode 100644 index 3b063bd9..00000000 --- a/include/Server/DocumentManager.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include "llvm/ADT/StringMap.h" - -namespace clice { - -class Document {}; - -/// Responsible for all opened files. -class DocumentManager { -public: - -private: - /// TODO: Use an LRU for this. - llvm::StringMap documents; -}; - -} // namespace clice diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h index 6d3d8556..6048c867 100644 --- a/include/Server/Indexer.h +++ b/include/Server/Indexer.h @@ -1,116 +1,77 @@ #pragma once -#include "Config.h" -#include "IncludeGraph.h" +#include #include "Async/Async.h" +#include "AST/SymbolID.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringMap.h" #include "Compiler/Command.h" -#include "Index/FeatureIndex.h" -#include "llvm/ADT/StringSet.h" -#include "Feature/Lookup.h" +#include "Index/Index.h" namespace clice { -class Indexer : public IncludeGraph { +class ASTInfo; + +class Indexer { public: - Indexer(CompilationDatabase& database, const config::IndexOptions& options); + /// Index an opened file, its AST is already builtin + /// and PCH is used for it. + async::Task<> index(ASTInfo& AST); - /// Add a file to wait for indexing. - void add(std::string file); + /// Index an static file. + async::Task<> index(llvm::StringRef file); - /// Remove a file from indexing. - void remove(std::string file); - - void indexAll(); - - void save(); - - void load(); + using Path = std::string; + using PathID = std::uint32_t; + using SymbolID = std::uint64_t; public: - Header* getHeader(llvm::StringRef file) const; + Indexer(CompilationDatabase& database) : database(database) {} - TranslationUnit* getTranslationUnit(llvm::StringRef file) const; + PathID getPath(llvm::StringRef path) { + auto it = paths.find(path); + if(it != paths.end()) { + return it->second; + } - /// Return current header context of given header file. If the header - /// does't have an active context, the result will be invalid. - std::optional currentContext(llvm::StringRef header) const; + auto id = path_storage.size(); + path_storage.emplace_back(path); + paths.try_emplace(path, id); + return id; + } - /// Switch the context of the header to given context. If success, - /// return true. - bool switchContext(llvm::StringRef header, proto::HeaderContext context); +private: + struct HeaderIndices { + using RawIndex = std::pair>; - /// Resolve the given header context to a group of locations. - std::vector resolveContext(proto::HeaderContext context) const; + /// The merged index. + std::unique_ptr merged; - /// Return all header contexts of given header file, note that a header may have thousands - /// of header contexts, of course we won't return them all at once. We would return a group - /// of contexts for each different header context. The maximum of group count is determined - /// by limit. Optionally, you can specify a 'contextFile' to filter the results, returning only - /// contexts related to that file. - std::vector - allContexts(llvm::StringRef headerFile, - uint32_t limit = 10, - llvm::StringRef contextFile = llvm::StringRef()) const; - -public: - struct SymbolID { - uint64_t hash; - std::string name; + llvm::DenseMap> unmergeds; }; - /// Return all indices of the given translation unit. If the file is empty, - /// return all indices of the IncludeGraph. - std::vector indices(TranslationUnit* tu = nullptr); - - /// Resolve the symbol at the given position. - async::Task> resolve(const proto::TextDocumentPositionParams& params); - - // using LookupCallback = llvm::unique_function; - // - // async::Task<> lookup(llvm::ArrayRef targets, - // llvm::ArrayRef files, - // LookupCallback callback); - // - ///// Lookup the reference information according to the given position. - // async::Task lookup(const proto::ReferenceParams& params, - // RelationKind kind); - // - ///// According to the given file and offset, resolve the symbol at the offset. - // async::Task - // prepareHierarchy(const proto::HierarchyPrepareParams& params); - // - // async::Task - // incomingCalls(const proto::HierarchyParams& params); - // - // async::Task - // outgoingCalls(const proto::HierarchyParams& params); - // - // async::Task typeHierarchy(const proto::HierarchyParams& params, - // bool super); - -public: - async::Task> getFeatureIndex(std::string& buffer, - llvm::StringRef file) const; - - async::Task> semanticTokens(llvm::StringRef file) const; - - async::Task> foldingRanges(llvm::StringRef file) const; - - async::Task> documentLinks(llvm::StringRef file) const; - -private: - async::Task<> index(std::string file); - -private: CompilationDatabase& database; - const config::IndexOptions& options; - llvm::StringMap> tasks; - llvm::StringSet<> pending; + /// All paths of indices. + std::vector path_storage; - std::size_t concurrency = std::thread::hardware_concurrency(); + /// A map between path and its id. + llvm::StringMap paths; + + /// A map between symbol id and files that contains it. + llvm::DenseMap> symbol_indices; + + /// A map between source file path and its static indices. + llvm::DenseMap static_indices; + + std::uint32_t unmerged_count = 0; + + /// In-memory header indices. + llvm::DenseMap> dynamic_header_indices; + + /// In-memory translation unit indices. + llvm::DenseMap> dynamic_tu_indices; }; -} // namespace clice +} // namespace clice diff --git a/include/Server/Indexer2.h b/include/Server/Indexer2.h deleted file mode 100644 index f53a02cf..00000000 --- a/include/Server/Indexer2.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include -#include "AST/SymbolID.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" - -namespace clice { - -class Indexer { -public: - -private: - /// All paths in indexes. - std::vector paths; - - /// A map between path and its index. - llvm::StringMap pathIndex; - - /// A map between source file path and its static index file. - llvm::DenseMap mapToIndex; - - /// A map between symbol id and files that contains it. - llvm::DenseMap> invertedSymbolMap; -}; - -} // namespace clice diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index b9bc1c42..1e3581d8 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -1,5 +1,6 @@ #pragma once +#include "Indexer.h" #include "Async/Async.h" #include "Compiler/AST.h" #include "Compiler/Module.h" @@ -34,8 +35,8 @@ struct OpenFile { class Scheduler { public: - Scheduler(LSPConverter& converter, CompilationDatabase& database) : - converter(converter), database(database) {} + Scheduler(Indexer& indexer, LSPConverter& converter, CompilationDatabase& database) : + indexer(indexer), converter(converter), database(database) {} /// Add or update a document. void addDocument(std::string path, std::string content); @@ -58,6 +59,7 @@ private: async::Task<> buildAST(std::string file, std::string content); private: + Indexer& indexer; LSPConverter& converter; CompilationDatabase& database; diff --git a/src/Index/Contexts.cpp b/src/Index/Contexts.cpp deleted file mode 100644 index 14ef593c..00000000 --- a/src/Index/Contexts.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include "Index/Contexts.h" -#include "Support/Ranges.h" - -namespace clice::index { - -std::uint32_t Contexts::alloc_hctx_id() { - std::uint32_t new_hctx_id; - if(erased_hctx_ids.empty()) { - new_hctx_id = max_hctx_id; - max_hctx_id += 1; - } else { - new_hctx_id = erased_hctx_ids.front(); - erased_hctx_ids.pop_front(); - } - return new_hctx_id; -} - -std::uint32_t Contexts::alloc_cctx_id() { - std::uint32_t new_cctx_id; - if(erased_cctx_ids.empty()) { - new_cctx_id = max_cctx_id; - max_cctx_id += 1; - cctx_hctx_refs.emplace_back(1); - cctx_element_refs.emplace_back(0); - } else { - new_cctx_id = erased_cctx_ids.front(); - erased_cctx_ids.pop_front(); - cctx_hctx_refs[new_cctx_id] = 1; - cctx_element_refs[new_cctx_id] = 0; - } - return new_cctx_id; -} - -void Contexts::remove(this Contexts& self, llvm::StringRef path) { - auto it = self.header_contexts.find(path); - - /// If no such file, nothing to do. - if(it == self.header_contexts.end()) { - return; - } - - llvm::SmallVector erased_hctx_ids; - llvm::SmallVector erased_cctx_ids; - - for(auto& context: it->second) { - erased_hctx_ids.push_back(context.hctx_id); - self.erased_hctx_ids.push_back(context.hctx_id); - - auto cctx_id = context.cctx_id; - auto& ref_count = self.cctx_hctx_refs[cctx_id]; - assert(ref_count > 0); - - /// If the ref count of the canonical context id drops to 0, - /// we need to delete it. - ref_count -= 1; - if(ref_count == 0) { - erased_cctx_ids.push_back(cctx_id); - self.erased_cctx_ids.push_back(cctx_id); - self.cctx_element_refs[cctx_id] = 0; - } - } - - self.header_contexts.erase(it); - - /// Remove all refs to this header context id. - for(auto& state: self.independent_elem_states) { - for(auto hctx_id: erased_hctx_ids) { - state.erase(hctx_id); - } - } - - /// Remove all refs to this canonical context id. - Bitmap erased_flag = self.erased_flag(); - - for(auto& state: self.dependent_elem_states) { - state &= erased_flag; - } -} - -} // namespace clice::index diff --git a/src/Index/Index2.cpp b/src/Index/HeaderIndex.cpp similarity index 57% rename from src/Index/Index2.cpp rename to src/Index/HeaderIndex.cpp index 8b01f422..8f803215 100644 --- a/src/Index/Index2.cpp +++ b/src/Index/HeaderIndex.cpp @@ -1,17 +1,88 @@ -#include "Index/Index2.h" -#include "Support/Ranges.h" +#include "Index/HeaderIndex.h" -namespace clice::index { +namespace clice::index::memory { -namespace memory2 { +std::uint32_t HeaderIndex::alloc_hctx_id() { + std::uint32_t new_hctx_id; + if(erased_hctx_ids.empty()) { + new_hctx_id = max_hctx_id; + max_hctx_id += 1; + } else { + new_hctx_id = erased_hctx_ids.front(); + erased_hctx_ids.pop_front(); + } + return new_hctx_id; +} + +std::uint32_t HeaderIndex::alloc_cctx_id() { + std::uint32_t new_cctx_id; + if(erased_cctx_ids.empty()) { + new_cctx_id = max_cctx_id; + max_cctx_id += 1; + cctx_hctx_refs.emplace_back(1); + cctx_element_refs.emplace_back(0); + } else { + new_cctx_id = erased_cctx_ids.front(); + erased_cctx_ids.pop_front(); + cctx_hctx_refs[new_cctx_id] = 1; + cctx_element_refs[new_cctx_id] = 0; + } + return new_cctx_id; +} + +void HeaderIndex::remove(this HeaderIndex& self, llvm::StringRef path) { + auto it = self.header_contexts.find(path); + + /// If no such file, nothing to do. + if(it == self.header_contexts.end()) { + return; + } + + llvm::SmallVector erased_hctx_ids; + llvm::SmallVector erased_cctx_ids; + + for(auto& context: it->second) { + erased_hctx_ids.push_back(context.hctx_id); + self.erased_hctx_ids.push_back(context.hctx_id); + + auto cctx_id = context.cctx_id; + auto& ref_count = self.cctx_hctx_refs[cctx_id]; + assert(ref_count > 0); + + /// If the ref count of the canonical context id drops to 0, + /// we need to delete it. + ref_count -= 1; + if(ref_count == 0) { + erased_cctx_ids.push_back(cctx_id); + self.erased_cctx_ids.push_back(cctx_id); + self.cctx_element_refs[cctx_id] = 0; + } + } + + self.header_contexts.erase(it); + + /// Remove all refs to this header context id. + for(auto& state: self.independent_elem_states) { + for(auto hctx_id: erased_hctx_ids) { + state.erase(hctx_id); + } + } + + /// Remove all refs to this canonical context id. + Bitmap erased_flag = self.erased_flag(); + + for(auto& state: self.dependent_elem_states) { + state &= erased_flag; + } +} /// Merge all elements from other into self. And update_context is invoked every time /// when a element is inserted. The second argument is inserted `Contextual` in the /// other, the first element is inserted element in the self, empty if the element /// is new to self. -static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_context) { +static void merge_elements(HeaderIndex& self, RawIndex& raw, auto& update_context) { /// Merge symbols from other into self. - for(auto& [symbol_id, symbol]: other.symbols) { + for(auto& [symbol_id, symbol]: raw.symbols) { auto [it, success] = self.symbols.try_emplace(symbol_id, std::move(symbol)); auto& self_symbol = it->second; @@ -19,7 +90,7 @@ static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_c /// If insert successfully, this is a new symbol and it means /// we need update all context states of this symbol. for(auto& relation: self_symbol.relations) { - update_context(relation.ctx, Contextual(relation.ctx), true); + update_context(relation.ctx, relation.ctx.is_dependent(), true); } continue; } @@ -27,18 +98,18 @@ static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_c /// If self already has this symbol, try to merge all relations. for(auto& relation: symbol.relations) { auto [it, success] = self_symbol.relations.insert(relation); - update_context(it->ctx, Contextual(relation.ctx), success); + update_context(it->ctx, relation.ctx.is_dependent(), success); } } - for(auto& [range, occurrence_group]: other.occurrences) { + for(auto& [range, occurrence_group]: raw.occurrences) { auto [it, success] = self.occurrences.try_emplace(range, std::move(occurrence_group)); auto& self_occurrence_group = it->second; if(success) [[unlikely]] { /// Insert successfully. for(auto& occurrence: self_occurrence_group) { - update_context(occurrence.ctx, Contextual(occurrence.ctx), true); + update_context(occurrence.ctx, occurrence.ctx.is_dependent(), true); } continue; } @@ -56,19 +127,20 @@ static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_c } if(i != self_occurrence_group.size()) { - update_context(self_occurrence_group[i].ctx, Contextual(occurrence.ctx), false); + update_context(self_occurrence_group[i].ctx, occurrence.ctx.is_dependent(), false); } else { /// If not found insert new occurrence. auto& o = self_occurrence_group.emplace_back(occurrence); - update_context(o.ctx, Contextual(occurrence.ctx), true); + update_context(o.ctx, occurrence.ctx.is_dependent(), true); } } } } -auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext { - assert(!other.merged && "quick merge could be only used for the unmerged index"); - +auto HeaderIndex::merge(this HeaderIndex& self, + llvm::StringRef path, + std::uint32_t include, + RawIndex& raw) -> HeaderContext { /// We could make sure the other has only one header context. std::uint32_t new_hctx_id = self.alloc_hctx_id(); @@ -80,7 +152,9 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea /// TODO: simplify the logic of update context. - auto update_context = [&](Contextual& self_elem, Contextual other_elem, bool is_new) { + std::uint32_t old_elements_refs = 0; + + auto update_context = [&](Contextual& self_elem, bool is_dependent, bool is_new) { std::uint32_t new_elem_id; if(is_new) { @@ -92,7 +166,8 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea new_cctx_id = self.alloc_cctx_id(); } - if(other_elem.is_dependent()) { + if(is_dependent) { + old_elements_refs += 1; new_elem_id = self.alloc_dependent_elem_id(); self.dependent_elem_states[new_elem_id].set(new_cctx_id); } else { @@ -100,9 +175,10 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea self.independent_elem_states[new_elem_id].insert(new_hctx_id); } - self_elem = Contextual::from(other_elem.is_dependent(), new_elem_id); + self_elem = Contextual::from(is_dependent, new_elem_id); } else { if(self_elem.is_dependent()) { + old_elements_refs += 1; if(is_new_cctx) { /// If this element is not new, but we already make sure the context is new /// add its context. @@ -123,7 +199,7 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea }; /// Merge all elements from other into self and calculate the bitmap state. - merge_elements(self, other, update_context); + merge_elements(self, raw, update_context); if(!is_new_cctx) { assert(new_cctx_id == -1 && flag.any()); @@ -132,7 +208,7 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea continue; } - if(self.cctx_element_refs[i] == other.cctx_element_refs.front()) { + if(self.cctx_element_refs[i] == old_elements_refs) { new_cctx_id = i; break; } @@ -149,72 +225,14 @@ auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> Hea for(auto id: visited_elem_ids) { self.dependent_elem_states[id].set(new_cctx_id); } - self.cctx_element_refs[new_cctx_id] = other.cctx_element_refs.front(); + self.cctx_element_refs[new_cctx_id] = old_elements_refs; } - auto& [path, old_contexts] = *other.header_contexts.begin(); return self.header_contexts[path].emplace_back(HeaderContext{ - .include = old_contexts[0].include, + .include = include, .hctx_id = new_hctx_id, .cctx_id = new_cctx_id, }); } -auto SymbolIndex::merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext { - return self.quick_merge(other); -} - -Symbol& SymbolIndex::getSymbol(std::uint64_t symbol_id) { - assert(canonical_context_count() == 1 && "please use merge for multiple contexts"); - if(auto it = symbols.find(symbol_id); it != symbols.end()) { - return it->second; - } - - /// If not found, create a new symbol and return it. - auto symbol_ref = symbols.size(); - auto [it, _] = symbols.try_emplace(symbol_id, Symbol{.id = symbol_id}); - return it->second; -} - -void SymbolIndex::addRelation(Symbol& symbol, Relation relation, bool is_dependent) { - assert(!merged && "add relation could be used in only not merged index"); - std::uint32_t element_id; - if(is_dependent) { - element_id = alloc_dependent_elem_id(); - dependent_elem_states[element_id].set(0); - cctx_element_refs[0] += 1; - } else { - element_id = alloc_independent_elem_id(); - independent_elem_states.emplace_back(0); - } - - relation.ctx = Contextual::from(is_dependent, element_id); - symbol.relations.insert(relation); -} - -void SymbolIndex::addOccurrence(LocalSourceRange range, - std::int64_t target_symbol, - bool is_dependent) { - assert(!merged && "add occurrence could be used in only not merged index"); - - auto& targets = occurrences[range]; - - std::uint32_t element_id; - if(is_dependent) { - element_id = alloc_dependent_elem_id(); - dependent_elem_states[element_id].set(0); - cctx_element_refs[0] += 1; - } else { - element_id = alloc_independent_elem_id(); - independent_elem_states.emplace_back(0); - } - - Occurrence occurrence; - occurrence.target_symbol = target_symbol; - occurrence.ctx = Contextual::from(is_dependent, element_id); - targets.emplace_back(occurrence); -} - -} // namespace memory2 - -} // namespace clice::index +} // namespace clice::index::memory diff --git a/src/Index/Index.cpp b/src/Index/Index.cpp index a1d61436..af0d27a5 100644 --- a/src/Index/Index.cpp +++ b/src/Index/Index.cpp @@ -1,36 +1,145 @@ +#include "AST/Semantic.h" #include "Index/Index.h" -#include "Compiler/AST.h" +#include "Index/IncludeGraph.h" +#include "Support/Format.h" -namespace clice::index { +namespace clice::index::memory { -Shared Index::build(ASTInfo& AST) { - llvm::DenseMap indices; +namespace { - auto symbolIndices = SymbolIndex::build(AST); - for(auto& [fid, index]: symbolIndices) { - indices[fid].symbol.emplace(std::move(index)); +class IndexBuilder : public Indices, public SemanticVisitor { +public: + IndexBuilder(ASTInfo& AST) : SemanticVisitor(AST, false) { + tu_index = std::make_unique(); + tu_index->path = AST.getFilePath(SM.getMainFileID()); + tu_index->content = AST.getFileContent(SM.getMainFileID()); + tu_index->graph = IncludeGraph::from(AST); } - auto featureIndices = FeatureIndex::build(AST); - for(auto& [fid, index]: featureIndices) { - indices[fid].feature.emplace(std::move(index)); - } - - for(auto& [fid, index]: indices) { - if(index.symbol) { - auto data = llvm::ArrayRef(reinterpret_cast(index.symbol->data()), - index.symbol->size()); - index.symbolHash = llvm::xxh3_128bits(data); + RawIndex& getIndex(clang::FileID fid) { + if(fid == SM.getMainFileID()) { + return *tu_index; } - if(index.feature) { - auto data = llvm::ArrayRef(reinterpret_cast(index.feature->data()), - index.feature->size()); - index.featureHash = llvm::xxh3_128bits(data); + if(auto it = header_indices.find(fid); it != header_indices.end()) { + return *it->second; + } + + auto [it, _] = header_indices.try_emplace(fid, new RawIndex()); + auto& index = *it->second; + index.path = AST.getFilePath(fid); + index.content = AST.getFileContent(fid); + return index; + } + + void handleDeclOccurrence(const clang::NamedDecl* decl, + RelationKind kind, + clang::SourceLocation location) { + assert(decl && "Invalid decl"); + decl = normalize(decl); + + if(location.isMacroID()) { + auto spelling = AST.getSpellingLoc(location); + auto expansion = AST.getExpansionLoc(location); + + /// FIXME: For location from macro, we only handle the case that the + /// spelling and expansion are in the same file currently. + if(AST.getFileID(spelling) != AST.getFileID(expansion)) { + return; + } + + /// For occurrence, we always use spelling location. + location = spelling; + } + + auto [fid, range] = AST.toLocalRange(location); + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(decl); + auto& symbol = index.get_symbol(symbol_id.hash); + symbol.kind = SymbolKind::from(decl); + index.add_occurrence(range, symbol_id.hash); + } + + void handleMacroOccurrence(const clang::MacroInfo* def, + RelationKind kind, + clang::SourceLocation location) { + /// FIXME: Figure out when location is MacroID. + if(location.isMacroID()) { + return; + } + + auto [fid, range] = AST.toLocalRange(location); + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(def); + auto& symbol = index.get_symbol(symbol_id.hash); + symbol.kind = SymbolKind::Macro; + symbol.name = getTokenSpelling(SM, def->getDefinitionLoc()); + index.add_occurrence(range, symbol_id.hash); + + if(kind & RelationKind::Definition) { + auto begin = def->getDefinitionLoc(); + auto end = def->getDefinitionEndLoc(); + assert(begin.isFileID() && end.isFileID() && "Invalid location"); + auto [fid2, definition_range] = AST.toLocalRange(clang::SourceRange(begin, end)); + assert(fid == fid2 && "Invalid macro definition location"); + /// definitionLoc = builder.getLocation(range); + + index.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = range, + .definition_range = definition_range, + }); + } else { + index.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = range, + .target_symbol = 0, + }); } } - return indices; + void handleRelation(const clang::NamedDecl* decl, + RelationKind kind, + const clang::NamedDecl* target, + clang::SourceRange range) { + auto [fid, relationRange] = AST.toLocalExpansionRange(range); + + Relation relation{.kind = kind}; + + if(kind.isDeclOrDef()) { + auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange()); + assert(fid == fid2 && "Invalid definition location"); + relation.range = relationRange; + relation.definition_range = definitionRange; + } else if(kind.isReference()) { + relation.range = relationRange; + relation.target_symbol = 0; + } else if(kind.isBetweenSymbol()) { + auto symbol_id = AST.getSymbolID(normalize(target)); + relation.target_symbol = symbol_id.hash; + } else if(kind.isCall()) { + auto symbol_id = AST.getSymbolID(normalize(target)); + relation.range = relationRange; + relation.target_symbol = symbol_id.hash; + } else { + std::unreachable(); + } + + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(normalize(decl)); + auto& symbol = index.get_symbol(symbol_id.hash); + index.add_relation(symbol, relation); + } +}; + +} // namespace + +Indices index(ASTInfo& AST) { + IndexBuilder builder(AST); + builder.run(); + return std::move(builder); } -} // namespace clice::index +} // namespace clice::index::memory diff --git a/src/Index/SymbolIndex.cpp b/src/Index/SymbolIndex.cpp deleted file mode 100644 index b26c0f62..00000000 --- a/src/Index/SymbolIndex.cpp +++ /dev/null @@ -1,537 +0,0 @@ -#include - -#include "AST/Semantic.h" -#include "Index/SymbolIndex.h" -#include "Support/Binary.h" -#include "Support/Compare.h" - -namespace clice::index { - -namespace { - -namespace memory { - -struct Relation { - RelationKind kind; - - /// The `data` array contains two fields whose meanings depend on the `kind`. - /// Each `RelationKind` specifies the interpretation of these fields as follows: - /// - /// - `Definition` and `Declaration`: - /// - `data[0]`: The range of the name token. - /// - `data[1]`: The range of the whole symbol. - /// - /// - `Reference` and `WeakReference`: - /// - `data[0]`: The range of the reference. - /// - `data[1]`: Empty (unused). - /// - /// - `Interface`, `Implementation`, `TypeDefinition`, `Base`, `Derived`, - /// `Constructor`, and `Destructor`: - /// - `data[0]`: Empty (unused). - /// - `data[1]`: The target symbol. - /// - /// - `Caller` and `Callee`: - /// - `data[0]`: The range of the call site. - /// - `data[1]`: The target symbol (e.g., the called function). - /// - std::uint32_t data = -1; - std::uint32_t data1 = -1; -}; - -struct Symbol { - /// The symbol id. - SymbolID id; - - /// The symbol kind. - SymbolKind kind; - - /// The relations of this symbol. - std::vector relations; -}; - -struct Occurrence { - /// The location(index) of this symbol occurrence. - std::uint32_t location = -1; - - /// The referenced symbol(index) of the this symbol occurrence. - std::uint32_t symbol = -1; -}; - -struct SymbolIndex { - /// The path of source file. - std::string path; - - /// The content of source file. - std::string content; - - /// FIXME: add includes or module names? - - /// All symbols in this file. - std::vector symbols; - - /// All occurrences in this file. - std::vector occurrences; - - /// All ranges in this file. - std::vector ranges; -}; - -} // namespace memory - -class SymbolIndexBuilder : public memory::SymbolIndex { -public: - SymbolIndexBuilder(ASTInfo& AST) : AST(AST) {} - - std::uint32_t getLocation(LocalSourceRange range) { - auto key = std::pair(range.begin, range.end); - auto [iter, success] = locationCache.try_emplace(key, ranges.size()); - if(success) { - ranges.emplace_back(range); - } - return iter->second; - } - - std::uint32_t getSymbol(const clang::NamedDecl* decl) { - auto [iter, success] = symbolCache.try_emplace(decl, symbols.size()); - if(success) { - symbols.emplace_back(memory::Symbol{ - .id = AST.getSymbolID(decl), - .kind = SymbolKind::from(decl), - }); - } - return iter->second; - } - - std::uint32_t getSymbol(const clang::MacroInfo* macro) { - auto [iter, success] = symbolCache.try_emplace(macro, symbols.size()); - if(success) { - symbols.emplace_back(memory::Symbol{ - .id = AST.getSymbolID(macro), - .kind = SymbolKind::Macro, - }); - } - return iter->second; - } - - void addOccurrence(uint32_t location, uint32_t symbol) { - occurrences.emplace_back(memory::Occurrence{ - .location = location, - .symbol = symbol, - }); - } - - void addRelation(uint32_t symbol, - RelationKind kind, - uint32_t data, - uint32_t data1 = std::numeric_limits::max()) { - symbols[symbol].relations.emplace_back(memory::Relation{ - .kind = kind, - .data = data, - .data1 = data1, - }); - } - - void sort() { - /// We will serialize the index to binary format and compare the data to - /// check whether they are the index. So here we need to sort all vectors - /// to make sure that the data is in the same order even they are in different - /// files. - - /// Map the old index to new index. - std::vector symbolMap(symbols.size()); - std::vector locationMap(ranges.size()); - - { - /// Sort symbols and update the symbolMap. - std::vector new2old(symbols.size()); - for(uint32_t i = 0; i < symbols.size(); ++i) { - new2old[i] = i; - } - - ranges::sort(views::zip(symbols, new2old), refl::less, [](const auto& element) { - auto& symbol = std::get<0>(element); - return std::tuple(symbol.id, symbol.kind); - }); - - for(uint32_t i = 0; i < symbols.size(); ++i) { - symbolMap[new2old[i]] = i; - } - } - - { - /// Sort locations and update the locationMap. - std::vector new2old(ranges.size()); - for(uint32_t i = 0; i < ranges.size(); ++i) { - new2old[i] = i; - } - - ranges::sort(views::zip(ranges, new2old), refl::less, [](const auto& element) { - return std::get<0>(element); - }); - - for(uint32_t i = 0; i < ranges.size(); ++i) { - locationMap[new2old[i]] = i; - } - } - - /// Sort occurrences and update the symbol and location references. - for(auto& occurrence: occurrences) { - occurrence.symbol = {symbolMap[occurrence.symbol]}; - occurrence.location = {locationMap[occurrence.location]}; - } - - /// Sort all occurrences and update the symbol and location references. - ranges::sort(occurrences, refl::less, [](const auto& occurrence) { - return occurrence.location; - }); - auto range = ranges::unique(occurrences, refl::equal); - occurrences.erase(range.begin(), range.end()); - - using enum RelationKind::Kind; - /// Sort all relations and update the symbol and location references. - for(auto& symbol: symbols) { - for(auto& relation: symbol.relations) { - auto kind = relation.kind; - if(kind.isDeclOrDef()) { - relation.data = locationMap[relation.data]; - relation.data1 = locationMap[relation.data1]; - } else if(kind.isReference()) { - relation.data = locationMap[relation.data]; - } else if(kind.isBetweenSymbol()) { - relation.data1 = symbolMap[relation.data1]; - } else if(kind.isCall()) { - relation.data = locationMap[relation.data]; - relation.data1 = symbolMap[relation.data1]; - } else { - assert(false && "Invalid relation kind"); - } - } - - ranges::sort(symbol.relations, refl::less); - - auto range = ranges::unique(symbol.relations, refl::equal); - symbol.relations.erase(range.begin(), range.end()); - } - } - - memory::SymbolIndex dump() { - return std::move(static_cast(*this)); - } - -private: - ASTInfo& AST; - llvm::DenseMap symbolCache; - llvm::DenseMap, uint32_t> locationCache; -}; - -class SymbolIndexCollector : public SemanticVisitor { -public: - SymbolIndexCollector(ASTInfo& AST) : SemanticVisitor(AST, false) {} - - SymbolIndexBuilder& getBuilder(clang::FileID fid) { - auto [it, success] = builders.try_emplace(fid, AST); - return it->second; - } - -public: - void handleDeclOccurrence(const clang::NamedDecl* decl, - RelationKind kind, - clang::SourceLocation location) { - assert(decl && "Invalid decl"); - decl = normalize(decl); - - if(location.isMacroID()) { - auto spelling = AST.getSpellingLoc(location); - auto expansion = AST.getExpansionLoc(location); - - /// FIXME: For location from macro, we only handle the case that the - /// spelling and expansion are in the same file currently. - if(AST.getFileID(spelling) != AST.getFileID(expansion)) { - return; - } - - /// For occurrence, we always use spelling location. - location = spelling; - } - - /// Add the occurrence. - auto [fid, range] = AST.toLocalRange(location); - auto& builder = getBuilder(fid); - auto loc = builder.getLocation(range); - auto symbol = builder.getSymbol(decl); - builder.addOccurrence(loc, symbol); - } - - void handleMacroOccurrence(const clang::MacroInfo* def, - RelationKind kind, - clang::SourceLocation location) { - /// FIXME: Figure out when location is MacroID. - if(location.isMacroID()) { - return; - } - - /// Add macro occurrence. - auto [fid, range] = AST.toLocalRange(location); - auto& builder = getBuilder(fid); - auto loc = builder.getLocation(range); - auto symbol = builder.getSymbol(def); - builder.addOccurrence(loc, symbol); - - /// If the macro is a definition, set definition range for it. - std::uint32_t definitionLoc = std::numeric_limits::max(); - - if(kind & RelationKind::Definition) { - auto begin = def->getDefinitionLoc(); - auto end = def->getDefinitionEndLoc(); - assert(begin.isFileID() && end.isFileID() && "Invalid location"); - auto [fid2, range] = AST.toLocalRange(clang::SourceRange(begin, end)); - assert(fid == fid2 && "Invalid macro definition location"); - definitionLoc = builder.getLocation(range); - } - - builder.addRelation(symbol, kind, loc, definitionLoc); - } - - void handleRelation(const clang::NamedDecl* decl, - RelationKind kind, - const clang::NamedDecl* target, - clang::SourceRange range) { - auto [fid, relationRange] = AST.toLocalExpansionRange(range); - auto& builder = getBuilder(fid); - - /// Calculate the data for the relation. - std::uint32_t data[2] = { - std::numeric_limits::max(), - std::numeric_limits::max(), - }; - - if(kind.isDeclOrDef()) { - auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange()); - assert(fid == fid2 && "Invalid definition location"); - data[0] = builder.getLocation(relationRange); - data[1] = builder.getLocation(definitionRange); - } else if(kind.isReference()) { - data[0] = builder.getLocation(relationRange); - } else if(kind.isBetweenSymbol()) { - data[1] = builder.getSymbol(normalize(target)); - } else if(kind.isCall()) { - data[0] = builder.getLocation(relationRange); - data[1] = builder.getSymbol(normalize(target)); - } else { - std::unreachable(); - } - - /// Add the relation. - auto symbol = builder.getSymbol(normalize(decl)); - builder.addRelation(symbol, kind, data[0], data[1]); - } - - auto build() { - run(); - - llvm::DenseMap> result; - for(auto& [fid, builder]: builders) { - builder.sort(); - auto index = builder.dump(); - index.path = AST.getFilePath(fid); - index.content = AST.getFileContent(fid); - auto [buffer, _] = binary::serialize(index); - result.try_emplace(fid, std::move(buffer)); - } - - return std::move(result); - } - -private: - llvm::DenseMap builders; -}; - -} // namespace - -RelationKind Relation::kind() const { - binary::Proxy proxy{base, data}; - return proxy->kind; -} - -LocalSourceRange Relation::range() const { - assert(!kind().isBetweenSymbol()); - binary::Proxy index{base, base}; - binary::Proxy proxy{base, data}; - return index.get<"ranges">().as_array()[proxy->data]; -} - -LocalSourceRange Relation::sourceRange() const { - assert(kind().isDeclOrDef() && "only declaration or definition has sourceRange range"); - binary::Proxy index{base, base}; - binary::Proxy proxy{base, data}; - return index.get<"ranges">().as_array()[proxy->data1]; -} - -Symbol Relation::target() const { - assert((kind().isBetweenSymbol() || kind().isCall()) && "only symbols has target"); - binary::Proxy index{base, base}; - binary::Proxy proxy{base, data}; - return Symbol{base, &index.get<"symbols">().as_array()[proxy->data1]}; -} - -SymbolID Symbol::id() const { - binary::Proxy proxy{base, data}; - return binary::deserialize(proxy.get<"id">()); -} - -std::uint64_t Symbol::hash() const { - binary::Proxy proxy{base, data}; - return proxy.get<"id">().get<"hash">().value(); -} - -llvm::StringRef Symbol::name() const { - binary::Proxy proxy{base, data}; - return proxy.get<"id">().get<"name">().as_string(); -} - -SymbolKind Symbol::kind() const { - binary::Proxy proxy{base, data}; - return proxy.get<"kind">(); -} - -template -static LazyArray getLazyArray(binary::Proxy> proxy) { - return LazyArray{ - proxy.base, - &proxy.as_array()[0], - proxy.size(), - sizeof(binary::binarify_t), - }; -} - -LazyArray Symbol::relations() const { - binary::Proxy symbol{base, data}; - return getLazyArray(symbol.get<"relations">()); -} - -LocalSourceRange Occurrence::range() const { - binary::Proxy index{base, base}; - binary::Proxy occurrence{base, data}; - return index.get<"ranges">().as_array()[occurrence->location]; -} - -Symbol Occurrence::symbol() const { - binary::Proxy index{base, base}; - binary::Proxy occurrence{base, data}; - return Symbol{ - base, - &index.get<"symbols">().as_array()[occurrence->symbol], - }; -} - -llvm::StringRef SymbolIndex::path() const { - binary::Proxy index{data, data}; - return index.get<"path">().as_string(); -} - -llvm::StringRef SymbolIndex::content() const { - binary::Proxy index{data, data}; - return index.get<"content">().as_string(); -} - -LazyArray SymbolIndex::symbols() const { - binary::Proxy index{data, data}; - return getLazyArray(index.get<"symbols">()); -} - -LazyArray SymbolIndex::occurrences() const { - binary::Proxy index{data, data}; - return getLazyArray(index.get<"occurrences">()); -} - -std::vector SymbolIndex::locateSymbol(uint32_t offset) const { - auto occurrences = this->occurrences(); - auto iter = ranges::lower_bound(occurrences, offset, {}, [](const Occurrence& occurrence) { - return occurrence.range().end; - }); - - std::vector result; - while(iter != occurrences.end()) { - auto occurrence = *iter; - if(occurrence.range().begin > offset) { - break; - } - result.emplace_back(occurrence.symbol()); - ++iter; - } - return result; -} - -std::optional SymbolIndex::locateSymbol(const SymbolID& id) const { - auto symbols = this->symbols(); - auto iter = ranges::lower_bound(symbols, id.hash, {}, [](const Symbol& symbol) { - return symbol.hash(); - }); - - auto symbol = *iter; - if(symbol.hash() == id.hash && symbol.name() == id.name) { - return symbol; - } - - return std::nullopt; -} - -Shared> SymbolIndex::build(ASTInfo& AST) { - return SymbolIndexCollector(AST).build(); -} - -json::Value SymbolIndex::toJSON(bool line) { - json::Array symbols; - for(auto symbol: this->symbols()) { - json::Array relations; - - for(auto relation: symbol.relations()) { - relations.push_back(json::Object{ - {"kind", llvm::StringRef(relation.kind().name())}, - }); - - using enum RelationKind::Kind; - auto kind = relation.kind(); - - if(kind.isDeclOrDef()) { - relations.back().getAsObject()->insert( - {"definitionRange", json::serialize(relation.sourceRange())}); - } - - if(!kind.isBetweenSymbol()) { - relations.back().getAsObject()->insert( - {"range", json::serialize(relation.range())}); - } else { - relations.back().getAsObject()->insert( - {"symbol", json::serialize(relation.target().id())}); - } - - if(kind.isCall()) { - relations.back().getAsObject()->insert( - {"symbol", json::serialize(relation.target().id())}); - } - } - - symbols.push_back(json::Object{ - {"hash", symbol.hash() }, - {"name", symbol.name() }, - {"kind", llvm::StringRef(symbol.kind().name())}, - {"relations", std::move(relations) }, - }); - } - - json::Array occurrences; - for(auto occurrence: this->occurrences()) { - occurrences.push_back(json::Object{ - {"range", json::serialize(occurrence.range()) }, - {"id", json::serialize(occurrence.symbol().id())} - }); - } - - return json::Object{ - {"symbols", std::move(symbols) }, - {"occurrences", std::move(occurrences)}, - }; -} - -} // namespace clice::index diff --git a/src/Index/SymbolIndex2.cpp b/src/Index/SymbolIndex2.cpp deleted file mode 100644 index cee7378d..00000000 --- a/src/Index/SymbolIndex2.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "AST/Semantic.h" -#include "Index/Index2.h" -#include "Index/IncludeGraph.h" -#include "Support/Format.h" - -namespace clice::index::memory2 { - -class SymbolIndexBuilder : public SemanticVisitor { -public: - SymbolIndexBuilder(ASTInfo& AST) : - SemanticVisitor(AST, false), graph(IncludeGraph::from(AST)), - context_path(AST.getFilePath(SM.getMainFileID())) {} - - SymbolIndex& getIndex(clang::FileID fid) { - if(auto it = indices.find(fid); it != indices.end()) { - return *it->second; - } - - auto [it, _] = indices.try_emplace(fid, new SymbolIndex()); - auto& index = *it->second; - /// Fix me build include graph here. - index.add_context(context_path, graph.getInclude(fid)); - return index; - } - - void handleDeclOccurrence(const clang::NamedDecl* decl, - RelationKind kind, - clang::SourceLocation location) { - assert(decl && "Invalid decl"); - decl = normalize(decl); - - if(location.isMacroID()) { - auto spelling = AST.getSpellingLoc(location); - auto expansion = AST.getExpansionLoc(location); - - /// FIXME: For location from macro, we only handle the case that the - /// spelling and expansion are in the same file currently. - if(AST.getFileID(spelling) != AST.getFileID(expansion)) { - return; - } - - /// For occurrence, we always use spelling location. - location = spelling; - } - - auto [fid, range] = AST.toLocalRange(location); - auto& index = getIndex(fid); - auto symbol_id = AST.getSymbolID(decl); - auto& symbol = index.getSymbol(symbol_id.hash); - symbol.kind = SymbolKind::from(decl); - index.addOccurrence(range, symbol_id.hash); - } - - void handleMacroOccurrence(const clang::MacroInfo* def, - RelationKind kind, - clang::SourceLocation location) { - /// FIXME: Figure out when location is MacroID. - if(location.isMacroID()) { - return; - } - - auto [fid, range] = AST.toLocalRange(location); - auto& index = getIndex(fid); - auto symbol_id = AST.getSymbolID(def); - auto& symbol = index.getSymbol(symbol_id.hash); - symbol.kind = SymbolKind::Macro; - symbol.name = getTokenSpelling(SM, def->getDefinitionLoc()); - index.addOccurrence(range, symbol_id.hash); - - if(kind & RelationKind::Definition) { - auto begin = def->getDefinitionLoc(); - auto end = def->getDefinitionEndLoc(); - assert(begin.isFileID() && end.isFileID() && "Invalid location"); - auto [fid2, definition_range] = AST.toLocalRange(clang::SourceRange(begin, end)); - assert(fid == fid2 && "Invalid macro definition location"); - /// definitionLoc = builder.getLocation(range); - - index.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = range, - .definition_range = definition_range, - }); - } else { - index.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = range, - .target_symbol = 0, - }); - } - } - - void handleRelation(const clang::NamedDecl* decl, - RelationKind kind, - const clang::NamedDecl* target, - clang::SourceRange range) { - auto [fid, relationRange] = AST.toLocalExpansionRange(range); - - Relation relation{.kind = kind}; - - if(kind.isDeclOrDef()) { - auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange()); - assert(fid == fid2 && "Invalid definition location"); - relation.range = relationRange; - relation.definition_range = definitionRange; - } else if(kind.isReference()) { - relation.range = relationRange; - relation.target_symbol = 0; - } else if(kind.isBetweenSymbol()) { - auto symbol_id = AST.getSymbolID(normalize(target)); - relation.target_symbol = symbol_id.hash; - } else if(kind.isCall()) { - auto symbol_id = AST.getSymbolID(normalize(target)); - relation.range = relationRange; - relation.target_symbol = symbol_id.hash; - } else { - std::unreachable(); - } - - auto& index = getIndex(fid); - auto symbol_id = AST.getSymbolID(normalize(decl)); - auto& symbol = index.getSymbol(symbol_id.hash); - index.addRelation(symbol, relation); - } - - auto build() { - run(); - - return std::move(indices); - } - -private: - IncludeGraph graph; - std::string context_path; - llvm::DenseMap> indices; -}; - -index::Shared> SymbolIndex::build(ASTInfo& AST) { - return SymbolIndexBuilder(AST).build(); -} - -} // namespace clice::index::memory2 diff --git a/src/Server/IncludeGraph.cpp b/src/Server/IncludeGraph.cpp index f45cdb33..f4250f01 100644 --- a/src/Server/IncludeGraph.cpp +++ b/src/Server/IncludeGraph.cpp @@ -264,62 +264,4 @@ void IncludeGraph::addContexts(ASTInfo& AST, } } -async::Task<> IncludeGraph::updateIndices(ASTInfo& info, - TranslationUnit* tu, - llvm::DenseMap& files) { - auto indices = co_await async::submit([&info] { return index::Index::build(info); }); - - auto& SM = info.srcMgr(); - - for(auto& [fid, index]: indices) { - if(fid == SM.getMainFileID()) { - if(tu->indexPath.empty()) { - tu->indexPath = getIndexPath(tu->srcPath); - } - - co_await index.write(tu->indexPath); - - continue; - } - - auto name = info.getFilePath(fid); - assert(headers.contains(name) && "Invalid header name"); - - auto header = headers[name]; - - auto include = files[fid]; - auto iter = ranges::find_if(header->contexts[tu], [&](const Context& context) { - return context.include == include; - }); - assert(iter != header->contexts[tu].end() && "Invalid include index"); - - /// Found whether the we already have the same index. If so, use it directly. - /// Otherwise, we need to create a new index. - auto& indices = header->indices; - - bool existed = false; - for(std::size_t i = 0; i < indices.size(); ++i) { - auto& element = indices[i]; - if(index.symbolHash == element.symbolHash && index.featureHash == element.featureHash) { - existed = true; - iter->index = i; - break; - } - } - - if(existed) { - continue; - } - - iter->index = static_cast(indices.size()); - indices.emplace_back(HeaderIndex{ - .path = getIndexPath(header->srcPath), - .symbolHash = index.symbolHash, - .featureHash = index.featureHash, - }); - - co_await index.write(indices.back().path); - } -} - } // namespace clice diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp index ffd41dba..44967844 100644 --- a/src/Server/Indexer.cpp +++ b/src/Server/Indexer.cpp @@ -1,373 +1,63 @@ +#include "Compiler/AST.h" +#include "Compiler/Compilation.h" +#include "Index/Index.h" #include "Server/Indexer.h" -#include "Server/IncludeGraph.h" #include "Support/Logger.h" namespace clice { -Indexer::Indexer(CompilationDatabase& database, const config::IndexOptions& options) : - IncludeGraph(options), database(database), options(options) {} +async::Task<> Indexer::index(ASTInfo& AST) { + auto [tu_index, header_indices] = + co_await async::submit([&] { return index::memory::index(AST); }); -void Indexer::add(std::string file) { - /// If the file is already indexed, cancel and start a new indexing task. - if(auto it = tasks.find(file); it != tasks.end()) [[unlikely]] { - auto& task = it->second; - assert(!task.done() && "if task is done, it should be removed from the map"); - task.cancel(); - task.dispose(); - task = index(file); - task.schedule(); - return; + auto tu_id = getPath(tu_index->path); + + llvm::DenseSet visited_headers; + + for(auto& [fid, index]: header_indices) { + auto id = getPath(index->path); + auto it = dynamic_header_indices.find(id); + + /// FIXME: If the value less than 0, it represents the file is from + /// PCH or module. How to handle such file? + if(fid < clang::FileID::getSentinel()) { + continue; + } + + auto include = tu_index->graph.getInclude(fid); + + if(it != dynamic_header_indices.end()) { + auto& indices = it->second; + + if(!visited_headers.contains(id)) { + indices->unmergeds[tu_id].clear(); + visited_headers.insert(id); + } + + indices->unmergeds[tu_id].emplace_back(include, std::move(index)); + } else { + auto [it, _] = dynamic_header_indices.try_emplace(id, new HeaderIndices()); + it->second->unmergeds[tu_id].emplace_back(include, std::move(index)); + visited_headers.insert(id); + } } - /// If the size of the tasks list is less than the concurrency, add the file - /// to the tasks list and start a new indexing task. - if(tasks.size() != concurrency) { - auto task = index(file); - task.schedule(); - tasks.try_emplace(file, std::move(task)); - pending.erase(file); - return; - } - - /// Finally, all tasks are running, add the file to the pending list. - pending.insert(file); + dynamic_tu_indices[tu_id] = std::move(tu_index); } -void Indexer::remove(std::string file) { - if(pending.contains(file)) { - pending.erase(file); - return; - } +async::Task<> Indexer::index(llvm::StringRef file) { + CompilationParams params; + params.srcPath = file; + params.command = database.getCommand(file); - if(auto it = tasks.find(file); it != tasks.end()) { - it->second.cancel(); - tasks.erase(it); + auto AST = co_await async::submit([&] { return compile(params); }); - return; - } -} - -void Indexer::indexAll() { - for(auto& entry: database) { - add(entry.first().str()); - } -} - -void Indexer::save() { - auto json = IncludeGraph::dump(); - auto result = fs::write(path::join(options.dir, "index.json"), std::format("{}", json)); - if(result) { - log::info("Successfully saved index to disk"); - } else { - log::warn("Failed to save index to disk: {}", result.error()); - } -} - -void Indexer::load() { - auto path = path::join(options.dir, "index.json"); - auto file = fs::read(path); - if(!file) { - log::warn("Failed to open index file: {}", file.error()); - return; - } - - if(auto result = json::parse(file.value())) { - IncludeGraph::load(*result); - log::info("Successfully loaded index from disk"); - } else { - log::warn("Failed to parse index file: {}", result.takeError()); - } -} - -async::Task<> Indexer::index(std::string file) { - assert(!pending.contains(file) && "file should not be in the pending list"); - assert(tasks.contains(file) && "file should not be in the tasks list"); - - auto task = IncludeGraph::index(file, database); - co_await task; - - log::info("index process: [running: {}, pending :{}], finish {}", - tasks.size() - 1, - pending.size(), - file); - - auto it = tasks.find(file); - assert(it != tasks.end() && "file should be in the tasks list"); - /// We cannot directly erase the task from the map, this will call - /// its destructor and destroy the coroutine. But we are still - /// executing the coroutine. So dispose it. - it->second.dispose(); - tasks.erase(it); - - if(pending.empty()) { + if(!AST) { + log::info("Fail to index background file {}", file); co_return; } - /// Create a new task for the next file and schedule it. - file = pending.begin()->first(); - auto next = index(file); - next.schedule(); - - /// Remove the file from the pending list and add it to the tasks list. - pending.erase(pending.begin()); - tasks.try_emplace(file, std::move(next)); -} - -Header* Indexer::getHeader(llvm::StringRef file) const { - auto it = headers.find(file); - if(it == headers.end()) { - return nullptr; - } - return it->second; -} - -TranslationUnit* Indexer::getTranslationUnit(llvm::StringRef file) const { - auto it = tus.find(file); - if(it == tus.end()) { - return nullptr; - } - return it->second; -} - -std::optional Indexer::currentContext(llvm::StringRef file) const { - auto header = getHeader(file); - if(!header || !header->active.valid()) { - return std::nullopt; - } - - auto& active = header->active; - return proto::HeaderContext{ - .file = active.tu->srcPath, - .version = active.tu->version, - .include = active.context.include, - }; -} - -bool Indexer::switchContext(llvm::StringRef headerFile, proto::HeaderContext context) { - Header* header = getHeader(headerFile); - if(!header) { - return false; - } - - TranslationUnit* tu = getTranslationUnit(context.file); - if(!tu || tu->version != context.version) { - return false; - } - - auto index = header->getIndex(tu, context.include); - if(!index) { - return false; - } - - header->active = HeaderContext{ - .tu = tu, - .context = {.index = *index, .include = context.include}, - }; - - return true; -} - -std::vector Indexer::resolveContext(proto::HeaderContext context) const { - auto tu = getTranslationUnit(context.file); - if(!tu || tu->version != context.version) { - return {}; - } - - auto include = context.include; - auto& locations = tu->locations; - if(locations.size() <= include) [[unlikely]] { - /// FIXME: This should never occur, we should log. - return {}; - } - - std::vector result; - while(include != -1) { - auto& location = locations[include]; - result.emplace_back(proto::IncludeLocation{ - .line = location.line - 1, - .file = pathPool[location.file], - }); - include = location.include; - } - return result; -} - -std::vector Indexer::allContexts(llvm::StringRef headerFile, - uint32_t limit, - llvm::StringRef contextFile) const { - auto header = getHeader(headerFile); - if(!header) { - return {}; - } - - llvm::DenseMap> groups; - - if(auto tu = getTranslationUnit(contextFile)) { - for(auto context: header->contexts[tu]) { - groups[context.index].emplace_back(proto::HeaderContext{ - .file = tu->srcPath, - .version = tu->version, - .include = context.include, - }); - } - } else { - for(auto&& [tu, contexts]: header->contexts) { - for(auto& context: contexts) { - auto& group = groups[context.index]; - if(group.size() >= 10) { - continue; - } - - group.emplace_back(proto::HeaderContext{ - .file = tu->srcPath, - .version = tu->version, - .include = context.include, - }); - } - } - } - - std::vector result; - for(auto& [index, group]: groups) { - result.emplace_back(header->indices[index].path, std::move(group)); - } - return result; -} - -std::vector Indexer::indices(TranslationUnit* tu) { - std::vector indices; - if(tu) { - indices.emplace_back(tu->indexPath); - for(auto& header: tu->headers) { - for(auto& context: header->contexts[tu]) { - indices.emplace_back(header->indices[context.index].path); - } - } - } else { - for(auto& [_, tu]: tus) { - indices.emplace_back(tu->indexPath); - } - - for(auto& [_, header]: headers) { - for(auto& index: header->indices) { - indices.emplace_back(index.path); - } - } - } - return indices; -} - -async::Task> - Indexer::resolve(const proto::TextDocumentPositionParams& params) { - std::vector ids; - // auto path = SourceConverter::toPath(params.textDocument.uri); - // - // std::uint32_t offset = 0; - //{ - // auto content = co_await async::fs::read(path); - // if(!content) { - // co_return ids; - // } - // offset = SC.toOffset(*content, params.position); - //} - // - // std::vector indices; - // if(auto iter = tus.find(path); iter != tus.end()) { - // indices.emplace_back(iter->second->indexPath); - //} else if(auto iter = headers.find(path); iter != headers.end()) { - // /// FIXME: What is the expected result of resolving a symbol that may refers different - // /// declaration in different header contexts? Currently, we just return the first one. - // for(auto& index: iter->second->indices) { - // indices.emplace_back(index.path); - // } - //} - // - // co_await async::gather(indices, [&](const std::string& path) -> async::Task { - // auto binary = co_await async::fs::read(path + ".sidx"); - // if(!binary) { - // co_return true; - // } - // - // llvm::SmallVector symbols; - // co_await async::submit([&] { - // index::SymbolIndex index(binary->data(), binary->size()); - // index.locateSymbols(offset, symbols); - // }); - // - // if(symbols.empty()) { - // co_return true; - // } - // - // /// If we found the symbol, we just return it. And break other tasks. - // for(auto& symbol: symbols) { - // ids.emplace_back(symbol.id(), symbol.name().str()); - // } - // co_return false; - //}); - // - // co_return ids; - co_return std::vector{}; -} - -async::Task> - Indexer::getFeatureIndex(std::string& buffer, llvm::StringRef file) const { - std::string path; - if(auto tu = tus.find(file); tu != tus.end()) { - path = tu->second->indexPath; - } - - if(auto header = headers.find(file); header != headers.end()) { - for(auto&& index: header->second->indices) { - path = index.path; - break; - } - } - - auto content = co_await async::fs::read(path + ".fidx"); - if(!content) { - co_return std::nullopt; - } - - buffer = std::move(*content); - - co_return index::FeatureIndex(buffer.data(), buffer.size()); -} - -async::Task> - Indexer::semanticTokens(llvm::StringRef file) const { - std::vector result; - - std::string buffer; - auto index = co_await getFeatureIndex(buffer, file); - if(!index) { - co_return result; - } - - co_return index->semanticTokens(); -} - -async::Task> Indexer::foldingRanges(llvm::StringRef file) const { - std::vector result; - - std::string buffer; - auto index = co_await getFeatureIndex(buffer, file); - if(!index) { - co_return result; - } - - co_return index->foldingRanges(); -} - -async::Task> Indexer::documentLinks(llvm::StringRef file) const { - std::vector result; - - std::string buffer; - auto index = co_await getFeatureIndex(buffer, file); - if(!index) { - co_return result; - } - - co_return index->documentLinks(); + co_await index(*AST); } } // namespace clice diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index b641d9a4..5c8a56c4 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -149,6 +149,12 @@ async::Task<> Scheduler::buildPCH(std::string path, std::string content) { } async::Task<> Scheduler::buildAST(std::string path, std::string content) { + auto file = &openFiles[path]; + + /// Try get the lock, the waiter on the lock will be resumed when + /// guard is destroyed. + auto guard = co_await file->ASTBuiltLock.try_lock(); + /// PCH is already updated. co_await buildPCH(path, content); @@ -171,16 +177,14 @@ async::Task<> Scheduler::buildAST(std::string path, std::string content) { co_return; } - auto& file = openFiles[path]; - - /// Try get the lock, the waiter on the lock will be resumed when - /// guard is destroyed. - auto guard = co_await file.ASTBuiltLock.try_lock(); + /// Index the source file. + co_await indexer.index(*info); + file = &openFiles[path]; /// Update built AST info. - file.AST = std::make_shared(std::move(*info)); + file->AST = std::make_shared(std::move(*info)); /// Dispose the task so that it will destroyed when task complete. - file.ASTBuild.dispose(); + file->ASTBuild.dispose(); log::info("Building AST successfully for {}", path); } diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 6bd4f7c0..d898ffbd 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -55,7 +55,7 @@ async::Task<> Server::registerCapacity(llvm::StringRef id, }); } -Server::Server() : indexer(database, config::index), scheduler(converter, database) { +Server::Server() : indexer(database), scheduler(indexer, converter, database) { onRequests.try_emplace("initialize", &Server::onInitialize); onRequests.try_emplace("textDocument/semanticTokens/full", &Server::onSemanticToken); onRequests.try_emplace("textDocument/completion", &Server::onCodeCompletion); diff --git a/unittests/Index/HeaderIndex.cpp b/unittests/Index/HeaderIndex.cpp new file mode 100644 index 00000000..6a7c09ed --- /dev/null +++ b/unittests/Index/HeaderIndex.cpp @@ -0,0 +1,888 @@ +#include "Test/CTest.h" +#include "Index/HeaderIndex.h" + +namespace clice::testing { + +namespace { + +using namespace clice::index::memory; + +struct DumpConfig { + bool enable_total = false; + bool enable_contexts = false; + bool enable_symbol = false; + bool enable_occurrence = false; +}; + +void dump(HeaderIndex& index, DumpConfig config) { + if(config.enable_total) { + println("\n---------------------------Total Info---------------------------"); + println("file count: {}", index.file_count()); + println("header context count: {}", index.header_context_count()); + println("canonical context count: {}", index.canonical_context_count()); + println("symbol count: {}", index.symbols.size()); + println("occurrence count: {}", index.occurrences.size()); + } + + if(config.enable_contexts) { + println("\n--------------------------Contexts Info-------------------------"); + for(auto& [path, contents]: index.header_contexts) { + println("{}:", path); + for(auto& context: contents) { + println(" include: {}, hctx_id: {}, cctx_id: {}", + context.include, + context.hctx_id, + context.cctx_id); + } + } + } + + if(config.enable_symbol) { + println("\n-------------------------Symbols Info--------------------------"); + for(auto& [symbol_id, symbol]: index.symbols) { + clice::println("symbol: {}, kind: {}", symbol.name, symbol.kind.name()); + for(auto& relation: symbol.relations) { + if(relation.ctx.is_dependent()) { + auto context = index.dependent_elem_states[relation.ctx.offset()]; + clice::println(" kind: {}, context: {:#b}", + relation.kind.name(), + context.to_ulong()); + } + } + } + } + + if(config.enable_occurrence) { + println("\n-------------------------Occurrences Info--------------------------"); + for(auto& [range, occurrences]: index.occurrences) { + println("occurrence: {} {}", range.begin, range.end); + for(auto& occurrence: occurrences) { + if(occurrence.ctx.is_dependent()) { + auto context = index.dependent_elem_states[occurrence.ctx.offset()]; + println(" target: {}, context: {:#b}", + occurrence.target_symbol, + context.to_ulong()); + } + } + } + } +} + +/// TODO: We should have a more clean way to save test data(like json), rather than out put code +/// directly. +std::string test_code(HeaderIndex& index, std::uint32_t id) { + std::string code; + std::string index_name = std::format("index{}", id); + code += std::format("RawIndex {};", index_name); + auto& [path, contexts] = *index.header_contexts.begin(); + code += std::format(R"({}.add_context("{}", {});)", index_name, path, contexts[0].include); + code += "\n"; + + for(auto& [symbol_id, symbol]: index.symbols) { + code += "{"; + code += std::format(R"( + auto& symbol = {}.get_symbol({}ull); + symbol.name = "{}"; + symbol.kind = SymbolKind::{}; + )", + index_name, + symbol_id, + symbol.name, + symbol.kind.name()); + code += "\n"; + + for(auto& relation: symbol.relations) { + code += std::format(R"({}.add_relation( + symbol, + Relation{{ + .kind = RelationKind::{}, + .range = {{ {}, {} }}, + .target_symbol = {}ull, + }} + );)", + index_name, + relation.kind.name(), + relation.range.begin, + relation.range.end, + relation.target_symbol); + } + code += "}\n"; + } + + // for(auto& [range, occurrence_group]: index.occurrences) { + // code += "{"; + // code += std::format("LocalSourceRange range{{ {}, {} }};\n", range.begin, range.end); + // for(auto& occurrence: occurrence_group) { + // code += std::format("index.add_occurrence(range, {});\n", occurrence.target_symbol); + // } + // code += "}"; + // } + + return code; +} + +TEST(HeaderIndex, AddRemoveContext) { + HeaderIndex index; + + { + auto context = index.add_context("test.h", 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(context.hctx_id, 0); + EXPECT_EQ(index.header_context_count(), 1); + EXPECT_EQ(index.canonical_context_count(), 1); + } + + { + auto context = index.add_context("test.h", 2); + EXPECT_EQ(context.cctx_id, 1); + EXPECT_EQ(context.hctx_id, 1); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 2); + } + + EXPECT_EQ(index.file_count(), 1); + + { + auto context = index.add_context("test2.h", 1); + EXPECT_EQ(context.cctx_id, 2); + EXPECT_EQ(context.hctx_id, 2); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 3); + } + + EXPECT_EQ(index.file_count(), 2); + + index.remove("test.h"); + + EXPECT_EQ(index.header_context_count(), 1); + EXPECT_EQ(index.canonical_context_count(), 1); + + /// Test reuse context id and context ref. + { + auto context = index.add_context("test3.h", 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(context.hctx_id, 0); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 2); + } + + { + index.add_context("test4.h", 1); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 3); + } + + { + index.add_context("test5.h", 1); + EXPECT_EQ(index.header_context_count(), 4); + EXPECT_EQ(index.canonical_context_count(), 4); + } + + EXPECT_EQ(index.file_count(), 4); +} + +TEST(HeaderIndex, SymbolInsert) { + HeaderIndex index; + index.add_context("test.h", 1); + index.add_occurrence({1, 2}, 1); +} + +TEST(HeaderIndex, MergeEmpty) { + HeaderIndex base; + + RawIndex index; + index.add_occurrence({1, 2}, 1); + base.merge("test.h", 1, index); + EXPECT_EQ(base.header_context_count(), 1); + EXPECT_EQ(base.canonical_context_count(), 1); + EXPECT_EQ(base.file_count(), 1); + + RawIndex index2; + base.merge("test2.h", 1, index2); + EXPECT_EQ(base.header_context_count(), 2); + EXPECT_EQ(base.canonical_context_count(), 2); + EXPECT_EQ(base.file_count(), 2); + + RawIndex index3; + base.merge("test3.h", 1, index3); + EXPECT_EQ(base.header_context_count(), 3); + EXPECT_EQ(base.canonical_context_count(), 2); + EXPECT_EQ(base.file_count(), 3); +} + +TEST(HeaderIndex, MergeOccurrence) { + HeaderIndex base; + + RawIndex index; + index.add_occurrence({1, 2}, 1); + base.merge("test.h", 1, index); + + RawIndex index2; + index2.add_occurrence({1, 2}, 1); + base.merge("test2.h", 1, index2); + EXPECT_EQ(base.header_context_count(), 2); + EXPECT_EQ(base.canonical_context_count(), 1); + EXPECT_EQ(base.file_count(), 2); + + RawIndex index3; + index3.add_occurrence({1, 2}, 2); + base.merge("test3.h", 1, index3); + EXPECT_EQ(base.header_context_count(), 3); + EXPECT_EQ(base.canonical_context_count(), 2); + EXPECT_EQ(base.file_count(), 3); +} + +TEST(HeaderIndex, MergeSymbol) { + LocalSourceRange range = {0, 0}; + + HeaderIndex base; + { + RawIndex index; + auto& symbol = index.get_symbol(1); + index.add_relation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); + base.merge("test.h", 1, index); + } + + /// Same canonical context. + { + RawIndex index; + auto& symbol = index.get_symbol(1); + index.add_relation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); + auto context = base.merge("test2.h", 1, index); + EXPECT_EQ(context.hctx_id, 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(base.header_context_count(), 2); + EXPECT_EQ(base.canonical_context_count(), 1); + EXPECT_EQ(base.file_count(), 2); + } + + /// New canonical context. + { + RawIndex index; + auto& symbol = index.get_symbol(1); + index.add_relation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); + + auto context = base.merge("test3.h", 1, index); + EXPECT_EQ(context.hctx_id, 2); + EXPECT_EQ(context.cctx_id, 1); + EXPECT_EQ(base.header_context_count(), 3); + EXPECT_EQ(base.canonical_context_count(), 2); + EXPECT_EQ(base.file_count(), 3); + } + + /// New canonical context. + { + RawIndex index; + auto& symbol = index.get_symbol(1); + index.add_relation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); + index.add_relation(symbol, Relation{.kind = RelationKind::Declaration, .range = range}); + + auto context = base.merge("test4.h", 1, index); + EXPECT_EQ(context.hctx_id, 3); + EXPECT_EQ(context.cctx_id, 2); + EXPECT_EQ(base.header_context_count(), 4); + EXPECT_EQ(base.canonical_context_count(), 3); + EXPECT_EQ(base.file_count(), 4); + } +} + +TEST(HeaderIndex, MergeReuse) { + LocalSourceRange range = {0, 0}; + HeaderIndex base; + + RawIndex index; + index.add_occurrence(range, 1); + base.merge("test.h", 1, index); + + /// Same context + RawIndex index2; + index2.add_occurrence(range, 1); + base.merge("test.h", 2, index2); + EXPECT_EQ(base.canonical_context_count(), 1); + + /// New Context + RawIndex index3; + index3.add_occurrence(range, 2); + base.merge("test.h", 3, index3); + EXPECT_EQ(base.canonical_context_count(), 2); + + /// Same Context + RawIndex index4; + index4.add_occurrence(range, 1); + base.merge("test.h", 4, index4); + EXPECT_EQ(base.canonical_context_count(), 2); +} + +TEST(HeaderIndex, MergeComplex) { + RawIndex index1; + { + auto& symbol = index1.get_symbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1948, 1962}, + .target_symbol = 8426725836700ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.get_symbol(17660704465322401956ull); + symbol.name = "__need_offsetof"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4720, 4735}, + .target_symbol = 0ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3433, 3448}, + .target_symbol = 14809047240041ull, + }); + } + { + auto& symbol = index1.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1734, 1747}, + .target_symbol = 7503307867846ull, + }); + } + { + auto& symbol = index1.get_symbol(447841485290421751ull); + symbol.name = "__need_max_align_t"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3399, 3417}, + .target_symbol = 14675903253831ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4592, 4610}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.get_symbol(3892980363519083943ull); + symbol.name = "__need_nullptr_t"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3138, 3154}, + .target_symbol = 13546326854722ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4328, 4344}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3005, 3016}, + .target_symbol = 12953621367741ull, + }); + } + { + auto& symbol = index1.get_symbol(13138966718646481517ull); + symbol.name = "__cplusplus"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3367, 3378}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.get_symbol(4048786579988097027ull); + symbol.name = "__need_ptrdiff_t"; + symbol.kind = SymbolKind::Macro; + + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1709, 1725}, + .target_symbol = 7408818587309ull, + }); + index1.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3747, 3763}, + .target_symbol = 0ull, + }); + } + + RawIndex index2; + { + auto& symbol = index2.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index2.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index2.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index2.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index3; + + { + auto& symbol = index3.get_symbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index3.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index3.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index3.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index3.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index3.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index4; + { + auto& symbol = index4.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index4.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index4.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index4.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index5; + { + auto& symbol = index5.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index5.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + + RawIndex index6; + { + auto& symbol = index6.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index6.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index6.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index6.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index7; + { + auto& symbol = index7.get_symbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1948, 1962}, + .target_symbol = 8426725836700ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.get_symbol(17660704465322401956ull); + symbol.name = "__need_offsetof"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4720, 4735}, + .target_symbol = 0ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3433, 3448}, + .target_symbol = 14809047240041ull, + }); + } + { + auto& symbol = index7.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1734, 1747}, + .target_symbol = 7503307867846ull, + }); + } + { + auto& symbol = index7.get_symbol(447841485290421751ull); + symbol.name = "__need_max_align_t"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3399, 3417}, + .target_symbol = 14675903253831ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4592, 4610}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.get_symbol(3892980363519083943ull); + symbol.name = "__need_nullptr_t"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3138, 3154}, + .target_symbol = 13546326854722ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4328, 4344}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3005, 3016}, + .target_symbol = 12953621367741ull, + }); + } + { + auto& symbol = index7.get_symbol(13138966718646481517ull); + symbol.name = "__cplusplus"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3367, 3378}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.get_symbol(4048786579988097027ull); + symbol.name = "__need_ptrdiff_t"; + symbol.kind = SymbolKind::Macro; + + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1709, 1725}, + .target_symbol = 7408818587309ull, + }); + index7.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3747, 3763}, + .target_symbol = 0ull, + }); + } + + RawIndex index8; + { + auto& symbol = index8.get_symbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index8.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index8.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index8.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index8.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index8.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index9; + { + auto& symbol = index9.get_symbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index9.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + RawIndex index10; + { + auto& symbol = index10.get_symbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index10.add_relation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + + { + HeaderIndex base; + auto context = base.merge("main.cpp", 56, index1); + EXPECT_EQ(context, HeaderIndex::HeaderContext{56, 0, 0}); + + context = base.merge("main.cpp", 83, index2); + EXPECT_EQ(context, HeaderIndex::HeaderContext{83, 1, 1}); + + context = base.merge("main.cpp", 87, index3); + EXPECT_EQ(context, HeaderIndex::HeaderContext{87, 2, 2}); + + context = base.merge("main.cpp", 118, index4); + EXPECT_EQ(context, HeaderIndex::HeaderContext{118, 3, 1}); + + context = base.merge("main.cpp", 135, index5); + EXPECT_EQ(context, HeaderIndex::HeaderContext{135, 4, 3}); + + context = base.merge("main.cpp", 147, index6); + EXPECT_EQ(context, HeaderIndex::HeaderContext{147, 5, 1}); + + context = base.merge("main.cpp", 150, index7); + EXPECT_EQ(context, HeaderIndex::HeaderContext{150, 6, 0}); + + context = base.merge("main.cpp", 178, index8); + EXPECT_EQ(context, HeaderIndex::HeaderContext{178, 7, 2}); + + context = base.merge("main.cpp", 212, index9); + EXPECT_EQ(context, HeaderIndex::HeaderContext{212, 8, 4}); + + context = base.merge("main.cpp", 226, index10); + EXPECT_EQ(context, HeaderIndex::HeaderContext{226, 9, 3}); + } +} + +#if 0 +/// Only for local tests. +TEST(HeaderIndex, Build) { + llvm::StringRef context = R"( +#include +)"; + + Tester tester; + tester.addMain("main.cpp", context); + tester.compile(); + + auto& AST = *tester.AST; + auto indices = RawIndex::build(AST); + + std::optional base = RawIndex(); + + llvm::StringRef path = + "/home/ykiko/C++/llvm-project/build-debug-install/lib/clang/21/include/stddef.h"; + + bool print_next = false; + + std::uint32_t id = 1; + for(auto& [fid, index]: indices) { + if(AST.getFilePath(fid) == path) { + /// println("{}", test_code(*index, id)); + id += 1; + if(!base) { + base = *std::move(index); + } else { + // if(print_next) { + // println("----------------------------------------------------------------\n"); + // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); + // } + + auto context = base->merge(*index); + + // if(print_next) { + // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); + // print_next = false; + // } + + if(context.include == 118) { + print_next = true; + } + } + } + } + + println("----------------------------------------------------------------\n"); + dump(*base, + { + .enable_contexts = true, + .enable_symbol = true, + .enable_occurrence = true, + }); +} +#endif + +} // namespace + +} // namespace clice::testing diff --git a/unittests/Index/SymbolIndex.cpp b/unittests/Index/SymbolIndex.cpp index f73c7c69..d434b1c7 100644 --- a/unittests/Index/SymbolIndex.cpp +++ b/unittests/Index/SymbolIndex.cpp @@ -3,143 +3,143 @@ namespace clice::testing { -struct SymbolIndex : ::testing::Test, Tester { - index::Shared> indices; - index::SymbolIndex index = {nullptr, 0}; - - void run(llvm::StringRef code) { - addMain("main.cpp", code); - Tester::compile(); - indices = index::SymbolIndex::build(*AST); - index = { - indices[AST->getInterestedFile()].data(), - static_cast(indices[AST->getInterestedFile()].size()), - }; - } - - void EXPECT_OCCURENCE(int index, - llvm::StringRef pos, - llvm::StringRef name, - LocationChain chain = LocationChain()) { - auto occurrence = this->index.occurrences()[index]; - EXPECT_EQ(occurrence.range().begin, this->offset(pos), chain); - EXPECT_EQ(occurrence.symbol().name(), name, chain); - } - - index::Symbol EXPECT_SYMBOL(llvm::StringRef name, - SymbolKind kind, - LocationChain chain = LocationChain()) { - for(auto symbol: index.symbols()) { - if(symbol.name() == name) { - EXPECT_EQ(symbol.kind(), kind, chain); - return symbol; - } - } - - EXPECT_FAILURE("can not find symbol", chain); - std::unreachable(); - } - - void EXPECT_RELATION(llvm::StringRef name, - RelationKind kind, - llvm::StringRef pos, - LocationChain chain = LocationChain()) { - bool foundSymbol = false; - for(auto symbol: index.symbols()) { - if(symbol.name() == name) { - foundSymbol = true; - - bool foundRelation = false; - for(auto relation: symbol.relations()) { - if((relation.kind() & kind) && relation.range().begin == offset(pos)) { - foundRelation = true; - break; - } - } - foundRelation = true; - EXPECT_EQ(foundRelation, true, chain); - - break; - } - } - EXPECT_EQ(foundSymbol, true, chain); - } -}; +// struct SymbolIndex : ::testing::Test, Tester { +// index::Shared> indices; +// index::SymbolIndex index = {nullptr, 0}; +// +// void run(llvm::StringRef code) { +// addMain("main.cpp", code); +// Tester::compile(); +// indices = index::SymbolIndex::build(*AST); +// index = { +// indices[AST->getInterestedFile()].data(), +// static_cast(indices[AST->getInterestedFile()].size()), +// }; +// } +// +// void EXPECT_OCCURENCE(int index, +// llvm::StringRef pos, +// llvm::StringRef name, +// LocationChain chain = LocationChain()) { +// auto occurrence = this->index.occurrences()[index]; +// EXPECT_EQ(occurrence.range().begin, this->offset(pos), chain); +// EXPECT_EQ(occurrence.symbol().name(), name, chain); +// } +// +// index::Symbol EXPECT_SYMBOL(llvm::StringRef name, +// SymbolKind kind, +// LocationChain chain = LocationChain()) { +// for(auto symbol: index.symbols()) { +// if(symbol.name() == name) { +// EXPECT_EQ(symbol.kind(), kind, chain); +// return symbol; +// } +// } +// +// EXPECT_FAILURE("can not find symbol", chain); +// std::unreachable(); +// } +// +// void EXPECT_RELATION(llvm::StringRef name, +// RelationKind kind, +// llvm::StringRef pos, +// LocationChain chain = LocationChain()) { +// bool foundSymbol = false; +// for(auto symbol: index.symbols()) { +// if(symbol.name() == name) { +// foundSymbol = true; +// +// bool foundRelation = false; +// for(auto relation: symbol.relations()) { +// if((relation.kind() & kind) && relation.range().begin == offset(pos)) { +// foundRelation = true; +// break; +// } +// } +// foundRelation = true; +// EXPECT_EQ(foundRelation, true, chain); +// +// break; +// } +// } +// EXPECT_EQ(foundSymbol, true, chain); +// } +// }; namespace { -TEST_F(SymbolIndex, Symbols) { - llvm::StringRef code = R"( -int x = 1; -int y = 2; +// TEST_F(SymbolIndex, Symbols) { +// llvm::StringRef code = R"( +// int x = 1; +// int y = 2; +// +// struct Foo {}; +//)"; +// +// run(code); +// +// EXPECT_EQ(index.symbols().length(), 3); +// EXPECT_EQ(index.path(), "main.cpp"); +// EXPECT_EQ(index.content(), code); +// } +// +// TEST_F(SymbolIndex, Occurrences) { +// llvm::StringRef code = R"( +// int @x = 1; +// int @y = 2; +// +// struct @Foo {}; +//)"; +// run(code); +// +// EXPECT_EQ(index.occurrences().length(), 3); +// EXPECT_EQ(index.path(), "main.cpp"); +// +// /// FIXME: Use StringMap to store sources. +// EXPECT_EQ(index.content(), sources[0]); +// +// EXPECT_OCCURENCE(0, "x", "x"); +// EXPECT_OCCURENCE(1, "y", "y"); +// EXPECT_OCCURENCE(2, "Foo", "Foo"); +// } -struct Foo {}; -)"; +// TEST_F(SymbolIndex, Relations) { +// llvm::StringRef code = R"( +// int $(1)x = 1; +// +// int main() { +// $(2)x = 2; +// } +//)"; +// run(code); +// +// EXPECT_SYMBOL("x", SymbolKind::Variable); +// EXPECT_RELATION("x", RelationKind::Definition, "1"); +// EXPECT_RELATION("x", RelationKind::Reference, "2"); +// } - run(code); - - EXPECT_EQ(index.symbols().length(), 3); - EXPECT_EQ(index.path(), "main.cpp"); - EXPECT_EQ(index.content(), code); -} - -TEST_F(SymbolIndex, Occurrences) { - llvm::StringRef code = R"( -int @x = 1; -int @y = 2; - -struct @Foo {}; -)"; - run(code); - - EXPECT_EQ(index.occurrences().length(), 3); - EXPECT_EQ(index.path(), "main.cpp"); - - /// FIXME: Use StringMap to store sources. - EXPECT_EQ(index.content(), sources[0]); - - EXPECT_OCCURENCE(0, "x", "x"); - EXPECT_OCCURENCE(1, "y", "y"); - EXPECT_OCCURENCE(2, "Foo", "Foo"); -} - -TEST_F(SymbolIndex, Relations) { - llvm::StringRef code = R"( -int $(1)x = 1; - -int main() { - $(2)x = 2; -} -)"; - run(code); - - EXPECT_SYMBOL("x", SymbolKind::Variable); - EXPECT_RELATION("x", RelationKind::Definition, "1"); - EXPECT_RELATION("x", RelationKind::Reference, "2"); -} - -TEST_F(SymbolIndex, LocateSymbol) { - llvm::StringRef code = R"( -int $(1)x = 1; - -int main() { - $(2)x = 2; -} -)"; - run(code); - - auto symbols = index.locateSymbol(offset("1")); - EXPECT_EQ(symbols.size(), 1); - EXPECT_EQ(symbols[0].name(), "x"); - - symbols = index.locateSymbol(offset("2")); - EXPECT_EQ(symbols.size(), 1); - EXPECT_EQ(symbols[0].name(), "x"); - - auto symbol = index.locateSymbol(symbols[0].id()); - EXPECT_EQ(symbol.has_value(), true); - EXPECT_EQ(symbols[0].name(), "x"); -} +// TEST_F(SymbolIndex, LocateSymbol) { +// llvm::StringRef code = R"( +// int $(1)x = 1; +// +// int main() { +// $(2)x = 2; +// } +//)"; +// run(code); +// +// auto symbols = index.locateSymbol(offset("1")); +// EXPECT_EQ(symbols.size(), 1); +// EXPECT_EQ(symbols[0].name(), "x"); +// +// symbols = index.locateSymbol(offset("2")); +// EXPECT_EQ(symbols.size(), 1); +// EXPECT_EQ(symbols[0].name(), "x"); +// +// auto symbol = index.locateSymbol(symbols[0].id()); +// EXPECT_EQ(symbol.has_value(), true); +// EXPECT_EQ(symbols[0].name(), "x"); +// } // FIXME: headers not found // @@ -148,12 +148,12 @@ int main() { /// #include /// )"; /// run(code); -/// +/// /// for(auto& [fid, index]: indices) { /// if(index.size() == 0) { /// continue; /// } -/// +/// /// index::SymbolIndex sindex(index.data(), index.size()); /// auto json = sindex.toJSON(); /// } diff --git a/unittests/Index/SymbolIndex2.cpp b/unittests/Index/SymbolIndex2.cpp deleted file mode 100644 index 0e5aec2e..00000000 --- a/unittests/Index/SymbolIndex2.cpp +++ /dev/null @@ -1,905 +0,0 @@ -#include "Test/CTest.h" -#include "Index/Index2.h" - -namespace clice::testing { - -namespace { - -using namespace clice::index::memory2; - -struct DumpConfig { - bool enable_total = false; - bool enable_contexts = false; - bool enable_symbol = false; - bool enable_occurrence = false; -}; - -void dump(SymbolIndex& index, DumpConfig config) { - if(config.enable_total) { - println("\n---------------------------Total Info---------------------------"); - println("file count: {}", index.file_count()); - println("header context count: {}", index.header_context_count()); - println("canonical context count: {}", index.canonical_context_count()); - println("symbol count: {}", index.symbols.size()); - println("occurrence count: {}", index.occurrences.size()); - } - - if(config.enable_contexts) { - println("\n--------------------------Contexts Info-------------------------"); - for(auto& [path, contents]: index.header_contexts) { - println("{}:", path); - for(auto& context: contents) { - println(" include: {}, hctx_id: {}, cctx_id: {}", - context.include, - context.hctx_id, - context.cctx_id); - } - } - } - - if(config.enable_symbol) { - println("\n-------------------------Symbols Info--------------------------"); - for(auto& [symbol_id, symbol]: index.symbols) { - clice::println("symbol: {}, kind: {}", symbol.name, symbol.kind.name()); - for(auto& relation: symbol.relations) { - if(relation.ctx.is_dependent()) { - auto context = index.dependent_elem_states[relation.ctx.offset()]; - clice::println(" kind: {}, context: {:#b}", - relation.kind.name(), - context.to_ulong()); - } - } - } - } - - if(config.enable_occurrence) { - println("\n-------------------------Occurrences Info--------------------------"); - for(auto& [range, occurrences]: index.occurrences) { - println("occurrence: {} {}", range.begin, range.end); - for(auto& occurrence: occurrences) { - if(occurrence.ctx.is_dependent()) { - auto context = index.dependent_elem_states[occurrence.ctx.offset()]; - println(" target: {}, context: {:#b}", - occurrence.target_symbol, - context.to_ulong()); - } - } - } - } -} - -/// TODO: We should have a more clean way to save test data(like json), rather than out put code -/// directly. -std::string test_code(SymbolIndex& index, std::uint32_t id) { - std::string code; - std::string index_name = std::format("index{}", id); - code += std::format("SymbolIndex {};", index_name); - auto& [path, contexts] = *index.header_contexts.begin(); - code += std::format(R"({}.add_context("{}", {});)", index_name, path, contexts[0].include); - code += "\n"; - - for(auto& [symbol_id, symbol]: index.symbols) { - code += "{"; - code += std::format(R"( - auto& symbol = {}.getSymbol({}ull); - symbol.name = "{}"; - symbol.kind = SymbolKind::{}; - )", - index_name, - symbol_id, - symbol.name, - symbol.kind.name()); - code += "\n"; - - for(auto& relation: symbol.relations) { - code += std::format(R"({}.addRelation( - symbol, - Relation{{ - .kind = RelationKind::{}, - .range = {{ {}, {} }}, - .target_symbol = {}ull, - }} - );)", - index_name, - relation.kind.name(), - relation.range.begin, - relation.range.end, - relation.target_symbol); - } - code += "}\n"; - } - - // for(auto& [range, occurrence_group]: index.occurrences) { - // code += "{"; - // code += std::format("LocalSourceRange range{{ {}, {} }};\n", range.begin, range.end); - // for(auto& occurrence: occurrence_group) { - // code += std::format("index.addOccurrence(range, {});\n", occurrence.target_symbol); - // } - // code += "}"; - // } - - return code; -} - -TEST(SymbolIndex2, AddRemoveContext) { - SymbolIndex index; - - { - auto context = index.add_context("test.h", 1); - EXPECT_EQ(context.cctx_id, 0); - EXPECT_EQ(context.hctx_id, 0); - EXPECT_EQ(index.header_context_count(), 1); - EXPECT_EQ(index.canonical_context_count(), 1); - } - - { - auto context = index.add_context("test.h", 2); - EXPECT_EQ(context.cctx_id, 1); - EXPECT_EQ(context.hctx_id, 1); - EXPECT_EQ(index.header_context_count(), 2); - EXPECT_EQ(index.canonical_context_count(), 2); - } - - EXPECT_EQ(index.file_count(), 1); - - { - auto context = index.add_context("test2.h", 1); - EXPECT_EQ(context.cctx_id, 2); - EXPECT_EQ(context.hctx_id, 2); - EXPECT_EQ(index.header_context_count(), 3); - EXPECT_EQ(index.canonical_context_count(), 3); - } - - EXPECT_EQ(index.file_count(), 2); - - index.remove("test.h"); - - EXPECT_EQ(index.header_context_count(), 1); - EXPECT_EQ(index.canonical_context_count(), 1); - - /// Test reuse context id and context ref. - { - auto context = index.add_context("test3.h", 1); - EXPECT_EQ(context.cctx_id, 0); - EXPECT_EQ(context.hctx_id, 0); - EXPECT_EQ(index.header_context_count(), 2); - EXPECT_EQ(index.canonical_context_count(), 2); - } - - { - index.add_context("test4.h", 1); - EXPECT_EQ(index.header_context_count(), 3); - EXPECT_EQ(index.canonical_context_count(), 3); - } - - { - index.add_context("test5.h", 1); - EXPECT_EQ(index.header_context_count(), 4); - EXPECT_EQ(index.canonical_context_count(), 4); - } - - EXPECT_EQ(index.file_count(), 4); -} - -TEST(SymbolIndex2, SymbolInsert) { - SymbolIndex index; - index.add_context("test.h", 1); - index.addOccurrence({1, 2}, 1); -} - -TEST(SymbolIndex2, MergeEmpty) { - SymbolIndex index; - index.add_context("test.h", 1); - index.addOccurrence({1, 2}, 1); - - SymbolIndex index2; - index2.add_context("test2.h", 1); - - index.merge(index2); - EXPECT_EQ(index.header_context_count(), 2); - EXPECT_EQ(index.canonical_context_count(), 2); - EXPECT_EQ(index.file_count(), 2); - - SymbolIndex index3; - index3.add_context("test3.h", 1); - - index.merge(index3); - EXPECT_EQ(index.header_context_count(), 3); - EXPECT_EQ(index.canonical_context_count(), 2); - EXPECT_EQ(index.file_count(), 3); -} - -TEST(SymbolIndex2, MergeOccurrence) { - SymbolIndex index; - index.add_context("test.h", 1); - index.addOccurrence({1, 2}, 1); - - SymbolIndex index2; - index2.add_context("test2.h", 1); - index2.addOccurrence({1, 2}, 1); - - index.merge(index2); - EXPECT_EQ(index.header_context_count(), 2); - EXPECT_EQ(index.canonical_context_count(), 1); - EXPECT_EQ(index.file_count(), 2); - - SymbolIndex index3; - index3.add_context("test3.h", 1); - index3.addOccurrence({1, 2}, 2); - - index.merge(index3); - EXPECT_EQ(index.header_context_count(), 3); - EXPECT_EQ(index.canonical_context_count(), 2); - EXPECT_EQ(index.file_count(), 3); -} - -TEST(SymbolIndex2, MergeSymbol) { - LocalSourceRange range = {0, 0}; - - SymbolIndex base; - { - auto context = base.add_context("test.h", 1); - auto& symbol = base.getSymbol(1); - base.addRelation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); - } - - /// Same canonical context. - { - SymbolIndex index; - index.add_context("test2.h", 1); - auto& symbol = index.getSymbol(1); - index.addRelation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); - - auto context = base.merge(index); - EXPECT_EQ(context.hctx_id, 1); - EXPECT_EQ(context.cctx_id, 0); - EXPECT_EQ(base.header_context_count(), 2); - EXPECT_EQ(base.canonical_context_count(), 1); - EXPECT_EQ(base.file_count(), 2); - } - - /// New canonical context. - { - SymbolIndex index; - index.add_context("test3.h", 1); - auto& symbol = index.getSymbol(1); - index.addRelation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); - - auto context = base.merge(index); - EXPECT_EQ(context.hctx_id, 2); - EXPECT_EQ(context.cctx_id, 1); - EXPECT_EQ(base.header_context_count(), 3); - EXPECT_EQ(base.canonical_context_count(), 2); - EXPECT_EQ(base.file_count(), 3); - } - - /// New canonical context. - { - SymbolIndex index; - index.add_context("test4.h", 1); - auto& symbol = index.getSymbol(1); - index.addRelation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); - index.addRelation(symbol, Relation{.kind = RelationKind::Declaration, .range = range}); - - auto context = base.merge(index); - EXPECT_EQ(context.hctx_id, 3); - EXPECT_EQ(context.cctx_id, 2); - EXPECT_EQ(base.header_context_count(), 4); - EXPECT_EQ(base.canonical_context_count(), 3); - EXPECT_EQ(base.file_count(), 4); - } -} - -TEST(SymbolIndex2, MergeReuse) { - LocalSourceRange range = {0, 0}; - SymbolIndex index; - index.add_context("test.h", 1); - index.addOccurrence(range, 1); - - SymbolIndex index2; - index2.add_context("test.h", 2); - index2.addOccurrence(range, 1); - - SymbolIndex index3; - index3.add_context("test.h", 3); - index3.addOccurrence(range, 2); - - SymbolIndex index4; - index4.add_context("test.h", 4); - index4.addOccurrence(range, 1); - - /// Same context - index.merge(index2); - EXPECT_EQ(index.canonical_context_count(), 1); - - /// New Context - index.merge(index3); - EXPECT_EQ(index.canonical_context_count(), 2); - - /// Same Context - index.merge(index4); - EXPECT_EQ(index.canonical_context_count(), 2); -} - -TEST(SymbolIndex2, MergeComplex) { - SymbolIndex index1; - index1.add_context("main.cpp", 56); - { - auto& symbol = index1.getSymbol(5617328926567294902ull); - symbol.name = "__need_wchar_t"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1948, 1962}, - .target_symbol = 8426725836700ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4100, 4114}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index1.getSymbol(17660704465322401956ull); - symbol.name = "__need_offsetof"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4720, 4735}, - .target_symbol = 0ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3433, 3448}, - .target_symbol = 14809047240041ull, - }); - } - { - auto& symbol = index1.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1734, 1747}, - .target_symbol = 7503307867846ull, - }); - } - { - auto& symbol = index1.getSymbol(447841485290421751ull); - symbol.name = "__need_max_align_t"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3399, 3417}, - .target_symbol = 14675903253831ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4592, 4610}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index1.getSymbol(3892980363519083943ull); - symbol.name = "__need_nullptr_t"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3138, 3154}, - .target_symbol = 13546326854722ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4328, 4344}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index1.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3005, 3016}, - .target_symbol = 12953621367741ull, - }); - } - { - auto& symbol = index1.getSymbol(13138966718646481517ull); - symbol.name = "__cplusplus"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3367, 3378}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index1.getSymbol(4048786579988097027ull); - symbol.name = "__need_ptrdiff_t"; - symbol.kind = SymbolKind::Macro; - - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1709, 1725}, - .target_symbol = 7408818587309ull, - }); - index1.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3747, 3763}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index2; - index2.add_context("main.cpp", 83); - { - auto& symbol = index2.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index2.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index2.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index2.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index3; - index3.add_context("main.cpp", 87); - { - auto& symbol = index3.getSymbol(5617328926567294902ull); - symbol.name = "__need_wchar_t"; - symbol.kind = SymbolKind::Macro; - - index3.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4100, 4114}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index3.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index3.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index3.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index3.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index4; - index4.add_context("main.cpp", 118); - { - auto& symbol = index4.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index4.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index4.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index4.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index5; - index5.add_context("main.cpp", 135); - { - auto& symbol = index5.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index5.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index6; - index6.add_context("main.cpp", 147); - { - auto& symbol = index6.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index6.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index6.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index6.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index7; - index7.add_context("main.cpp", 150); - { - auto& symbol = index7.getSymbol(5617328926567294902ull); - symbol.name = "__need_wchar_t"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1948, 1962}, - .target_symbol = 8426725836700ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4100, 4114}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index7.getSymbol(17660704465322401956ull); - symbol.name = "__need_offsetof"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4720, 4735}, - .target_symbol = 0ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3433, 3448}, - .target_symbol = 14809047240041ull, - }); - } - { - auto& symbol = index7.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1734, 1747}, - .target_symbol = 7503307867846ull, - }); - } - { - auto& symbol = index7.getSymbol(447841485290421751ull); - symbol.name = "__need_max_align_t"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3399, 3417}, - .target_symbol = 14675903253831ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4592, 4610}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index7.getSymbol(3892980363519083943ull); - symbol.name = "__need_nullptr_t"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3138, 3154}, - .target_symbol = 13546326854722ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4328, 4344}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index7.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {3005, 3016}, - .target_symbol = 12953621367741ull, - }); - } - { - auto& symbol = index7.getSymbol(13138966718646481517ull); - symbol.name = "__cplusplus"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3367, 3378}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index7.getSymbol(4048786579988097027ull); - symbol.name = "__need_ptrdiff_t"; - symbol.kind = SymbolKind::Macro; - - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Definition, - .range = {1709, 1725}, - .target_symbol = 7408818587309ull, - }); - index7.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3747, 3763}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index8; - index8.add_context("main.cpp", 178); - { - auto& symbol = index8.getSymbol(5617328926567294902ull); - symbol.name = "__need_wchar_t"; - symbol.kind = SymbolKind::Macro; - - index8.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4100, 4114}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index8.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index8.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - { - auto& symbol = index8.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index8.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index9; - index9.add_context("main.cpp", 212); - { - auto& symbol = index9.getSymbol(6389328935281374692ull); - symbol.name = "__need_NULL"; - symbol.kind = SymbolKind::Macro; - - index9.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {4212, 4223}, - .target_symbol = 0ull, - }); - } - - SymbolIndex index10; - index10.add_context("main.cpp", 226); - { - auto& symbol = index10.getSymbol(12199573421319547529ull); - symbol.name = "__need_size_t"; - symbol.kind = SymbolKind::Macro; - - index10.addRelation(symbol, - Relation{ - .kind = RelationKind::Reference, - .range = {3867, 3880}, - .target_symbol = 0ull, - }); - } - - { - SymbolIndex base; - auto context = base.merge(index1); - EXPECT_EQ(context, SymbolIndex::HeaderContext{56, 0, 0}); - - context = base.merge(index2); - EXPECT_EQ(context, SymbolIndex::HeaderContext{83, 1, 1}); - - context = base.merge(index3); - EXPECT_EQ(context, SymbolIndex::HeaderContext{87, 2, 2}); - - context = base.merge(index4); - EXPECT_EQ(context, SymbolIndex::HeaderContext{118, 3, 1}); - - context = base.merge(index5); - EXPECT_EQ(context, SymbolIndex::HeaderContext{135, 4, 3}); - - context = base.merge(index6); - EXPECT_EQ(context, SymbolIndex::HeaderContext{147, 5, 1}); - - context = base.merge(index7); - EXPECT_EQ(context, SymbolIndex::HeaderContext{150, 6, 0}); - - context = base.merge(index8); - EXPECT_EQ(context, SymbolIndex::HeaderContext{178, 7, 2}); - - context = base.merge(index9); - EXPECT_EQ(context, SymbolIndex::HeaderContext{212, 8, 4}); - - context = base.merge(index10); - EXPECT_EQ(context, SymbolIndex::HeaderContext{226, 9, 3}); - } -} - -#if 0 -/// Only for local tests. -TEST(SymbolIndex2, Build) { - llvm::StringRef context = R"( -#include -)"; - - Tester tester; - tester.addMain("main.cpp", context); - tester.compile(); - - auto& AST = *tester.AST; - auto indices = SymbolIndex::build(AST); - - std::optional base = SymbolIndex(); - - llvm::StringRef path = - "/home/ykiko/C++/llvm-project/build-debug-install/lib/clang/21/include/stddef.h"; - - bool print_next = false; - - std::uint32_t id = 1; - for(auto& [fid, index]: indices) { - if(AST.getFilePath(fid) == path) { - /// println("{}", test_code(*index, id)); - id += 1; - if(!base) { - base = *std::move(index); - } else { - // if(print_next) { - // println("----------------------------------------------------------------\n"); - // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); - // } - - auto context = base->merge(*index); - - // if(print_next) { - // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); - // print_next = false; - // } - - if(context.include == 118) { - print_next = true; - } - } - } - } - - println("----------------------------------------------------------------\n"); - dump(*base, - { - .enable_contexts = true, - .enable_symbol = true, - .enable_occurrence = true, - }); -} -#endif - -} // namespace - -} // namespace clice::testing