diff --git a/CMakeLists.txt b/CMakeLists.txt index a4b92ac5..50127c78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,7 +131,7 @@ if(CLICE_ENABLE_TEST) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG main + GIT_TAG v1.17.x ) FetchContent_MakeAvailable(googletest) diff --git a/include/AST/Context.h b/include/AST/Context.h new file mode 100644 index 00000000..ecea0d7c --- /dev/null +++ b/include/AST/Context.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace clice { + +struct HeaderContext { + /// Absolute file system path of the translation unit + /// that includes this header. + std::string tu; + + /// The canonical form of the compilation command used to + /// compile the translation unit. + std::string command; + + /// Zero-based ordinal of the include directive in the + /// translation unit that introduces this header. + std::uint32_t ordinal; +}; + +struct TUContext { + /// The canonical form of the compilation command for + /// this translation unit. + std::string command; +}; + +} // namespace clice diff --git a/include/Compiler/AST.h b/include/Compiler/AST.h index 911d16e6..eb7de96e 100644 --- a/include/Compiler/AST.h +++ b/include/Compiler/AST.h @@ -169,7 +169,7 @@ private: llvm::DenseMap pathCache; /// Cache for symbol id. - llvm::DenseMap symbolHashCache; + llvm::DenseMap symbolHashCache; llvm::BumpPtrAllocator pathStorage; }; diff --git a/include/Index/Contexts.h b/include/Index/Contexts.h new file mode 100644 index 00000000..67dfdcdf --- /dev/null +++ b/include/Index/Contexts.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/BitVector.h" + +namespace clice::index { + +struct Contextual { + /// The actual element id, + std::uint32_t element_id; + + constexpr inline static std::uint32_t FLAG = (1ull << 31); + + static Contextual from(bool is_dependent, std::uint32_t offset) { + Contextual ctx; + ctx.element_id = offset; + if(!is_dependent) { + ctx.element_id |= FLAG; + } + return ctx; + } + + bool is_dependent() { + return (element_id & FLAG) == 0; + } + + std::uint32_t offset() { + return element_id & ~FLAG; + } +}; + +/// A header context could be represented by file:include. +/// In the following context, hctx means "header context" and cctx means +/// "canonical context". So hcid is header context id and ccid is +/// canonical context id. +class Contexts { +public: + std::uint32_t file_count() { + return header_contexts.size(); + } + + /// The count of active header contexts in this index. + std::uint32_t header_context_count() { + return max_hctx_id - erased_hctx_ids.size(); + } + + /// The count of active canonical contexts in this index. + std::uint32_t canonical_context_count() { + return max_cctx_id - erased_cctx_ids.size(); + } + + /// Whether this contexts has only one single context. + bool is_single_header_context() { + return max_hctx_id == 1 && erased_hctx_ids.empty(); + } + + auto erased_flag() { + Bitmap map; + map.set(); + for(auto cctx_id: erased_cctx_ids) { + map.reset(cctx_id); + } + return map; + } + + + + /// Get a new header context id. + std::uint32_t alloc_hctx_id(); + + /// Get a new canonical context id. + std::uint32_t alloc_cctx_id(); + + std::uint32_t alloc_dependent_elem_id() { + auto id = dependent_elem_states.size(); + dependent_elem_states.emplace_back(false); + return id; + } + + std::uint32_t alloc_independent_elem_id() { + auto id = independent_elem_states.size(); + independent_elem_states.emplace_back(); + return id; + } + + void remove(this Contexts& self, llvm::StringRef path); + +public: + /// The max header context id. + std::uint32_t max_hctx_id = 0; + + /// The max canonical context id. + std::uint32_t max_cctx_id = 0; + + /// The erased header context id. if a header context is erased, + /// we add its id for later reusing. + std::deque erased_hctx_ids; + + /// Same as above but for canonical context id. + std::deque erased_cctx_ids; + + struct HeaderContext { + /// The include location id of this header context. + std::uint32_t include; + + /// The header context id of this header context. + std::uint32_t hctx_id; + + /// The canonical context id of this header context. + std::uint32_t cctx_id; + }; + + /// A map between source file path and its header contexts. + llvm::StringMap> header_contexts; + + /// A map between canonical context id and corresponding ref counts + /// referenced by header contexts. + llvm::SmallVector cctx_hctx_refs; + + /// A map between canonical context id and corresponding ref counts + /// referenced by contextual elements. + llvm::SmallVector cctx_element_refs; + + using Bitmap = std::bitset<64>; /// use llvm::BitVector? + + /// A map between dependent element id and its state, for dependent element + /// we use bitmap to store states. Each bit in bitmap represents whether + /// this element occurs in corresponding canonical context id. + llvm::SmallVector dependent_elem_states; + + /// A map between independent element id and its state, for independent element + /// we directly store the header context ids that it occurs in. + std::vector> independent_elem_states; +}; + +} // namespace clice::index diff --git a/include/Index/IncludeGraph.h b/include/Index/IncludeGraph.h new file mode 100644 index 00000000..399c9c08 --- /dev/null +++ b/include/Index/IncludeGraph.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include "AST/SourceCode.h" +#include "llvm/ADT/DenseMap.h" + +namespace clice { +class ASTInfo; +} + +namespace clice::index { + +struct IncludeLocation { + /// The file path of the include directive. + std::uint32_t path = -1; + + /// The line number of the include directive, 1-based. + std::uint32_t line = -1; + + /// The include location that introduces this file. + std::uint32_t include = -1; +}; + +struct IncludeGraph { + /// If a header file doesn't have a #pragma once or guard macro, + /// each inclusion of it will introduce a new header context, we + /// don't want to save its path repeatedly, so cache it here. + std::vector paths; + + /// All include locations in this tu. + std::vector locations; + + /// Each `FileID` represents a new header context and is introduced + /// by a new include directive. So a include directive is a new header + /// context. A map between FileID and its include location. + llvm::DenseMap file_table; + + static IncludeGraph from(ASTInfo& AST); + + std::string getPath(std::uint32_t path_ref) const { + assert(path_ref < paths.size()); + return paths[path_ref]; + } + + std::uint32_t getInclude(clang::FileID fid) { + auto it = file_table.find(fid); + assert(it != file_table.end()); + return it->second; + } +}; + +} // namespace clice::index diff --git a/include/Index/Index2.h b/include/Index/Index2.h new file mode 100644 index 00000000..6983955d --- /dev/null +++ b/include/Index/Index2.h @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include + +#include "Shared.h" +#include "Contexts.h" +#include "AST/SymbolID.h" +#include "AST/SymbolKind.h" +#include "AST/RelationKind.h" +#include "AST/SourceCode.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" + +namespace clice::index::memory2 { + +using SymbolID = std::uint64_t; + +using SourceRange = LocalSourceRange; + +struct Relation { + Contextual ctx; + RelationKind kind; + + /// The range of this relation. + SourceRange range; + + union { + SymbolID target_symbol; + SourceRange definition_range; + }; +}; + +struct Symbol { + /// The symbol id. + SymbolID id; + + /// The symbol kind. + SymbolKind kind; + + /// Whether this symbol is not visible to other tu. + bool is_tu_local = false; + + /// Whether this symbol is defined in function scope. + bool is_function_local = false; + + /// The symbol name. + std::string name; + + /// All relations of this symbol. + llvm::DenseSet relations; +}; + +struct Occurrence { + Contextual ctx; + SymbolID target_symbol; +}; + +/// For most of symbol occurrence, it has only one corresponding symbol. +using OccurrenceGroup = llvm::SmallVector; + +class SymbolIndex : public Contexts { +public: + static index::Shared> build(ASTInfo& AST); + + Symbol& getSymbol(std::uint64_t symbol_id); + + HeaderContext add_context(llvm::StringRef path, std::uint32_t include) { + assert(!merged && ""); + auto& context = header_contexts[path].emplace_back(); + context.include = include; + context.cctx_id = alloc_cctx_id(); + context.hctx_id = alloc_hctx_id(); + return context; + } + + void addRelation(Symbol& symbol, Relation relation, bool is_dependent = true); + + void addOccurrence(LocalSourceRange range, + std::int64_t target_symbol, + bool is_dependent = true); + + HeaderContext merge(this SymbolIndex& self, SymbolIndex& other); + +private: + /// Merge another index into this. Most of header file is actually + /// self contained file and has only one canonical context. This + /// is a fast path for it. + HeaderContext quick_merge(this SymbolIndex& self, SymbolIndex& other); + + /// Merge another index into this, this could handle even though + /// another has multiple canonical context. But of course slow than + /// the fast path. + /// TODO: This function hasn't been implemented. + void slow_merge(this SymbolIndex& self, SymbolIndex& other); + +public: + /// Whether this has been merged with other files. + bool merged = false; + + /// All symbols in this index. + llvm::DenseMap symbols; + + /// All occurrences in this index. + llvm::DenseMap occurrences; +}; + +} // namespace clice::index::memory2 + +namespace llvm { + +template +unsigned dense_hash(const Ts&... ts) { + return llvm::DenseMapInfo>::getHashValue(std::tuple{ts...}); +} + +template <> +struct DenseMapInfo { + using R = clice::LocalSourceRange; + + inline static R getEmptyKey() { + return R(0, -1); + } + + inline static R getTombstoneKey() { + return R(-1, 0); + } + + static auto getHashValue(const R& r) { + return dense_hash(r.begin, r.end); + } + + static bool isEqual(const R& lhs, const R& rhs) { + return lhs == rhs; + } +}; + +template <> +struct DenseMapInfo { + using R = clice::index::memory2::Relation; + + inline static R getEmptyKey() { + return R{ + .kind = clice::RelationKind(), + .range = clice::LocalSourceRange(0, 0), + .target_symbol = 0, + }; + } + + inline static R getTombstoneKey() { + return R{ + .kind = clice::RelationKind(), + .range = clice::LocalSourceRange(-1, -1), + .target_symbol = 0, + }; + } + + /// Contextual doen't take part in hashing and equality. + static auto getHashValue(const R& relation) { + return dense_hash(relation.kind.value(), + relation.range.begin, + relation.range.end, + relation.target_symbol); + } + + static bool isEqual(const R& lhs, const R& rhs) { + return lhs.kind == rhs.kind && lhs.range == rhs.range && + lhs.target_symbol == rhs.target_symbol; + } +}; + +} // namespace llvm diff --git a/include/Server/Indexer2.h b/include/Server/Indexer2.h new file mode 100644 index 00000000..f53a02cf --- /dev/null +++ b/include/Server/Indexer2.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include "AST/SymbolID.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" + +namespace clice { + +class Indexer { +public: + +private: + /// All paths in indexes. + std::vector paths; + + /// A map between path and its index. + llvm::StringMap pathIndex; + + /// A map between source file path and its static index file. + llvm::DenseMap mapToIndex; + + /// A map between symbol id and files that contains it. + llvm::DenseMap> invertedSymbolMap; +}; + +} // namespace clice diff --git a/include/Support/JSON.h b/include/Support/JSON.h index d74ea2b1..f6356bdf 100644 --- a/include/Support/JSON.h +++ b/include/Support/JSON.h @@ -300,32 +300,28 @@ constexpr inline bool is_optional_v> = true; template struct Serde { template - static json::Value serialize(const T& t, Serdes&&... serdes) { + static json::Value serialize(const T& t) { json::Object object; refl::foreach(t, [&](std::string_view name, const Field& field) { if constexpr(is_optional_v) { if(field) { - object.try_emplace(llvm::StringRef(name), - json::serialize(*field, std::forward(serdes)...)); + object.try_emplace(llvm::StringRef(name), json::serialize(*field)); } } else { - object.try_emplace(llvm::StringRef(name), - json::serialize(field, std::forward(serdes)...)); + object.try_emplace(llvm::StringRef(name), json::serialize(field)); } }); return object; } template - static T deserialize(const json::Value& value, Serdes&&... serdes) { + static T deserialize(const json::Value& value) { T t = {}; if constexpr(!std::is_empty_v) { assert(value.kind() == json::Value::Object && "Expect an object"); refl::foreach(t, [&](std::string_view name, auto&& member) { if(auto v = value.getAsObject()->get(llvm::StringRef(name))) { - member = json::deserialize>( - *v, - std::forward(serdes)...); + member = json::deserialize>(*v); } }); } diff --git a/src/Compiler/AST.cpp b/src/Compiler/AST.cpp index a730af21..305bc908 100644 --- a/src/Compiler/AST.cpp +++ b/src/Compiler/AST.cpp @@ -56,10 +56,10 @@ llvm::StringRef ASTInfo::getFilePath(clang::FileID fid) { auto entry = SM.getFileEntryRefForID(fid); assert(entry && "Invalid file entry"); - auto name = entry->getName(); llvm::SmallString<128> path; /// Try to get the real path of the file. + auto name = entry->getName(); if(auto error = llvm::sys::fs::real_path(name, path)) { /// If failed, use the virtual path. path = name; @@ -72,7 +72,7 @@ llvm::StringRef ASTInfo::getFilePath(clang::FileID fid) { memcpy(data, path.data(), size); data[size] = '\0'; - auto [it, inserted] = pathCache.try_emplace(fid, data, size); + auto [it, inserted] = pathCache.try_emplace(fid, llvm::StringRef(data, size)); assert(inserted && "File path already exists"); return it->second; } @@ -131,7 +131,7 @@ index::SymbolID ASTInfo::getSymbolID(const clang::NamedDecl* decl) { } index::SymbolID ASTInfo::getSymbolID(const clang::MacroInfo* macro) { - uint64_t hash; + std::uint64_t hash; auto name = getTokenSpelling(SM, macro->getDefinitionLoc()); auto iter = symbolHashCache.find(macro); if(iter != symbolHashCache.end()) { diff --git a/src/Index/Contexts.cpp b/src/Index/Contexts.cpp new file mode 100644 index 00000000..14ef593c --- /dev/null +++ b/src/Index/Contexts.cpp @@ -0,0 +1,80 @@ +#include "Index/Contexts.h" +#include "Support/Ranges.h" + +namespace clice::index { + +std::uint32_t Contexts::alloc_hctx_id() { + std::uint32_t new_hctx_id; + if(erased_hctx_ids.empty()) { + new_hctx_id = max_hctx_id; + max_hctx_id += 1; + } else { + new_hctx_id = erased_hctx_ids.front(); + erased_hctx_ids.pop_front(); + } + return new_hctx_id; +} + +std::uint32_t Contexts::alloc_cctx_id() { + std::uint32_t new_cctx_id; + if(erased_cctx_ids.empty()) { + new_cctx_id = max_cctx_id; + max_cctx_id += 1; + cctx_hctx_refs.emplace_back(1); + cctx_element_refs.emplace_back(0); + } else { + new_cctx_id = erased_cctx_ids.front(); + erased_cctx_ids.pop_front(); + cctx_hctx_refs[new_cctx_id] = 1; + cctx_element_refs[new_cctx_id] = 0; + } + return new_cctx_id; +} + +void Contexts::remove(this Contexts& self, llvm::StringRef path) { + auto it = self.header_contexts.find(path); + + /// If no such file, nothing to do. + if(it == self.header_contexts.end()) { + return; + } + + llvm::SmallVector erased_hctx_ids; + llvm::SmallVector erased_cctx_ids; + + for(auto& context: it->second) { + erased_hctx_ids.push_back(context.hctx_id); + self.erased_hctx_ids.push_back(context.hctx_id); + + auto cctx_id = context.cctx_id; + auto& ref_count = self.cctx_hctx_refs[cctx_id]; + assert(ref_count > 0); + + /// If the ref count of the canonical context id drops to 0, + /// we need to delete it. + ref_count -= 1; + if(ref_count == 0) { + erased_cctx_ids.push_back(cctx_id); + self.erased_cctx_ids.push_back(cctx_id); + self.cctx_element_refs[cctx_id] = 0; + } + } + + self.header_contexts.erase(it); + + /// Remove all refs to this header context id. + for(auto& state: self.independent_elem_states) { + for(auto hctx_id: erased_hctx_ids) { + state.erase(hctx_id); + } + } + + /// Remove all refs to this canonical context id. + Bitmap erased_flag = self.erased_flag(); + + for(auto& state: self.dependent_elem_states) { + state &= erased_flag; + } +} + +} // namespace clice::index diff --git a/src/Index/IncludeGraph.cpp b/src/Index/IncludeGraph.cpp new file mode 100644 index 00000000..1e9dc2e3 --- /dev/null +++ b/src/Index/IncludeGraph.cpp @@ -0,0 +1,52 @@ +#include "Index/IncludeGraph.h" +#include "Compiler/AST.h" + +namespace clice::index { + +static std::uint32_t addIncludeChain(ASTInfo& AST, + clang::FileID fid, + IncludeGraph& graph, + llvm::StringMap& path_table) { + auto& SM = AST.srcMgr(); + auto& [paths, locations, file_table] = graph; + auto [iter, success] = file_table.try_emplace(fid, locations.size()); + if(!success) { + return iter->second; + } + + auto index = iter->second; + + auto includeLoc = SM.getIncludeLoc(fid); + if(includeLoc.isValid()) { + auto presumed = SM.getPresumedLoc(includeLoc, false); + locations.emplace_back(); + locations[index].line = presumed.getLine(); + + auto path = AST.getFilePath(presumed.getFileID()); + auto [iter, success] = path_table.try_emplace(path, paths.size()); + if(success) { + paths.emplace_back(path); + } + locations[index].path = iter->second; + + uint32_t include = -1; + if(presumed.getIncludeLoc().isValid()) { + include = + addIncludeChain(AST, SM.getFileID(presumed.getIncludeLoc()), graph, path_table); + } + locations[index].include = include; + } + + return index; +} + +IncludeGraph IncludeGraph::from(ASTInfo& AST) { + llvm::StringMap path_table; + IncludeGraph graph; + for(auto fid: AST.files()) { + addIncludeChain(AST, fid, graph, path_table); + } + return graph; +} + +} // namespace clice::index diff --git a/src/Index/Index2.cpp b/src/Index/Index2.cpp new file mode 100644 index 00000000..8b01f422 --- /dev/null +++ b/src/Index/Index2.cpp @@ -0,0 +1,220 @@ +#include "Index/Index2.h" +#include "Support/Ranges.h" + +namespace clice::index { + +namespace memory2 { + +/// Merge all elements from other into self. And update_context is invoked every time +/// when a element is inserted. The second argument is inserted `Contextual` in the +/// other, the first element is inserted element in the self, empty if the element +/// is new to self. +static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_context) { + /// Merge symbols from other into self. + for(auto& [symbol_id, symbol]: other.symbols) { + auto [it, success] = self.symbols.try_emplace(symbol_id, std::move(symbol)); + auto& self_symbol = it->second; + + if(success) [[unlikely]] { + /// If insert successfully, this is a new symbol and it means + /// we need update all context states of this symbol. + for(auto& relation: self_symbol.relations) { + update_context(relation.ctx, Contextual(relation.ctx), true); + } + continue; + } + + /// If self already has this symbol, try to merge all relations. + for(auto& relation: symbol.relations) { + auto [it, success] = self_symbol.relations.insert(relation); + update_context(it->ctx, Contextual(relation.ctx), success); + } + } + + for(auto& [range, occurrence_group]: other.occurrences) { + auto [it, success] = self.occurrences.try_emplace(range, std::move(occurrence_group)); + auto& self_occurrence_group = it->second; + + if(success) [[unlikely]] { + /// Insert successfully. + for(auto& occurrence: self_occurrence_group) { + update_context(occurrence.ctx, Contextual(occurrence.ctx), true); + } + continue; + } + + for(auto& occurrence: occurrence_group) { + auto i = 0; + + /// In most of cases, there is only one element in the group. + /// So don't worry about the performance. + for(auto& self_occurrence: self_occurrence_group) { + if(occurrence.target_symbol == self_occurrence.target_symbol) { + break; + } + i += 1; + } + + if(i != self_occurrence_group.size()) { + update_context(self_occurrence_group[i].ctx, Contextual(occurrence.ctx), false); + } else { + /// If not found insert new occurrence. + auto& o = self_occurrence_group.emplace_back(occurrence); + update_context(o.ctx, Contextual(occurrence.ctx), true); + } + } + } +} + +auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext { + assert(!other.merged && "quick merge could be only used for the unmerged index"); + + /// We could make sure the other has only one header context. + std::uint32_t new_hctx_id = self.alloc_hctx_id(); + + Bitmap flag = self.erased_flag(); + bool is_new_cctx = false; + std::uint32_t new_cctx_id = -1; + + llvm::SmallVector visited_elem_ids; + + /// TODO: simplify the logic of update context. + + auto update_context = [&](Contextual& self_elem, Contextual other_elem, bool is_new) { + std::uint32_t new_elem_id; + + if(is_new) { + /// If this a new element, it means that the other index must introduce + /// a new canonical context id, we don't need to do following calculation. + is_new_cctx = true; + + if(new_cctx_id == -1) { + new_cctx_id = self.alloc_cctx_id(); + } + + if(other_elem.is_dependent()) { + new_elem_id = self.alloc_dependent_elem_id(); + self.dependent_elem_states[new_elem_id].set(new_cctx_id); + } else { + new_elem_id = self.alloc_independent_elem_id(); + self.independent_elem_states[new_elem_id].insert(new_hctx_id); + } + + self_elem = Contextual::from(other_elem.is_dependent(), new_elem_id); + } else { + if(self_elem.is_dependent()) { + if(is_new_cctx) { + /// If this element is not new, but we already make sure the context is new + /// add its context. + self.dependent_elem_states[self_elem.offset()].set(new_cctx_id); + } else { + /// If this element is not new and we still cannot make sure whether this is + /// new canonical context. + flag &= self.dependent_elem_states[self_elem.offset()]; + visited_elem_ids.emplace_back(self_elem.offset()); + if(flag.none()) { + is_new_cctx = true; + } + } + } else { + self.independent_elem_states[self_elem.offset()].insert(new_hctx_id); + } + } + }; + + /// Merge all elements from other into self and calculate the bitmap state. + merge_elements(self, other, update_context); + + if(!is_new_cctx) { + assert(new_cctx_id == -1 && flag.any()); + for(auto i = 0; i < self.max_cctx_id; i++) { + if(!flag.test(i)) { + continue; + } + + if(self.cctx_element_refs[i] == other.cctx_element_refs.front()) { + new_cctx_id = i; + break; + } + } + } + + if(new_cctx_id == -1) { + new_cctx_id = self.alloc_cctx_id(); + is_new_cctx = true; + } + + if(is_new_cctx) { + /// In the end we set all visited element ids. + for(auto id: visited_elem_ids) { + self.dependent_elem_states[id].set(new_cctx_id); + } + self.cctx_element_refs[new_cctx_id] = other.cctx_element_refs.front(); + } + + auto& [path, old_contexts] = *other.header_contexts.begin(); + return self.header_contexts[path].emplace_back(HeaderContext{ + .include = old_contexts[0].include, + .hctx_id = new_hctx_id, + .cctx_id = new_cctx_id, + }); +} + +auto SymbolIndex::merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext { + return self.quick_merge(other); +} + +Symbol& SymbolIndex::getSymbol(std::uint64_t symbol_id) { + assert(canonical_context_count() == 1 && "please use merge for multiple contexts"); + if(auto it = symbols.find(symbol_id); it != symbols.end()) { + return it->second; + } + + /// If not found, create a new symbol and return it. + auto symbol_ref = symbols.size(); + auto [it, _] = symbols.try_emplace(symbol_id, Symbol{.id = symbol_id}); + return it->second; +} + +void SymbolIndex::addRelation(Symbol& symbol, Relation relation, bool is_dependent) { + assert(!merged && "add relation could be used in only not merged index"); + std::uint32_t element_id; + if(is_dependent) { + element_id = alloc_dependent_elem_id(); + dependent_elem_states[element_id].set(0); + cctx_element_refs[0] += 1; + } else { + element_id = alloc_independent_elem_id(); + independent_elem_states.emplace_back(0); + } + + relation.ctx = Contextual::from(is_dependent, element_id); + symbol.relations.insert(relation); +} + +void SymbolIndex::addOccurrence(LocalSourceRange range, + std::int64_t target_symbol, + bool is_dependent) { + assert(!merged && "add occurrence could be used in only not merged index"); + + auto& targets = occurrences[range]; + + std::uint32_t element_id; + if(is_dependent) { + element_id = alloc_dependent_elem_id(); + dependent_elem_states[element_id].set(0); + cctx_element_refs[0] += 1; + } else { + element_id = alloc_independent_elem_id(); + independent_elem_states.emplace_back(0); + } + + Occurrence occurrence; + occurrence.target_symbol = target_symbol; + occurrence.ctx = Contextual::from(is_dependent, element_id); + targets.emplace_back(occurrence); +} + +} // namespace memory2 + +} // namespace clice::index diff --git a/src/Index/SymbolIndex.cpp b/src/Index/SymbolIndex.cpp index 24f539fd..b26c0f62 100644 --- a/src/Index/SymbolIndex.cpp +++ b/src/Index/SymbolIndex.cpp @@ -305,8 +305,6 @@ public: std::numeric_limits::max(), }; - using enum RelationKind::Kind; - if(kind.isDeclOrDef()) { auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange()); assert(fid == fid2 && "Invalid definition location"); @@ -345,7 +343,6 @@ public: } private: - llvm::DenseMap symbolIDs; llvm::DenseMap builders; }; diff --git a/src/Index/SymbolIndex2.cpp b/src/Index/SymbolIndex2.cpp new file mode 100644 index 00000000..cee7378d --- /dev/null +++ b/src/Index/SymbolIndex2.cpp @@ -0,0 +1,143 @@ +#include "AST/Semantic.h" +#include "Index/Index2.h" +#include "Index/IncludeGraph.h" +#include "Support/Format.h" + +namespace clice::index::memory2 { + +class SymbolIndexBuilder : public SemanticVisitor { +public: + SymbolIndexBuilder(ASTInfo& AST) : + SemanticVisitor(AST, false), graph(IncludeGraph::from(AST)), + context_path(AST.getFilePath(SM.getMainFileID())) {} + + SymbolIndex& getIndex(clang::FileID fid) { + if(auto it = indices.find(fid); it != indices.end()) { + return *it->second; + } + + auto [it, _] = indices.try_emplace(fid, new SymbolIndex()); + auto& index = *it->second; + /// Fix me build include graph here. + index.add_context(context_path, graph.getInclude(fid)); + return index; + } + + void handleDeclOccurrence(const clang::NamedDecl* decl, + RelationKind kind, + clang::SourceLocation location) { + assert(decl && "Invalid decl"); + decl = normalize(decl); + + if(location.isMacroID()) { + auto spelling = AST.getSpellingLoc(location); + auto expansion = AST.getExpansionLoc(location); + + /// FIXME: For location from macro, we only handle the case that the + /// spelling and expansion are in the same file currently. + if(AST.getFileID(spelling) != AST.getFileID(expansion)) { + return; + } + + /// For occurrence, we always use spelling location. + location = spelling; + } + + auto [fid, range] = AST.toLocalRange(location); + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(decl); + auto& symbol = index.getSymbol(symbol_id.hash); + symbol.kind = SymbolKind::from(decl); + index.addOccurrence(range, symbol_id.hash); + } + + void handleMacroOccurrence(const clang::MacroInfo* def, + RelationKind kind, + clang::SourceLocation location) { + /// FIXME: Figure out when location is MacroID. + if(location.isMacroID()) { + return; + } + + auto [fid, range] = AST.toLocalRange(location); + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(def); + auto& symbol = index.getSymbol(symbol_id.hash); + symbol.kind = SymbolKind::Macro; + symbol.name = getTokenSpelling(SM, def->getDefinitionLoc()); + index.addOccurrence(range, symbol_id.hash); + + if(kind & RelationKind::Definition) { + auto begin = def->getDefinitionLoc(); + auto end = def->getDefinitionEndLoc(); + assert(begin.isFileID() && end.isFileID() && "Invalid location"); + auto [fid2, definition_range] = AST.toLocalRange(clang::SourceRange(begin, end)); + assert(fid == fid2 && "Invalid macro definition location"); + /// definitionLoc = builder.getLocation(range); + + index.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = range, + .definition_range = definition_range, + }); + } else { + index.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = range, + .target_symbol = 0, + }); + } + } + + void handleRelation(const clang::NamedDecl* decl, + RelationKind kind, + const clang::NamedDecl* target, + clang::SourceRange range) { + auto [fid, relationRange] = AST.toLocalExpansionRange(range); + + Relation relation{.kind = kind}; + + if(kind.isDeclOrDef()) { + auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange()); + assert(fid == fid2 && "Invalid definition location"); + relation.range = relationRange; + relation.definition_range = definitionRange; + } else if(kind.isReference()) { + relation.range = relationRange; + relation.target_symbol = 0; + } else if(kind.isBetweenSymbol()) { + auto symbol_id = AST.getSymbolID(normalize(target)); + relation.target_symbol = symbol_id.hash; + } else if(kind.isCall()) { + auto symbol_id = AST.getSymbolID(normalize(target)); + relation.range = relationRange; + relation.target_symbol = symbol_id.hash; + } else { + std::unreachable(); + } + + auto& index = getIndex(fid); + auto symbol_id = AST.getSymbolID(normalize(decl)); + auto& symbol = index.getSymbol(symbol_id.hash); + index.addRelation(symbol, relation); + } + + auto build() { + run(); + + return std::move(indices); + } + +private: + IncludeGraph graph; + std::string context_path; + llvm::DenseMap> indices; +}; + +index::Shared> SymbolIndex::build(ASTInfo& AST) { + return SymbolIndexBuilder(AST).build(); +} + +} // namespace clice::index::memory2 diff --git a/unittests/Index/SymbolIndex2.cpp b/unittests/Index/SymbolIndex2.cpp new file mode 100644 index 00000000..052d1348 --- /dev/null +++ b/unittests/Index/SymbolIndex2.cpp @@ -0,0 +1,905 @@ +#include "Test/CTest.h" +#include "Index/Index2.h" + +namespace clice::testing { + +namespace { + +using namespace clice::index::memory2; + +struct DumpConfig { + bool enable_total = false; + bool enable_contexts = false; + bool enable_symbol = false; + bool enable_occurrence = false; +}; + +void dump(SymbolIndex& index, DumpConfig config) { + if(config.enable_total) { + println("\n---------------------------Total Info---------------------------"); + println("file count: {}", index.file_count()); + println("header context count: {}", index.header_context_count()); + println("canonical context count: {}", index.canonical_context_count()); + println("symbol count: {}", index.symbols.size()); + println("occurrence count: {}", index.occurrences.size()); + } + + if(config.enable_contexts) { + println("\n--------------------------Contexts Info-------------------------"); + for(auto& [path, contents]: index.header_contexts) { + println("{}:", path); + for(auto& context: contents) { + println(" include: {}, hctx_id: {}, cctx_id: {}", + context.include, + context.hctx_id, + context.cctx_id); + } + } + } + + if(config.enable_symbol) { + println("\n-------------------------Symbols Info--------------------------"); + for(auto& [symbol_id, symbol]: index.symbols) { + println("symbol: {}, kind: {}", symbol.name, symbol.kind.name()); + for(auto& relation: symbol.relations) { + if(relation.ctx.is_dependent()) { + auto context = index.dependent_elem_states[relation.ctx.offset()]; + println(" kind: {}, context: {:#b}", + relation.kind.name(), + context.to_ulong()); + } + } + } + } + + if(config.enable_occurrence) { + println("\n-------------------------Occurrences Info--------------------------"); + for(auto& [range, occurrences]: index.occurrences) { + println("occurrence: {} {}", range.begin, range.end); + for(auto& occurrence: occurrences) { + if(occurrence.ctx.is_dependent()) { + auto context = index.dependent_elem_states[occurrence.ctx.offset()]; + println(" target: {}, context: {:#b}", + occurrence.target_symbol, + context.to_ulong()); + } + } + } + } +} + +/// TODO: We should have a more clean way to save test data(like json), rather than out put code +/// directly. +std::string test_code(SymbolIndex& index, std::uint32_t id) { + std::string code; + std::string index_name = std::format("index{}", id); + code += std::format("SymbolIndex {};", index_name); + auto& [path, contexts] = *index.header_contexts.begin(); + code += std::format(R"({}.add_context("{}", {});)", index_name, path, contexts[0].include); + code += "\n"; + + for(auto& [symbol_id, symbol]: index.symbols) { + code += "{"; + code += std::format(R"( + auto& symbol = {}.getSymbol({}ull); + symbol.name = "{}"; + symbol.kind = SymbolKind::{}; + )", + index_name, + symbol_id, + symbol.name, + symbol.kind.name()); + code += "\n"; + + for(auto& relation: symbol.relations) { + code += std::format(R"({}.addRelation( + symbol, + Relation{{ + .kind = RelationKind::{}, + .range = {{ {}, {} }}, + .target_symbol = {}ull, + }} + );)", + index_name, + relation.kind.name(), + relation.range.begin, + relation.range.end, + relation.target_symbol); + } + code += "}\n"; + } + + // for(auto& [range, occurrence_group]: index.occurrences) { + // code += "{"; + // code += std::format("LocalSourceRange range{{ {}, {} }};\n", range.begin, range.end); + // for(auto& occurrence: occurrence_group) { + // code += std::format("index.addOccurrence(range, {});\n", occurrence.target_symbol); + // } + // code += "}"; + // } + + return code; +} + +TEST(SymbolIndex2, AddRemoveContext) { + SymbolIndex index; + + { + auto context = index.add_context("test.h", 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(context.hctx_id, 0); + EXPECT_EQ(index.header_context_count(), 1); + EXPECT_EQ(index.canonical_context_count(), 1); + } + + { + auto context = index.add_context("test.h", 2); + EXPECT_EQ(context.cctx_id, 1); + EXPECT_EQ(context.hctx_id, 1); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 2); + } + + EXPECT_EQ(index.file_count(), 1); + + { + auto context = index.add_context("test2.h", 1); + EXPECT_EQ(context.cctx_id, 2); + EXPECT_EQ(context.hctx_id, 2); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 3); + } + + EXPECT_EQ(index.file_count(), 2); + + index.remove("test.h"); + + EXPECT_EQ(index.header_context_count(), 1); + EXPECT_EQ(index.canonical_context_count(), 1); + + /// Test reuse context id and context ref. + { + auto context = index.add_context("test3.h", 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(context.hctx_id, 0); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 2); + } + + { + index.add_context("test4.h", 1); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 3); + } + + { + index.add_context("test5.h", 1); + EXPECT_EQ(index.header_context_count(), 4); + EXPECT_EQ(index.canonical_context_count(), 4); + } + + EXPECT_EQ(index.file_count(), 4); +} + +TEST(SymbolIndex2, SymbolInsert) { + SymbolIndex index; + index.add_context("test.h", 1); + index.addOccurrence({1, 2}, 1); +} + +TEST(SymbolIndex2, MergeEmpty) { + SymbolIndex index; + index.add_context("test.h", 1); + index.addOccurrence({1, 2}, 1); + + SymbolIndex index2; + index2.add_context("test2.h", 1); + + index.merge(index2); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 2); + EXPECT_EQ(index.file_count(), 2); + + SymbolIndex index3; + index3.add_context("test3.h", 1); + + index.merge(index3); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 2); + EXPECT_EQ(index.file_count(), 3); +} + +TEST(SymbolIndex2, MergeOccurrence) { + SymbolIndex index; + index.add_context("test.h", 1); + index.addOccurrence({1, 2}, 1); + + SymbolIndex index2; + index2.add_context("test2.h", 1); + index2.addOccurrence({1, 2}, 1); + + index.merge(index2); + EXPECT_EQ(index.header_context_count(), 2); + EXPECT_EQ(index.canonical_context_count(), 1); + EXPECT_EQ(index.file_count(), 2); + + SymbolIndex index3; + index3.add_context("test3.h", 1); + index3.addOccurrence({1, 2}, 2); + + index.merge(index3); + EXPECT_EQ(index.header_context_count(), 3); + EXPECT_EQ(index.canonical_context_count(), 2); + EXPECT_EQ(index.file_count(), 3); +} + +TEST(SymbolIndex2, MergeSymbol) { + LocalSourceRange range = {0, 0}; + + SymbolIndex base; + { + auto context = base.add_context("test.h", 1); + auto& symbol = base.getSymbol(1); + base.addRelation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); + } + + /// Same canonical context. + { + SymbolIndex index; + index.add_context("test2.h", 1); + auto& symbol = index.getSymbol(1); + index.addRelation(symbol, Relation{.kind = RelationKind::Reference, .range = range}); + + auto context = base.merge(index); + EXPECT_EQ(context.hctx_id, 1); + EXPECT_EQ(context.cctx_id, 0); + EXPECT_EQ(base.header_context_count(), 2); + EXPECT_EQ(base.canonical_context_count(), 1); + EXPECT_EQ(base.file_count(), 2); + } + + /// New canonical context. + { + SymbolIndex index; + index.add_context("test3.h", 1); + auto& symbol = index.getSymbol(1); + index.addRelation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); + + auto context = base.merge(index); + EXPECT_EQ(context.hctx_id, 2); + EXPECT_EQ(context.cctx_id, 1); + EXPECT_EQ(base.header_context_count(), 3); + EXPECT_EQ(base.canonical_context_count(), 2); + EXPECT_EQ(base.file_count(), 3); + } + + /// New canonical context. + { + SymbolIndex index; + index.add_context("test4.h", 1); + auto& symbol = index.getSymbol(1); + index.addRelation(symbol, Relation{.kind = RelationKind::Definition, .range = range}); + index.addRelation(symbol, Relation{.kind = RelationKind::Declaration, .range = range}); + + auto context = base.merge(index); + EXPECT_EQ(context.hctx_id, 3); + EXPECT_EQ(context.cctx_id, 2); + EXPECT_EQ(base.header_context_count(), 4); + EXPECT_EQ(base.canonical_context_count(), 3); + EXPECT_EQ(base.file_count(), 4); + } +} + +TEST(SymbolIndex2, MergeReuse) { + LocalSourceRange range = {0, 0}; + SymbolIndex index; + index.add_context("test.h", 1); + index.addOccurrence(range, 1); + + SymbolIndex index2; + index2.add_context("test.h", 2); + index2.addOccurrence(range, 1); + + SymbolIndex index3; + index3.add_context("test.h", 3); + index3.addOccurrence(range, 2); + + SymbolIndex index4; + index4.add_context("test.h", 4); + index4.addOccurrence(range, 1); + + /// Same context + index.merge(index2); + EXPECT_EQ(index.canonical_context_count(), 1); + + /// New Context + index.merge(index3); + EXPECT_EQ(index.canonical_context_count(), 2); + + /// Same Context + index.merge(index4); + EXPECT_EQ(index.canonical_context_count(), 2); +} + +TEST(SymbolIndex2, MergeComplex) { + SymbolIndex index1; + index1.add_context("main.cpp", 56); + { + auto& symbol = index1.getSymbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1948, 1962}, + .target_symbol = 8426725836700ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.getSymbol(17660704465322401956ull); + symbol.name = "__need_offsetof"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4720, 4735}, + .target_symbol = 0ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3433, 3448}, + .target_symbol = 14809047240041ull, + }); + } + { + auto& symbol = index1.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1734, 1747}, + .target_symbol = 7503307867846ull, + }); + } + { + auto& symbol = index1.getSymbol(447841485290421751ull); + symbol.name = "__need_max_align_t"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3399, 3417}, + .target_symbol = 14675903253831ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4592, 4610}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.getSymbol(3892980363519083943ull); + symbol.name = "__need_nullptr_t"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3138, 3154}, + .target_symbol = 13546326854722ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4328, 4344}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3005, 3016}, + .target_symbol = 12953621367741ull, + }); + } + { + auto& symbol = index1.getSymbol(13138966718646481517ull); + symbol.name = "__cplusplus"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3367, 3378}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index1.getSymbol(4048786579988097027ull); + symbol.name = "__need_ptrdiff_t"; + symbol.kind = SymbolKind::Macro; + + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1709, 1725}, + .target_symbol = 7408818587309ull, + }); + index1.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3747, 3763}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index2; + index2.add_context("main.cpp", 83); + { + auto& symbol = index2.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index2.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index2.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index2.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index3; + index3.add_context("main.cpp", 87); + { + auto& symbol = index3.getSymbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index3.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index3.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index3.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index3.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index3.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index4; + index4.add_context("main.cpp", 118); + { + auto& symbol = index4.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index4.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index4.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index4.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index5; + index5.add_context("main.cpp", 135); + { + auto& symbol = index5.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index5.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index6; + index6.add_context("main.cpp", 147); + { + auto& symbol = index6.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index6.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index6.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index6.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index7; + index7.add_context("main.cpp", 150); + { + auto& symbol = index7.getSymbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1948, 1962}, + .target_symbol = 8426725836700ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.getSymbol(17660704465322401956ull); + symbol.name = "__need_offsetof"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4720, 4735}, + .target_symbol = 0ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3433, 3448}, + .target_symbol = 14809047240041ull, + }); + } + { + auto& symbol = index7.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1734, 1747}, + .target_symbol = 7503307867846ull, + }); + } + { + auto& symbol = index7.getSymbol(447841485290421751ull); + symbol.name = "__need_max_align_t"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3399, 3417}, + .target_symbol = 14675903253831ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4592, 4610}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.getSymbol(3892980363519083943ull); + symbol.name = "__need_nullptr_t"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3138, 3154}, + .target_symbol = 13546326854722ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4328, 4344}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {3005, 3016}, + .target_symbol = 12953621367741ull, + }); + } + { + auto& symbol = index7.getSymbol(13138966718646481517ull); + symbol.name = "__cplusplus"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3367, 3378}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index7.getSymbol(4048786579988097027ull); + symbol.name = "__need_ptrdiff_t"; + symbol.kind = SymbolKind::Macro; + + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Definition, + .range = {1709, 1725}, + .target_symbol = 7408818587309ull, + }); + index7.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3747, 3763}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index8; + index8.add_context("main.cpp", 178); + { + auto& symbol = index8.getSymbol(5617328926567294902ull); + symbol.name = "__need_wchar_t"; + symbol.kind = SymbolKind::Macro; + + index8.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4100, 4114}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index8.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index8.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + { + auto& symbol = index8.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index8.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index9; + index9.add_context("main.cpp", 212); + { + auto& symbol = index9.getSymbol(6389328935281374692ull); + symbol.name = "__need_NULL"; + symbol.kind = SymbolKind::Macro; + + index9.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {4212, 4223}, + .target_symbol = 0ull, + }); + } + + SymbolIndex index10; + index10.add_context("main.cpp", 226); + { + auto& symbol = index10.getSymbol(12199573421319547529ull); + symbol.name = "__need_size_t"; + symbol.kind = SymbolKind::Macro; + + index10.addRelation(symbol, + Relation{ + .kind = RelationKind::Reference, + .range = {3867, 3880}, + .target_symbol = 0ull, + }); + } + + { + SymbolIndex base; + auto context = base.merge(index1); + EXPECT_EQ(context, SymbolIndex::HeaderContext{56, 0, 0}); + + context = base.merge(index2); + EXPECT_EQ(context, SymbolIndex::HeaderContext{83, 1, 1}); + + context = base.merge(index3); + EXPECT_EQ(context, SymbolIndex::HeaderContext{87, 2, 2}); + + context = base.merge(index4); + EXPECT_EQ(context, SymbolIndex::HeaderContext{118, 3, 1}); + + context = base.merge(index5); + EXPECT_EQ(context, SymbolIndex::HeaderContext{135, 4, 3}); + + context = base.merge(index6); + EXPECT_EQ(context, SymbolIndex::HeaderContext{147, 5, 1}); + + context = base.merge(index7); + EXPECT_EQ(context, SymbolIndex::HeaderContext{150, 6, 0}); + + context = base.merge(index8); + EXPECT_EQ(context, SymbolIndex::HeaderContext{178, 7, 2}); + + context = base.merge(index9); + EXPECT_EQ(context, SymbolIndex::HeaderContext{212, 8, 4}); + + context = base.merge(index10); + EXPECT_EQ(context, SymbolIndex::HeaderContext{226, 9, 3}); + } +} + +#if 0 +/// Only for local tests. +TEST(SymbolIndex2, Build) { + llvm::StringRef context = R"( +#include +)"; + + Tester tester; + tester.addMain("main.cpp", context); + tester.compile(); + + auto& AST = *tester.AST; + auto indices = SymbolIndex::build(AST); + + std::optional base = SymbolIndex(); + + llvm::StringRef path = + "/home/ykiko/C++/llvm-project/build-debug-install/lib/clang/21/include/stddef.h"; + + bool print_next = false; + + std::uint32_t id = 1; + for(auto& [fid, index]: indices) { + if(AST.getFilePath(fid) == path) { + /// println("{}", test_code(*index, id)); + id += 1; + if(!base) { + base = *std::move(index); + } else { + // if(print_next) { + // println("----------------------------------------------------------------\n"); + // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); + // } + + auto context = base->merge(*index); + + // if(print_next) { + // dump(*base, {.enable_symbol = true, .enable_occurrence = false}); + // print_next = false; + // } + + if(context.include == 118) { + print_next = true; + } + } + } + } + + println("----------------------------------------------------------------\n"); + dump(*base, + { + .enable_contexts = true, + .enable_symbol = true, + .enable_occurrence = true, + }); +} +#endif + +} // namespace + +} // namespace clice::testing