Merge index from different header contexts (#121)

src/Compiler/AST.cpp

@@ -56,10 +56,10 @@ llvm::StringRef ASTInfo::getFilePath(clang::FileID fid) {
     auto entry = SM.getFileEntryRefForID(fid);
     assert(entry && "Invalid file entry");
 
-    auto name = entry->getName();
     llvm::SmallString<128> path;
 
     /// Try to get the real path of the file.
+    auto name = entry->getName();
     if(auto error = llvm::sys::fs::real_path(name, path)) {
         /// If failed, use the virtual path.
         path = name;
@@ -72,7 +72,7 @@ llvm::StringRef ASTInfo::getFilePath(clang::FileID fid) {
     memcpy(data, path.data(), size);
     data[size] = '\0';
 
-    auto [it, inserted] = pathCache.try_emplace(fid, data, size);
+    auto [it, inserted] = pathCache.try_emplace(fid, llvm::StringRef(data, size));
     assert(inserted && "File path already exists");
     return it->second;
 }
@@ -131,7 +131,7 @@ index::SymbolID ASTInfo::getSymbolID(const clang::NamedDecl* decl) {
 }
 
 index::SymbolID ASTInfo::getSymbolID(const clang::MacroInfo* macro) {
-    uint64_t hash;
+    std::uint64_t hash;
     auto name = getTokenSpelling(SM, macro->getDefinitionLoc());
     auto iter = symbolHashCache.find(macro);
     if(iter != symbolHashCache.end()) {

src/Index/Contexts.cpp

@@ -0,0 +1,80 @@
+#include "Index/Contexts.h"
+#include "Support/Ranges.h"
+
+namespace clice::index {
+
+std::uint32_t Contexts::alloc_hctx_id() {
+    std::uint32_t new_hctx_id;
+    if(erased_hctx_ids.empty()) {
+        new_hctx_id = max_hctx_id;
+        max_hctx_id += 1;
+    } else {
+        new_hctx_id = erased_hctx_ids.front();
+        erased_hctx_ids.pop_front();
+    }
+    return new_hctx_id;
+}
+
+std::uint32_t Contexts::alloc_cctx_id() {
+    std::uint32_t new_cctx_id;
+    if(erased_cctx_ids.empty()) {
+        new_cctx_id = max_cctx_id;
+        max_cctx_id += 1;
+        cctx_hctx_refs.emplace_back(1);
+        cctx_element_refs.emplace_back(0);
+    } else {
+        new_cctx_id = erased_cctx_ids.front();
+        erased_cctx_ids.pop_front();
+        cctx_hctx_refs[new_cctx_id] = 1;
+        cctx_element_refs[new_cctx_id] = 0;
+    }
+    return new_cctx_id;
+}
+
+void Contexts::remove(this Contexts& self, llvm::StringRef path) {
+    auto it = self.header_contexts.find(path);
+
+    /// If no such file, nothing to do.
+    if(it == self.header_contexts.end()) {
+        return;
+    }
+
+    llvm::SmallVector<std::uint32_t> erased_hctx_ids;
+    llvm::SmallVector<std::uint32_t> erased_cctx_ids;
+
+    for(auto& context: it->second) {
+        erased_hctx_ids.push_back(context.hctx_id);
+        self.erased_hctx_ids.push_back(context.hctx_id);
+
+        auto cctx_id = context.cctx_id;
+        auto& ref_count = self.cctx_hctx_refs[cctx_id];
+        assert(ref_count > 0);
+
+        /// If the ref count of the canonical context id drops to 0,
+        /// we need to delete it.
+        ref_count -= 1;
+        if(ref_count == 0) {
+            erased_cctx_ids.push_back(cctx_id);
+            self.erased_cctx_ids.push_back(cctx_id);
+            self.cctx_element_refs[cctx_id] = 0;
+        }
+    }
+
+    self.header_contexts.erase(it);
+
+    /// Remove all refs to this header context id.
+    for(auto& state: self.independent_elem_states) {
+        for(auto hctx_id: erased_hctx_ids) {
+            state.erase(hctx_id);
+        }
+    }
+
+    /// Remove all refs to this canonical context id.
+    Bitmap erased_flag = self.erased_flag();
+
+    for(auto& state: self.dependent_elem_states) {
+        state &= erased_flag;
+    }
+}
+
+}  // namespace clice::index

src/Index/IncludeGraph.cpp

@@ -0,0 +1,52 @@
+#include "Index/IncludeGraph.h"
+#include "Compiler/AST.h"
+
+namespace clice::index {
+
+static std::uint32_t addIncludeChain(ASTInfo& AST,
+                                     clang::FileID fid,
+                                     IncludeGraph& graph,
+                                     llvm::StringMap<std::uint32_t>& path_table) {
+    auto& SM = AST.srcMgr();
+    auto& [paths, locations, file_table] = graph;
+    auto [iter, success] = file_table.try_emplace(fid, locations.size());
+    if(!success) {
+        return iter->second;
+    }
+
+    auto index = iter->second;
+
+    auto includeLoc = SM.getIncludeLoc(fid);
+    if(includeLoc.isValid()) {
+        auto presumed = SM.getPresumedLoc(includeLoc, false);
+        locations.emplace_back();
+        locations[index].line = presumed.getLine();
+
+        auto path = AST.getFilePath(presumed.getFileID());
+        auto [iter, success] = path_table.try_emplace(path, paths.size());
+        if(success) {
+            paths.emplace_back(path);
+        }
+        locations[index].path = iter->second;
+
+        uint32_t include = -1;
+        if(presumed.getIncludeLoc().isValid()) {
+            include =
+                addIncludeChain(AST, SM.getFileID(presumed.getIncludeLoc()), graph, path_table);
+        }
+        locations[index].include = include;
+    }
+
+    return index;
+}
+
+IncludeGraph IncludeGraph::from(ASTInfo& AST) {
+    llvm::StringMap<std::uint32_t> path_table;
+    IncludeGraph graph;
+    for(auto fid: AST.files()) {
+        addIncludeChain(AST, fid, graph, path_table);
+    }
+    return graph;
+}
+
+}  // namespace clice::index

src/Index/Index2.cpp

@@ -0,0 +1,220 @@
+#include "Index/Index2.h"
+#include "Support/Ranges.h"
+
+namespace clice::index {
+
+namespace memory2 {
+
+/// Merge all elements from other into self. And update_context is invoked every time
+/// when a element is inserted. The second argument is inserted `Contextual` in the
+/// other, the first element is inserted element in the self, empty if the element
+/// is new to self.
+static void merge_elements(SymbolIndex& self, SymbolIndex& other, auto& update_context) {
+    /// Merge symbols from other into self.
+    for(auto& [symbol_id, symbol]: other.symbols) {
+        auto [it, success] = self.symbols.try_emplace(symbol_id, std::move(symbol));
+        auto& self_symbol = it->second;
+
+        if(success) [[unlikely]] {
+            /// If insert successfully, this is a new symbol and it means
+            /// we need update all context states of this symbol.
+            for(auto& relation: self_symbol.relations) {
+                update_context(relation.ctx, Contextual(relation.ctx), true);
+            }
+            continue;
+        }
+
+        /// If self already has this symbol, try to merge all relations.
+        for(auto& relation: symbol.relations) {
+            auto [it, success] = self_symbol.relations.insert(relation);
+            update_context(it->ctx, Contextual(relation.ctx), success);
+        }
+    }
+
+    for(auto& [range, occurrence_group]: other.occurrences) {
+        auto [it, success] = self.occurrences.try_emplace(range, std::move(occurrence_group));
+        auto& self_occurrence_group = it->second;
+
+        if(success) [[unlikely]] {
+            /// Insert successfully.
+            for(auto& occurrence: self_occurrence_group) {
+                update_context(occurrence.ctx, Contextual(occurrence.ctx), true);
+            }
+            continue;
+        }
+
+        for(auto& occurrence: occurrence_group) {
+            auto i = 0;
+
+            /// In most of cases, there is only one element in the group.
+            /// So don't worry about the performance.
+            for(auto& self_occurrence: self_occurrence_group) {
+                if(occurrence.target_symbol == self_occurrence.target_symbol) {
+                    break;
+                }
+                i += 1;
+            }
+
+            if(i != self_occurrence_group.size()) {
+                update_context(self_occurrence_group[i].ctx, Contextual(occurrence.ctx), false);
+            } else {
+                /// If not found insert new occurrence.
+                auto& o = self_occurrence_group.emplace_back(occurrence);
+                update_context(o.ctx, Contextual(occurrence.ctx), true);
+            }
+        }
+    }
+}
+
+auto SymbolIndex::quick_merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext {
+    assert(!other.merged && "quick merge could be only used for the unmerged index");
+
+    /// We could make sure the other has only one header context.
+    std::uint32_t new_hctx_id = self.alloc_hctx_id();
+
+    Bitmap flag = self.erased_flag();
+    bool is_new_cctx = false;
+    std::uint32_t new_cctx_id = -1;
+
+    llvm::SmallVector<std::uint32_t> visited_elem_ids;
+
+    /// TODO: simplify the logic of update context.
+
+    auto update_context = [&](Contextual& self_elem, Contextual other_elem, bool is_new) {
+        std::uint32_t new_elem_id;
+
+        if(is_new) {
+            /// If this a new element, it means that the other index must introduce
+            /// a new canonical context id, we don't need to do following calculation.
+            is_new_cctx = true;
+
+            if(new_cctx_id == -1) {
+                new_cctx_id = self.alloc_cctx_id();
+            }
+
+            if(other_elem.is_dependent()) {
+                new_elem_id = self.alloc_dependent_elem_id();
+                self.dependent_elem_states[new_elem_id].set(new_cctx_id);
+            } else {
+                new_elem_id = self.alloc_independent_elem_id();
+                self.independent_elem_states[new_elem_id].insert(new_hctx_id);
+            }
+
+            self_elem = Contextual::from(other_elem.is_dependent(), new_elem_id);
+        } else {
+            if(self_elem.is_dependent()) {
+                if(is_new_cctx) {
+                    /// If this element is not new, but we already make sure the context is new
+                    /// add its context.
+                    self.dependent_elem_states[self_elem.offset()].set(new_cctx_id);
+                } else {
+                    /// If this element is not new and we still cannot make sure whether this is
+                    /// new canonical context.
+                    flag &= self.dependent_elem_states[self_elem.offset()];
+                    visited_elem_ids.emplace_back(self_elem.offset());
+                    if(flag.none()) {
+                        is_new_cctx = true;
+                    }
+                }
+            } else {
+                self.independent_elem_states[self_elem.offset()].insert(new_hctx_id);
+            }
+        }
+    };
+
+    /// Merge all elements from other into self and calculate the bitmap state.
+    merge_elements(self, other, update_context);
+
+    if(!is_new_cctx) {
+        assert(new_cctx_id == -1 && flag.any());
+        for(auto i = 0; i < self.max_cctx_id; i++) {
+            if(!flag.test(i)) {
+                continue;
+            }
+
+            if(self.cctx_element_refs[i] == other.cctx_element_refs.front()) {
+                new_cctx_id = i;
+                break;
+            }
+        }
+    }
+
+    if(new_cctx_id == -1) {
+        new_cctx_id = self.alloc_cctx_id();
+        is_new_cctx = true;
+    }
+
+    if(is_new_cctx) {
+        /// In the end we set all visited element ids.
+        for(auto id: visited_elem_ids) {
+            self.dependent_elem_states[id].set(new_cctx_id);
+        }
+        self.cctx_element_refs[new_cctx_id] = other.cctx_element_refs.front();
+    }
+
+    auto& [path, old_contexts] = *other.header_contexts.begin();
+    return self.header_contexts[path].emplace_back(HeaderContext{
+        .include = old_contexts[0].include,
+        .hctx_id = new_hctx_id,
+        .cctx_id = new_cctx_id,
+    });
+}
+
+auto SymbolIndex::merge(this SymbolIndex& self, SymbolIndex& other) -> HeaderContext {
+    return self.quick_merge(other);
+}
+
+Symbol& SymbolIndex::getSymbol(std::uint64_t symbol_id) {
+    assert(canonical_context_count() == 1 && "please use merge for multiple contexts");
+    if(auto it = symbols.find(symbol_id); it != symbols.end()) {
+        return it->second;
+    }
+
+    /// If not found, create a new symbol and return it.
+    auto symbol_ref = symbols.size();
+    auto [it, _] = symbols.try_emplace(symbol_id, Symbol{.id = symbol_id});
+    return it->second;
+}
+
+void SymbolIndex::addRelation(Symbol& symbol, Relation relation, bool is_dependent) {
+    assert(!merged && "add relation could be used in only not merged index");
+    std::uint32_t element_id;
+    if(is_dependent) {
+        element_id = alloc_dependent_elem_id();
+        dependent_elem_states[element_id].set(0);
+        cctx_element_refs[0] += 1;
+    } else {
+        element_id = alloc_independent_elem_id();
+        independent_elem_states.emplace_back(0);
+    }
+
+    relation.ctx = Contextual::from(is_dependent, element_id);
+    symbol.relations.insert(relation);
+}
+
+void SymbolIndex::addOccurrence(LocalSourceRange range,
+                                std::int64_t target_symbol,
+                                bool is_dependent) {
+    assert(!merged && "add occurrence could be used in only not merged index");
+
+    auto& targets = occurrences[range];
+
+    std::uint32_t element_id;
+    if(is_dependent) {
+        element_id = alloc_dependent_elem_id();
+        dependent_elem_states[element_id].set(0);
+        cctx_element_refs[0] += 1;
+    } else {
+        element_id = alloc_independent_elem_id();
+        independent_elem_states.emplace_back(0);
+    }
+
+    Occurrence occurrence;
+    occurrence.target_symbol = target_symbol;
+    occurrence.ctx = Contextual::from(is_dependent, element_id);
+    targets.emplace_back(occurrence);
+}
+
+}  // namespace memory2
+
+}  // namespace clice::index

src/Index/SymbolIndex.cpp

@@ -305,8 +305,6 @@ public:
             std::numeric_limits<std::uint32_t>::max(),
         };
 
-        using enum RelationKind::Kind;
-
         if(kind.isDeclOrDef()) {
             auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange());
             assert(fid == fid2 && "Invalid definition location");
@@ -345,7 +343,6 @@ public:
     }
 
 private:
-    llvm::DenseMap<const void*, uint64_t> symbolIDs;
     llvm::DenseMap<clang::FileID, SymbolIndexBuilder> builders;
 };
 

src/Index/SymbolIndex2.cpp

@@ -0,0 +1,143 @@
+#include "AST/Semantic.h"
+#include "Index/Index2.h"
+#include "Index/IncludeGraph.h"
+#include "Support/Format.h"
+
+namespace clice::index::memory2 {
+
+class SymbolIndexBuilder : public SemanticVisitor<SymbolIndexBuilder> {
+public:
+    SymbolIndexBuilder(ASTInfo& AST) :
+        SemanticVisitor(AST, false), graph(IncludeGraph::from(AST)),
+        context_path(AST.getFilePath(SM.getMainFileID())) {}
+
+    SymbolIndex& getIndex(clang::FileID fid) {
+        if(auto it = indices.find(fid); it != indices.end()) {
+            return *it->second;
+        }
+
+        auto [it, _] = indices.try_emplace(fid, new SymbolIndex());
+        auto& index = *it->second;
+        /// Fix me build include graph here.
+        index.add_context(context_path, graph.getInclude(fid));
+        return index;
+    }
+
+    void handleDeclOccurrence(const clang::NamedDecl* decl,
+                              RelationKind kind,
+                              clang::SourceLocation location) {
+        assert(decl && "Invalid decl");
+        decl = normalize(decl);
+
+        if(location.isMacroID()) {
+            auto spelling = AST.getSpellingLoc(location);
+            auto expansion = AST.getExpansionLoc(location);
+
+            /// FIXME: For location from macro, we only handle the case that the
+            /// spelling and expansion are in the same file currently.
+            if(AST.getFileID(spelling) != AST.getFileID(expansion)) {
+                return;
+            }
+
+            /// For occurrence, we always use spelling location.
+            location = spelling;
+        }
+
+        auto [fid, range] = AST.toLocalRange(location);
+        auto& index = getIndex(fid);
+        auto symbol_id = AST.getSymbolID(decl);
+        auto& symbol = index.getSymbol(symbol_id.hash);
+        symbol.kind = SymbolKind::from(decl);
+        index.addOccurrence(range, symbol_id.hash);
+    }
+
+    void handleMacroOccurrence(const clang::MacroInfo* def,
+                               RelationKind kind,
+                               clang::SourceLocation location) {
+        /// FIXME: Figure out when location is MacroID.
+        if(location.isMacroID()) {
+            return;
+        }
+
+        auto [fid, range] = AST.toLocalRange(location);
+        auto& index = getIndex(fid);
+        auto symbol_id = AST.getSymbolID(def);
+        auto& symbol = index.getSymbol(symbol_id.hash);
+        symbol.kind = SymbolKind::Macro;
+        symbol.name = getTokenSpelling(SM, def->getDefinitionLoc());
+        index.addOccurrence(range, symbol_id.hash);
+
+        if(kind & RelationKind::Definition) {
+            auto begin = def->getDefinitionLoc();
+            auto end = def->getDefinitionEndLoc();
+            assert(begin.isFileID() && end.isFileID() && "Invalid location");
+            auto [fid2, definition_range] = AST.toLocalRange(clang::SourceRange(begin, end));
+            assert(fid == fid2 && "Invalid macro definition location");
+            /// definitionLoc = builder.getLocation(range);
+
+            index.addRelation(symbol,
+                              Relation{
+                                  .kind = RelationKind::Definition,
+                                  .range = range,
+                                  .definition_range = definition_range,
+                              });
+        } else {
+            index.addRelation(symbol,
+                              Relation{
+                                  .kind = RelationKind::Reference,
+                                  .range = range,
+                                  .target_symbol = 0,
+                              });
+        }
+    }
+
+    void handleRelation(const clang::NamedDecl* decl,
+                        RelationKind kind,
+                        const clang::NamedDecl* target,
+                        clang::SourceRange range) {
+        auto [fid, relationRange] = AST.toLocalExpansionRange(range);
+
+        Relation relation{.kind = kind};
+
+        if(kind.isDeclOrDef()) {
+            auto [fid2, definitionRange] = AST.toLocalExpansionRange(decl->getSourceRange());
+            assert(fid == fid2 && "Invalid definition location");
+            relation.range = relationRange;
+            relation.definition_range = definitionRange;
+        } else if(kind.isReference()) {
+            relation.range = relationRange;
+            relation.target_symbol = 0;
+        } else if(kind.isBetweenSymbol()) {
+            auto symbol_id = AST.getSymbolID(normalize(target));
+            relation.target_symbol = symbol_id.hash;
+        } else if(kind.isCall()) {
+            auto symbol_id = AST.getSymbolID(normalize(target));
+            relation.range = relationRange;
+            relation.target_symbol = symbol_id.hash;
+        } else {
+            std::unreachable();
+        }
+
+        auto& index = getIndex(fid);
+        auto symbol_id = AST.getSymbolID(normalize(decl));
+        auto& symbol = index.getSymbol(symbol_id.hash);
+        index.addRelation(symbol, relation);
+    }
+
+    auto build() {
+        run();
+
+        return std::move(indices);
+    }
+
+private:
+    IncludeGraph graph;
+    std::string context_path;
+    llvm::DenseMap<clang::FileID, std::unique_ptr<SymbolIndex>> indices;
+};
+
+index::Shared<std::unique_ptr<SymbolIndex>> SymbolIndex::build(ASTInfo& AST) {
+    return SymbolIndexBuilder(AST).build();
+}
+
+}  // namespace clice::index::memory2