caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serialize index to binary (#273)

Browse Source
This commit is contained in:
ykiko
2025-10-07 18:21:44 +08:00
committed by GitHub
parent b705560557
commit 4c63c52487
21 changed files with 505 additions and 1706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
include/AST/Semantic.h
View File

@@ -29,13 +29,15 @@ public:
void handleDeclOccurrence(const clang::NamedDecl* decl,
RelationKind kind,
clang::SourceLocation location) {
assert(decl && "Invalid decl");
if(!decl || location.isInvalid()) {
return;
}
assert(kind.is_one_of(RelationKind::Declaration,
RelationKind::Definition,
RelationKind::Reference,
RelationKind::WeakReference) &&
"Invalid kind");
assert(location.isValid() && "Invalid location");
/// Forwards to the derived class. Check whether the derived class has
/// its own implementation to avoid infinite recursion.
@@ -94,8 +96,9 @@ public:
RelationKind kind,
const clang::NamedDecl* target,
clang::SourceRange range) {
assert(decl && "Invalid decl");
assert(target && "Invalid target");
if(!decl || !target || range.isInvalid()) {
return;
}
if constexpr(!std::same_as<decltype(&SemanticVisitor::handleRelation),
decltype(&Derived::handleRelation)>) {

12
include/Compiler/Command.h
View File

@@ -147,6 +147,18 @@ public:
llvm::ArrayRef<std::string> compile_commands_dirs,
llvm::StringRef workspace) -> void;
auto commands_size() {
return command_infos.size();
}
auto begin() {
return command_infos.begin();
}
auto end() {
return command_infos.end();
}
private:
/// If file not found in CDB file, try to guess commands or use the default case.
auto guess_or_fallback(this Self& self, llvm::StringRef file) -> LookupInfo;

38
include/Index/Contextual.h
View File

@@ -1,38 +0,0 @@
#pragma once
#include <bitset>
#include <cstdint>
#include <deque>
#include <vector>
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/BitVector.h"
namespace clice::index {
struct Contextual {
/// The actual element id,
std::uint32_t element_id;
constexpr inline static std::uint32_t FLAG = (1ull << 31);
static Contextual from(bool is_dependent, std::uint32_t offset) {
Contextual ctx;
ctx.element_id = offset;
if(!is_dependent) {
ctx.element_id |= FLAG;
}
return ctx;
}
bool is_dependent() {
return (element_id & FLAG) == 0;
}
std::uint32_t offset() {
return element_id & ~FLAG;
}
};
} // namespace clice::index

184
include/Index/HeaderIndex.h
View File

@@ -1,184 +0,0 @@
#pragma once
#include "RawIndex.h"
namespace clice::index::memory {
/// HeaderIndex store extra information to merge raw index from different header contexts.
class HeaderIndex : public RawIndex {
public:
std::uint32_t file_count() {
return header_contexts.size();
}
/// The count of active header contexts in this index.
std::uint32_t header_context_count() {
return max_hctx_id - erased_hctx_ids.size();
}
/// The count of active canonical contexts in this index.
std::uint32_t canonical_context_count() {
return max_cctx_id - erased_cctx_ids.size();
}
/// Whether this contexts has only one single context.
bool is_single_header_context() {
return max_hctx_id == 1 && erased_hctx_ids.empty();
}
auto erased_flag() {
Bitmap map;
map.set();
for(auto cctx_id: erased_cctx_ids) {
map.reset(cctx_id);
}
return map;
}
/// Get a new header context id.
std::uint32_t alloc_hctx_id();
/// Get a new canonical context id.
std::uint32_t alloc_cctx_id();
std::uint32_t alloc_dependent_elem_id() {
auto id = dependent_elem_states.size();
dependent_elem_states.emplace_back(false);
return id;
}
std::uint32_t alloc_independent_elem_id() {
auto id = independent_elem_states.size();
independent_elem_states.emplace_back();
return id;
}
struct HeaderContext {
/// The include location id of this header context.
std::uint32_t include;
/// The header context id of this header context.
std::uint32_t hctx_id;
/// The canonical context id of this header context.
std::uint32_t cctx_id;
};
void remove(this HeaderIndex& self, llvm::StringRef path);
HeaderContext add_context(llvm::StringRef path, std::uint32_t include) {
assert(!merged && "");
auto& context = header_contexts[path].emplace_back();
context.include = include;
context.cctx_id = alloc_cctx_id();
context.hctx_id = alloc_hctx_id();
return context;
}
HeaderContext
merge(this HeaderIndex& self, llvm::StringRef path, std::uint32_t include, RawIndex& raw);
public:
bool merged = false;
/// The max header context id.
std::uint32_t max_hctx_id = 0;
/// The max canonical context id.
std::uint32_t max_cctx_id = 0;
/// The erased header context id. if a header context is erased,
/// we add its id for later reusing.
std::deque<std::uint32_t> erased_hctx_ids;
/// Same as above but for canonical context id.
std::deque<std::uint32_t> erased_cctx_ids;
/// A map between source file path and its header contexts.
llvm::StringMap<llvm::SmallVector<HeaderContext>> header_contexts;
/// A map between canonical context id and corresponding ref counts
/// referenced by header contexts.
llvm::SmallVector<std::uint32_t> cctx_hctx_refs;
/// A map between canonical context id and corresponding ref counts
/// referenced by contextual elements.
llvm::SmallVector<std::uint32_t> cctx_element_refs;
using Bitmap = std::bitset<64>; /// use llvm::BitVector?
/// A map between dependent element id and its state, for dependent element
/// we use bitmap to store states. Each bit in bitmap represents whether
/// this element occurs in corresponding canonical context id.
llvm::SmallVector<Bitmap> dependent_elem_states;
/// A map between independent element id and its state, for independent element
/// we directly store the header context ids that it occurs in.
std::vector<llvm::DenseSet<std::uint32_t>> independent_elem_states;
};
} // namespace clice::index::memory
namespace llvm {
template <typename... Ts>
unsigned dense_hash(const Ts&... ts) {
return llvm::DenseMapInfo<std::tuple<Ts...>>::getHashValue(std::tuple{ts...});
}
template <>
struct DenseMapInfo<clice::LocalSourceRange> {
using R = clice::LocalSourceRange;
inline static R getEmptyKey() {
return R(0, -1);
}
inline static R getTombstoneKey() {
return R(-1, 0);
}
static auto getHashValue(const R& r) {
return dense_hash(r.begin, r.end);
}
static bool isEqual(const R& lhs, const R& rhs) {
return lhs == rhs;
}
};
template <>
struct DenseMapInfo<clice::index::memory::Relation> {
using R = clice::index::memory::Relation;
inline static R getEmptyKey() {
return R{
.kind = clice::RelationKind(),
.range = clice::LocalSourceRange(0, 0),
.target_symbol = 0,
};
}
inline static R getTombstoneKey() {
return R{
.kind = clice::RelationKind(),
.range = clice::LocalSourceRange(-1, -1),
.target_symbol = 0,
};
}
/// Contextual doen't take part in hashing and equality.
static auto getHashValue(const R& relation) {
return dense_hash(relation.kind.value(),
relation.range.begin,
relation.range.end,
relation.target_symbol);
}
static bool isEqual(const R& lhs, const R& rhs) {
return lhs.kind == rhs.kind && lhs.range == rhs.range &&
lhs.target_symbol == rhs.target_symbol;
}
};
} // namespace llvm

11
include/Index/IncludeGraph.h
View File

@@ -42,11 +42,20 @@ struct IncludeGraph {
return paths[path_ref];
}
std::uint32_t getInclude(clang::FileID fid) const {
std::uint32_t include_location_id(clang::FileID fid) const {
auto it = file_table.find(fid);
assert(it != file_table.end());
return it->second;
}
std::uint32_t path_id(clang::FileID fid) {
auto include = include_location_id(fid);
if(include != -1) {
return locations[include].path;
} else {
return paths.size() - 1;
}
}
};
} // namespace clice::index

143
include/Index/MergedIndex.h
View File

@@ -1,77 +1,9 @@
#pragma once
#include "TUIndex.h"
#define ROARING_EXCEPTIONS 0
#define ROARING_TERMINATE(message) std::abort()
#include "roaring/roaring.hh"
#include "Support/Bitmap.h"
#include "llvm/Support/Allocator.h"
namespace clice::index {
/// struct CompilationContext {
/// /// The target of this compilation.
/// llvm::StringRef target;
///
/// /// The canonical compilation command.
/// llvm::StringRef command;
///
/// /// A version field for verification.
/// std::uint32_t version;
/// };
///
/// struct HeaderContext : CompilationContext {
/// /// The include location in the include graph.
/// std::uint32_t include;
///
/// /// The path of the file includes this header.
/// llvm::StringRef path;
/// };
struct HeaderContexts {
std::uint32_t version = 0;
using Context = std::pair<std::uint32_t, std::uint32_t>;
/// A array of include location and its context id.
llvm::SmallVector<Context> includes;
};
struct MergedIndex {
/// For each merged index, we will give it a canonical id.
/// The max canonical id.
std::uint32_t max_canonical_id = 0;
/// We use the value of SHA256 to judge whether two indices are same.
/// Index with same content will be given same canonical id.
llvm::DenseMap<llvm::StringRef, std::uint32_t> canonical_cache;
/// The allocator for storing sha256 hash.
llvm::BumpPtrAllocator allocator;
/// The reference count of each canonical id.
std::vector<std::uint32_t> canonical_ref_counts;
/// The canonical id set of removed index.
roaring::Roaring removed;
/// A map between source file path and its header contexts.
llvm::StringMap<HeaderContexts> contexts;
/// All merged symbol occurrences.
llvm::DenseMap<Occurrence, roaring::Roaring> occurrences;
/// All merged symbol relations.
llvm::DenseMap<SymbolHash, llvm::DenseMap<Relation, roaring::Roaring>> relations;
void remove(llvm::StringRef path);
void merge(llvm::StringRef path, std::uint32_t include, FileIndex& index);
};
} // namespace clice::index
namespace llvm {
template <typename... Ts>
@@ -136,3 +68,76 @@ struct DenseMapInfo<clice::index::Relation> {
};
} // namespace llvm
namespace clice::index {
/// struct CompilationContext {
/// /// The target of this compilation.
/// llvm::StringRef target;
///
/// /// The canonical compilation command.
/// llvm::StringRef command;
///
/// /// A version field for verification.
/// std::uint32_t version;
/// };
///
/// struct HeaderContext : CompilationContext {
/// /// The include location in the include graph.
/// std::uint32_t include;
///
/// /// The path of the file includes this header.
/// llvm::StringRef path;
/// };
struct HeaderContexts {
std::uint32_t version = 0;
using Context = std::pair<std::uint32_t, std::uint32_t>;
/// A array of include location and its context id.
llvm::SmallVector<Context> includes;
friend bool operator== (const HeaderContexts& lhs, const HeaderContexts& rhs) = default;
};
struct MergedIndex {
/// For each merged index, we will give it a canonical id.
/// The max canonical id.
std::uint32_t max_canonical_id = 0;
/// We use the value of SHA256 to judge whether two indices are same.
/// Index with same content will be given same canonical id.
llvm::StringMap<std::uint32_t> canonical_cache;
/// The reference count of each canonical id.
std::vector<std::uint32_t> canonical_ref_counts;
/// The canonical id set of removed index.
roaring::Roaring removed;
/// A map between source file path and its header contexts.
llvm::StringMap<HeaderContexts> contexts;
/// All merged symbol occurrences.
llvm::DenseMap<Occurrence, roaring::Roaring> occurrences;
/// All merged symbol relations.
llvm::DenseMap<SymbolHash, llvm::DenseMap<Relation, roaring::Roaring>> relations;
void remove(llvm::StringRef path);
void merge(llvm::StringRef path, std::uint32_t include, FileIndex& index);
void serialize(this MergedIndex& self, llvm::raw_ostream& out);
friend bool operator== (const MergedIndex& lhs, const MergedIndex& rhs) = default;
};
struct MergedIndexView {
const void* data;
MergedIndex deserialize();
};
} // namespace clice::index

90
include/Index/RawIndex.h
View File

@@ -1,90 +0,0 @@
#pragma once
#include "Contextual.h"
#include "AST/SymbolID.h"
#include "AST/SymbolKind.h"
#include "AST/RelationKind.h"
#include "AST/SourceCode.h"
namespace clice::index::memory {
using SymbolID = std::uint64_t;
struct Relation {
/// The context of this relation.
Contextual ctx;
/// The relation kind of this relation.
RelationKind kind;
/// The range of this relation.
LocalSourceRange range;
/// A field to store extra information depend on kind.
union {
SymbolID target_symbol;
LocalSourceRange definition_range;
};
};
struct Symbol {
/// The symbol id.
SymbolID id;
/// The symbol kind.
SymbolKind kind;
/// Whether this symbol is not visible to other tu.
bool is_tu_local = false;
/// Whether this symbol is defined in function scope.
bool is_function_local = false;
/// The symbol name.
std::string name;
/// All relations of this symbol.
llvm::DenseSet<Relation> relations;
};
struct Occurrence {
/// The context of this occurrence.
Contextual ctx;
/// The target symbol of this occurrence.
SymbolID target_symbol;
};
/// The raw index directly generated by indexing the source file.
struct RawIndex {
public:
Symbol& get_symbol(SymbolID ID) {
/// If not found, create a new symbol and return it.
auto [it, _] = symbols.try_emplace(ID, Symbol{.id = ID});
return it->second;
}
void add_relation(Symbol& symbol, Relation relation, bool is_dependent = true) {
relation.ctx = Contextual::from(is_dependent, 0);
symbol.relations.insert(relation);
}
void add_occurrence(LocalSourceRange range, SymbolID target_symbol, bool is_dependent = true) {
occurrences[range].emplace_back(Contextual::from(is_dependent, 0), target_symbol);
}
public:
/// The path of source file.
std::string path;
/// The content of source file.
std::string content;
/// All symbols in this index.
llvm::DenseMap<SymbolID, Symbol> symbols;
/// All occurrences in this index.
llvm::DenseMap<LocalSourceRange, llvm::SmallVector<Occurrence, 1>> occurrences;
};
} // namespace clice::index::memory

21
include/Index/TUIndex.h
View File

@@ -2,7 +2,9 @@
#include "IncludeGraph.h"
#include "AST/SourceCode.h"
#include "AST/SymbolKind.h"
#include "AST/RelationKind.h"
#include "Support/Bitmap.h"
namespace clice::index {
@@ -12,15 +14,19 @@ using SymbolHash = std::uint64_t;
struct Relation {
RelationKind kind;
char padding[4] = {0, 0, 0, 0};
std::uint32_t padding = 0;
LocalSourceRange range;
union {
LocalSourceRange definition_range;
SymbolHash target_symbol;
SymbolHash target_symbol;
};
void set_definition_range(LocalSourceRange range) {
target_symbol = std::bit_cast<SymbolHash>(range);
}
auto definition_range() {
return std::bit_cast<LocalSourceRange>(target_symbol);
}
};
struct Occurrence {
@@ -40,7 +46,10 @@ struct FileIndex {
struct Symbol {
std::string name;
/// ...
SymbolKind kind;
/// All files that referenced this symbol.
Bitmap reference_files;
};
struct TUIndex {

4
include/Index/schema.fbs
View File

@@ -11,8 +11,8 @@ struct Occurrence {
}
struct Relation {
kind: int;
padding: int;
kind: uint;
padding: uint;
range: Range;
target_symbol: ulong;
}

11
include/Support/Bitmap.h Normal file
View File

@@ -0,0 +1,11 @@
#pragma once
#define ROARING_EXCEPTIONS 0
#define ROARING_TERMINATE(message) std::abort()
#include "roaring/roaring.hh"
namespace clice {
using Bitmap = roaring::Roaring;
}