caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store indices to disk (#279)

Browse Source
This commit is contained in:
ykiko
2025-10-19 21:53:57 +08:00
committed by GitHub
parent eda7f03a37
commit dffa884e5a
22 changed files with 979 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
include/Compiler/CompilationUnit.h
View File

@@ -1,5 +1,7 @@
#pragma once
#include <chrono>
#include "Directive.h"
#include "Compiler/Diagnostic.h"
#include "AST/SymbolID.h"
@@ -166,6 +168,10 @@ public:
auto top_level_decls() -> llvm::ArrayRef<clang::Decl*>;
std::chrono::milliseconds build_at();
std::chrono::milliseconds build_duration();
clang::LangOptions& lang_options();
clang::ASTContext& context();

6
include/Index/IncludeGraph.h
View File

@@ -12,13 +12,15 @@ namespace clice::index {
struct IncludeLocation {
/// The file path of the include directive.
std::uint32_t path = -1;
std::uint32_t path_id = -1;
/// The line number of the include directive, 1-based.
std::uint32_t line = -1;
/// The include location that introduces this file.
std::uint32_t include = -1;
friend bool operator== (const IncludeLocation&, const IncludeLocation&) = default;
};
struct IncludeGraph {
@@ -51,7 +53,7 @@ struct IncludeGraph {
std::uint32_t path_id(clang::FileID fid) {
auto include = include_location_id(fid);
if(include != -1) {
return locations[include].path;
return locations[include].path_id;
} else {
return paths.size() - 1;
}

177
include/Index/MergedIndex.h
View File

@@ -1,156 +1,83 @@
#pragma once
#include "TUIndex.h"
#include "Support/Bitmap.h"
#include "llvm/Support/Allocator.h"
namespace llvm {
template <typename... Ts>
unsigned dense_hash(const Ts&... ts) {
return llvm::DenseMapInfo<std::tuple<Ts...>>::getHashValue(std::tuple{ts...});
}
template <>
struct DenseMapInfo<clice::index::Occurrence> {
using R = clice::LocalSourceRange;
using V = clice::index::Occurrence;
inline static V getEmptyKey() {
return V(R(-1, 0), 0);
}
inline static V getTombstoneKey() {
return V(R(-2, 0), 0);
}
static auto getHashValue(const V& v) {
return dense_hash(v.range.begin, v.range.end, v.target);
}
static bool isEqual(const V& lhs, const V& rhs) {
return lhs.range == rhs.range && lhs.target == rhs.target;
}
};
template <>
struct DenseMapInfo<clice::index::Relation> {
using R = clice::index::Relation;
inline static R getEmptyKey() {
return R{
.kind = clice::RelationKind(),
.range = clice::LocalSourceRange(-1, 0),
.target_symbol = 0,
};
}
inline static R getTombstoneKey() {
return R{
.kind = clice::RelationKind(),
.range = clice::LocalSourceRange(-2, 0),
.target_symbol = 0,
};
}
/// Contextual doen't take part in hashing and equality.
static auto getHashValue(const R& relation) {
return dense_hash(relation.kind.value(),
relation.range.begin,
relation.range.end,
relation.target_symbol);
}
static bool isEqual(const R& lhs, const R& rhs) {
return lhs.kind == rhs.kind && lhs.range == rhs.range &&
lhs.target_symbol == rhs.target_symbol;
}
};
} // namespace llvm
#include "llvm/Support/MemoryBuffer.h"
namespace clice::index {
/// struct CompilationContext {
/// /// The target of this compilation.
/// llvm::StringRef target;
///
/// /// The canonical compilation command.
/// llvm::StringRef command;
///
/// /// A version field for verification.
/// std::uint32_t version;
/// };
///
/// struct HeaderContext : CompilationContext {
/// /// The include location in the include graph.
/// std::uint32_t include;
///
/// /// The path of the file includes this header.
/// llvm::StringRef path;
/// };
class MergedIndex {
private:
struct Impl;
struct HeaderContexts {
std::uint32_t version = 0;
using Self = MergedIndex;
struct Context {
std::uint32_t include;
std::uint32_t canonical_id;
MergedIndex(std::unique_ptr<llvm::MemoryBuffer> buffer, std::unique_ptr<Impl> impl);
friend bool operator== (const Context&, const Context&) = default;
};
void load_in_memory(this Self& self);
/// A array of include location and its context id.
llvm::SmallVector<Context> includes;
public:
MergedIndex();
friend bool operator== (const HeaderContexts&, const HeaderContexts&) = default;
};
MergedIndex(llvm::StringRef data);
struct MergedIndex {
/// For each merged index, we will give it a canonical id.
/// The max canonical id.
std::uint32_t max_canonical_id = 0;
MergedIndex(const MergedIndex&) = delete;
/// We use the value of SHA256 to judge whether two indices are same.
/// Index with same content will be given same canonical id.
llvm::StringMap<std::uint32_t> canonical_cache;
MergedIndex(MergedIndex&& other);
/// The reference count of each canonical id.
std::vector<std::uint32_t> canonical_ref_counts;
MergedIndex& operator= (const MergedIndex&) = delete;
/// The canonical id set of removed index.
roaring::Roaring removed;
MergedIndex& operator= (MergedIndex&& other);
/// A map between source file path and its header contexts.
llvm::StringMap<HeaderContexts> contexts;
~MergedIndex();
/// All merged symbol occurrences.
llvm::DenseMap<Occurrence, roaring::Roaring> occurrences;
/// Load merged index from disk
static MergedIndex load(llvm::StringRef path);
/// All merged symbol relations.
llvm::DenseMap<SymbolHash, llvm::DenseMap<Relation, roaring::Roaring>> relations;
/// Serialize it to binary format.
void serialize(this const Self& self, llvm::raw_ostream& out);
/// FIXME: The content of this file.
/// std::string content;
/// Lookup the occurrence in corresponding offset.
void lookup(this const Self& self,
std::uint32_t offset,
llvm::function_ref<bool(const Occurrence&)> callback);
/// Sorted occurrences cache for fast lookup.
std::vector<Occurrence> cache_occurrences;
/// Lookup the relations of given symbol.
void lookup(this const Self& self,
SymbolHash symbol,
RelationKind kind,
llvm::function_ref<bool(const Relation&)> callback);
void remove(llvm::StringRef path);
/// Whether this index needs rebuilding.
bool need_update(this const Self& self, llvm::ArrayRef<llvm::StringRef> path_mapping);
void merge(llvm::StringRef path, std::uint32_t include, FileIndex& index);
bool need_rewrite() {
return impl != nullptr;
}
std::vector<Occurrence> lookup(std::uint32_t offset);
/// Remove the index of specific path id.
void remove(this Self& self, std::uint32_t path_id);
void serialize(this MergedIndex& self, llvm::raw_ostream& out);
/// Merge the index with given compilation context.
void merge(this Self& self,
std::uint32_t path_id,
std::chrono::milliseconds build_at,
std::vector<IncludeLocation> include_locations,
FileIndex& index);
friend bool operator== (const MergedIndex&, const MergedIndex&) = default;
};
/// Merge the index with given header context.
void merge(this Self& self, std::uint32_t path_id, std::uint32_t include_id, FileIndex& index);
struct MergedIndexView {
const void* data;
friend bool operator== (MergedIndex& lhs, MergedIndex& rhs);
MergedIndex deserialize();
private:
/// The binary serialization data of index. If you load merged index
/// from disk, we use directly access the data without deserialization
/// unless you want to modify it.
std::unique_ptr<llvm::MemoryBuffer> buffer;
/// The in memory data of the index.
std::unique_ptr<Impl> impl;
};
} // namespace clice::index

2
include/Index/ProjectIndex.h
View File

@@ -47,7 +47,7 @@ struct ProjectIndex {
SymbolTable symbols;
void merge(this ProjectIndex& self, TUIndex& index);
llvm::SmallVector<std::uint32_t> merge(this ProjectIndex& self, TUIndex& index);
void serialize(this ProjectIndex& self, llvm::raw_ostream& os);

9
include/Index/TUIndex.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include <chrono>
#include "IncludeGraph.h"
#include "AST/SourceCode.h"
#include "AST/SymbolKind.h"
@@ -43,6 +44,8 @@ struct FileIndex {
llvm::DenseMap<SymbolHash, std::vector<Relation>> relations;
std::vector<Occurrence> occurrences;
std::array<std::uint8_t, 32> hash();
};
struct Symbol {
@@ -59,12 +62,18 @@ struct Symbol {
using SymbolTable = llvm::DenseMap<SymbolHash, Symbol>;
struct TUIndex {
/// The building timestamp of this file.
std::chrono::milliseconds built_at;
/// The include information of this file.
IncludeGraph graph;
SymbolTable symbols;
llvm::DenseMap<clang::FileID, FileIndex> file_indices;
FileIndex main_file_index;
static TUIndex build(CompilationUnit& unit);
};

28
include/Index/schema.fbs
View File

@@ -22,19 +22,29 @@ table CacheEntry {
canonical_id: uint;
}
struct Context {
include_: uint;
struct IncludeContext {
include_id: uint;
canonical_id: uint;
}
table HeaderContexts {
table HeaderContextEntry {
path_id: uint;
version: uint;
includes: [Context];
includes: [IncludeContext];
}
table HeaderContextsEntry {
path: string;
contexts: HeaderContexts;
struct IncludeLocation {
path_id: uint;
line: uint;
include_id: uint;
}
table CompilationContextEntry {
path_id: uint;
version: uint;
canonical_id: uint;
build_at: ulong;
include_locations: [IncludeLocation];
}
table OccurrenceEntry {
@@ -67,7 +77,9 @@ table MergedIndex {
canonical_cache: [CacheEntry];
contexts: [HeaderContextsEntry];
header_contexts: [HeaderContextEntry];
compilation_contexts: [CompilationContextEntry];
occurrences: [OccurrenceEntry];

28
include/Server/Indexer.h
View File

@@ -3,6 +3,8 @@
#include <deque>
#include <vector>
#include "Config.h"
#include "Convert.h"
#include "Async/Async.h"
#include "Compiler/Command.h"
#include "Index/MergedIndex.h"
@@ -19,7 +21,8 @@ class CompilationUnit;
class Indexer {
public:
Indexer(CompilationDatabase& database) : database(database) {}
Indexer(CompilationDatabase& database, config::Config& config) :
database(database), config(config) {}
async::Task<> index(llvm::StringRef path);
@@ -29,8 +32,27 @@ public:
async::Task<> index_all();
index::MergedIndex& get_index(std::uint32_t path_id) {
auto [it, success] = in_memory_indices.try_emplace(path_id);
if(!success) {
return it->second;
}
auto it2 = project_index.indices.find(path_id);
if(it2 != project_index.indices.end()) {
auto path = project_index.path_pool.path(it2->second);
it->second = index::MergedIndex::load(path);
}
return it->second;
}
using Result = async::Task<std::vector<proto::Location>>;
void load_from_disk();
void save_to_disk();
auto lookup(llvm::StringRef path, std::uint32_t offset, RelationKind kind) -> Result;
auto declaration(llvm::StringRef path, std::uint32_t offset) -> Result;
@@ -46,8 +68,12 @@ public:
private:
CompilationDatabase& database;
config::Config& config;
index::ProjectIndex project_index;
PathMapping mapping;
llvm::DenseMap<std::uint32_t, index::MergedIndex> in_memory_indices;
/// Currently indexes tasks ...