caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor the definition of Index.

Browse Source
This commit is contained in:
ykiko
2024-10-22 20:42:16 +08:00
parent 7205e6805c
commit eaba0f4c5b
12 changed files with 256 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
include/Index/Index.def Normal file
View File

@@ -0,0 +1,67 @@
#ifndef MAKE_CLANGD_HAPPY
#include <Support/ADT.h>
template <typename T>
using Ref = T;
using llvm::ArrayRef;
using llvm::StringRef;
struct Position {};
enum RelationKind : std::uint32_t {};
#endif
/// If USR is not empty, value is the hash of USR.
/// Otherwise, value is used to represent the kind of builtin symbols.
struct SymbolID {
std::uint64_t value;
StringRef USR;
};
struct Location {
Position begin;
Position end;
StringRef file;
};
struct Relation {
RelationKind kind;
Ref<Location> location;
};
struct Symbol {
Ref<SymbolID> ID;
/// The name of this symbol.
StringRef name;
/// The document of this symbol.
StringRef document;
ArrayRef<Relation> relations;
};
/// Represents a symbol occurrence in the source code.
struct Occurrence {
Ref<SymbolID> symbol;
Ref<Location> location;
};
struct Index {
/// The version of the index format.
StringRef version;
/// The language of the indexed code, currently only supports "C" and "C++".
StringRef language;
/// The URI of the source file.
StringRef URI;
/// The context of the source file.
StringRef context;
/// The commands used to compile the source file.
ArrayRef<StringRef> commands;
/// All the symbols in the source file.
ArrayRef<Symbol> symbols;
/// All the occurrences in the source file.
ArrayRef<Occurrence> occurrences;
};
#undef MAKE_CLANGD_HAPPY

163
include/Index/Index.h
View File

@@ -3,49 +3,18 @@
#include <Support/ADT.h>
#include <Protocol/Basic.h>
#include <Index/SymbolID.h>
// #include <Index/SymbolID.h>
namespace clice {
namespace clice::index {
struct Symbol;
struct Occurrence;
/// Note that we have two kinds of `Index` definitions. One for collecting data from AST,
/// and the other is used to serialize data to the binary format. The key difference is that
/// one uses pointers and the other uses offsets to store data. All structures that uses ArrayRef
/// or StringRef are defined in `Index.def` so that they could have different definitions in
/// context.
// struct Diagnostic {};
// struct InlayHint {};
/// CSIF stands for "C/C++ Semantic Index Format".
/// It is an efficient binary format for storing the semantic information of C/C++ source code.
/// The main references are [SCIP](https://sourcegraph.com/blog/announcing-scip) and
/// [SemanticDB](https://scalameta.org/docs/semanticdb/specification.html).
struct CSIF {
/// The version of the CSIF format.
llvm::StringRef version;
/// The language of the source code, currently only supports "c" and "c++".
llvm::StringRef language;
/// The URI of the source file.
llvm::StringRef uri;
/// The context of the source file, used to check whether need to re-index the source file.
llvm::StringRef content;
/// The commands used to compile the source file.
llvm::ArrayRef<llvm::StringRef> commands;
/// The symbols in the source file.
llvm::ArrayRef<Symbol> symbols;
/// The occurrences in the source file.
llvm::ArrayRef<Occurrence> occurrences;
///// The semantic tokens in the source file.
// llvm::ArrayRef<std::uint32_t> semanticTokens;
// FIXME:
/// The diagnostics in the source file.
// llvm::ArrayRef<Diagnostic> diagnostics;
/// The inlay hints in the source file.
// llvm::ArrayRef<InlayHint> inlayHints;
};
/// Note that it's possible to have multiple roles at the same time.
enum class Role {
/// Used to discribe the kind of relation between two symbols.
enum RelationKind : std::uint32_t {
Invalid,
Declaration,
Definition,
@@ -81,48 +50,100 @@ enum class Role {
Callee,
};
struct Location {
proto::DocumentUri uri;
proto::Range range;
/// Represent a position in the source code, the line and column are 1-based.
struct Position {
std::uint32_t line;
std::uint32_t column;
friend std::strong_ordering operator<=> (const Location& lhs, const Location& rhs) = default;
friend std::strong_ordering operator<=> (const Position&, const Position&) = default;
};
/// If symbol A has a relation to symbol B with role R.
/// For example, `Caller`. Then we say B is a caller of A.
struct Relation {
/// The role of the relation.
Role role;
/// The location of the related symbol.
Location location;
} // namespace clice::index
friend std::strong_ordering operator<=> (const Relation& lhs, const Relation& rhs) = default;
namespace clice::index::in {
template <typename T>
using Ref = T;
using llvm::ArrayRef;
using llvm::StringRef;
#define MAKE_CLANGD_HAPPY
#include "Index.def"
inline SymbolID kindToSymbolID(std::uint64_t kind) {
return SymbolID{kind, ""};
}
inline SymbolID USRToSymbolID(llvm::StringRef USR) {
return SymbolID{llvm::hash_value(USR), USR};
}
inline std::strong_ordering operator<=> (const SymbolID& lhs, const SymbolID& rhs) {
auto cmp = lhs.value <=> rhs.value;
if(cmp != std::strong_ordering::equal) {
return cmp;
}
return lhs.USR.compare(rhs.USR) <=> 0;
}
inline std::strong_ordering operator<=> (const Location& lhs, const Location& rhs) {
auto cmp = lhs.file.compare(rhs.file);
if(cmp != 0) {
return cmp <=> 0;
}
return std::tuple{lhs.begin, lhs.end} <=> std::tuple{rhs.begin, rhs.end};
};
struct Symbol {
/// The ID of the symbol.
SymbolID ID;
/// display when hover.
llvm::StringRef document;
} // namespace clice::index::in
// TODO: append more useful information.
namespace llvm {
/// The relations of the symbol.
llvm::ArrayRef<Relation> relations;
using clice::index::in::kindToSymbolID;
using SymbolID = clice::index::in::SymbolID;
template <>
struct DenseMapInfo<SymbolID> {
inline static SymbolID getEmptyKey() {
static SymbolID EMPTY_KEY = kindToSymbolID(std::numeric_limits<uint64_t>::max());
return EMPTY_KEY;
}
inline static SymbolID getTombstoneKey() {
static SymbolID TOMBSTONE_KEY = kindToSymbolID(std::numeric_limits<uint64_t>::max() - 1);
return TOMBSTONE_KEY;
}
inline static llvm::hash_code getHashValue(const SymbolID& ID) {
return ID.value;
}
inline static bool isEqual(const SymbolID& LHS, const SymbolID& RHS) {
return LHS.value == RHS.value && LHS.USR == RHS.USR;
}
};
struct Occurrence {
/// The ID of the symbol.
SymbolID symbol;
/// The range of the occurrence.
Location location;
} // namespace llvm
namespace clice::index::out {
/// Because `SymbolID` and `Location` are duplicate referenced by `Relation`, `Symbol` and `Occurrence`,
/// To save space, we use offsets to index them.
template <typename T>
struct Ref {
std::uint32_t offset;
};
enum BuiltinSymbolKind {
#define SYMBOL(name, description) name,
#include <Index/Symbols.def>
#undef SYMBOL
template <typename T>
struct ArrayRef {
std::uint32_t offset;
std::uint32_t length;
};
} // namespace clice
using StringRef = ArrayRef<char>;
#define MAKE_CLANGD_HAPPY
#include "Index.def"
} // namespace clice::index::out

10
include/Index/Indexer.h
View File

@@ -3,13 +3,13 @@
#include <Index/Index.h>
#include <Compiler/Clang.h>
namespace clice {
namespace clice::index::in {
class Indexer {
public:
Indexer(clang::Sema& sema, clang::syntax::TokenBuffer& tokBuf) : sema(sema), tokBuf(tokBuf) {}
CSIF index();
Index index();
std::size_t lookup(const clang::NamedDecl* decl);
@@ -19,7 +19,9 @@ public:
Indexer& addOccurrence(const clang::NamedDecl* decl, clang::SourceRange range);
Indexer& addRelation(const clang::NamedDecl* from, clang::SourceRange range, std::initializer_list<Role> roles);
Indexer& addRelation(const clang::NamedDecl* from,
clang::SourceRange range,
std::initializer_list<RelationKind> roles);
private:
clang::Sema& sema;
@@ -36,5 +38,5 @@ private:
llvm::DenseMap<const clang::Decl*, std::size_t> cache;
};
} // namespace clice
} // namespace clice::index::in

8
include/Index/Loader.h
View File

@@ -2,11 +2,11 @@
#include <Index/Index.h>
namespace clice {
namespace clice::index::in {
class Loader {
public:
Loader(CSIF csif, char* data) : csif(csif), data(data) {}
Loader(Index csif, char* data) : csif(csif), data(data) {}
const Symbol& locate(Location loc) const {
auto iter = std::partition_point(csif.occurrences.begin(), csif.occurrences.end(), [&](const auto& occurrence) {
@@ -30,8 +30,8 @@ public:
}
private:
CSIF csif;
Index csif;
char* data;
};
} // namespace clice
} // namespace clice::index::in

32
include/Index/Pack.h
View File

@@ -1,32 +0,0 @@
#pragma once
#include "Index.h"
namespace clice {
// We use an efficient way to pack the CSIF structure into a binary format.
//
// The serialization process follows these steps:
// 1. Write the `CSIF` structure directly into the binary buffer.
// 2. For each reference member (e.g., arrays and strings):
// a. Write the referenced data (array elements or string characters) into the buffer.
// b. Replace the pointer in the `CSIF` structure with the offset pointing to the actual data.
//
// Data Layout in the binary buffer:
// | CSIF structure | array offsets | string offsets | array data | string data |
//
// - `CSIF structure`: The first `sizeof(CSIF)` bytes store the `CSIF` structure itself.
// - `array offsets`: Offsets pointing to the actual array data stored later in the buffer.
// - `string offsets`: Offsets pointing to the actual string data stored later in the buffer.
// - `array data`: Contains all array elements, stored with 8-byte alignment to improve access efficiency.
// - `string data`: Contains all string characters, stored sequentially.
/// Pack the CSIF into a binary buffer.
std::unique_ptr<char[]> pack(CSIF csif);
/// Unpack the binary buffer into a CSIF.
/// NOTE: the data should be mutable. when the first load it,
/// We need to replace all offset to actual pointer.
CSIF unpack(char* data);
} // namespace clice