caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Index.

Browse Source
This commit is contained in:
ykiko
2024-10-19 15:35:20 +08:00
parent 578624ea05
commit 325a2fb84c
10 changed files with 191 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/Index/CSIF.h → include/Index/Index.h
View File

@@ -84,6 +84,8 @@ enum class Role {
struct Location {
proto::DocumentUri uri;
proto::Range range;
friend std::strong_ordering operator<=> (const Location& lhs, const Location& rhs) = default;
};
/// If symbol A has a relation to symbol B with role R.
@@ -93,6 +95,8 @@ struct Relation {
Role role;
/// The location of the related symbol.
Location location;
friend std::strong_ordering operator<=> (const Relation& lhs, const Relation& rhs) = default;
};
struct Symbol {

40
include/Index/Indexer.h Normal file
View File

@@ -0,0 +1,40 @@
#pragma once
#include <Index/Index.h>
#include <Compiler/Clang.h>
namespace clice {
class Indexer {
public:
Indexer(clang::Sema& sema, clang::syntax::TokenBuffer& tokBuf) : sema(sema), tokBuf(tokBuf) {}
CSIF index();
std::size_t lookup(const clang::NamedDecl* decl);
Indexer& addSymbol(const clang::NamedDecl* decl);
Indexer& addOccurrence(int Kind, clang::SourceLocation location);
Indexer& addOccurrence(const clang::NamedDecl* decl, clang::SourceRange range);
Indexer& addRelation(const clang::NamedDecl* from, clang::SourceRange range, std::initializer_list<Role> roles);
private:
clang::Sema& sema;
clang::syntax::TokenBuffer& tokBuf;
llvm::BumpPtrAllocator allocator;
llvm::StringSaver saver{allocator};
std::vector<Symbol> symbols;
std::vector<Occurrence> occurrences;
std::vector<std::vector<Relation>> relations;
llvm::DenseMap<SymbolID, std::size_t> symbolIndex;
llvm::DenseMap<const clang::Decl*, std::size_t> cache;
};
} // namespace clice

37
include/Index/Loader.h Normal file
View File

@@ -0,0 +1,37 @@
#pragma once
#include <Index/Index.h>
namespace clice {
class Loader {
public:
Loader(CSIF csif, char* data) : csif(csif), data(data) {}
const Symbol& locate(Location loc) const {
auto iter = std::partition_point(csif.occurrences.begin(), csif.occurrences.end(), [&](const auto& occurrence) {
return occurrence.location < loc;
});
if(iter == csif.occurrences.end()) {
std::terminate();
}
auto id = iter->symbol;
auto symbol = std::partition_point(csif.symbols.begin(), csif.symbols.end(), [&](const auto& symbol) {
return symbol.ID < id;
});
if(symbol == csif.symbols.end()) {
std::terminate();
}
return *symbol;
}
private:
CSIF csif;
char* data;
};
} // namespace clice

2
include/Index/Pack.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include "CSIF.h"
#include "Index.h"
namespace clice {

44
include/Index/SymbolSlab.h
View File

@@ -1,44 +0,0 @@
#pragma once
#include <Index/CSIF.h>
#include <Compiler/Clang.h>
namespace clice {
class SymbolSlab {
public:
SymbolSlab(clang::Sema& sema, clang::syntax::TokenBuffer& tokBuf) : sema(sema), tokBuf(tokBuf) {}
CSIF index();
std::size_t lookup(const clang::NamedDecl* decl);
SymbolSlab& addSymbol(const clang::NamedDecl* decl);
SymbolSlab& addOccurrence(int Kind, clang::SourceLocation location);
SymbolSlab& addOccurrence(const clang::NamedDecl* decl, clang::SourceLocation location);
SymbolSlab& addOccurrence(const clang::NamedDecl* decl, clang::SourceRange range);
SymbolSlab& addRelation(const clang::NamedDecl* from, clang::SourceLocation location, std::initializer_list<Role> roles);
SymbolSlab& addRelation(const clang::NamedDecl* from, clang::SourceRange range, std::initializer_list<Role> roles);
private:
clang::Sema& sema;
clang::syntax::TokenBuffer& tokBuf;
llvm::BumpPtrAllocator allocator;
llvm::StringSaver saver{allocator};
std::vector<Symbol> symbols;
std::vector<Occurrence> occurrences;
std::vector<std::vector<Relation>> relations;
llvm::DenseMap<SymbolID, std::size_t> symbolIndex;
llvm::DenseMap<const clang::Decl*, std::size_t> cache;
};
} // namespace clice

4
include/Protocol/Basic.h
View File

@@ -17,11 +17,15 @@ using DocumentUri = std::string;
struct Position {
uinteger line;
uinteger character;
friend std::strong_ordering operator<=> (const Position& lhs, const Position& rhs) = default;
};
struct Range {
Position start;
Position end;
friend std::strong_ordering operator<=> (const Range& lhs, const Range& rhs) = default;
};
struct TextDocumentItem {

29
src/Index/Index.cpp
View File

@@ -1,4 +1,4 @@
#include <Index/SymbolSlab.h>
#include <Index/Indexer.h>
namespace clice {
@@ -8,7 +8,7 @@ class SymbolCollector : public clang::RecursiveASTVisitor<SymbolCollector> {
using Base = clang::RecursiveASTVisitor<SymbolCollector>;
public:
SymbolCollector(SymbolSlab& slab, clang::ASTContext& context) :
SymbolCollector(Indexer& slab, clang::ASTContext& context) :
slab(slab), context(context), srcMgr(context.getSourceManager()) {}
/// we don't care about the node without location information, so skip them.
@@ -176,7 +176,7 @@ public:
// If it's not full(explicit) specialization, find the primary template.
if(!spec->isExplicitInstantiationOrSpecialization()) {
auto specialized = spec->getSpecializedTemplateOrPartial();
if(auto CTD = specialized.dyn_cast<clang::ClassTemplateDecl*>()) {
if(specialized.is<clang::ClassTemplateDecl*>()) {
slab.addOccurrence(CTD, nameLoc)
.addRelation(CTD, nameLoc, {Role::Reference, Role::ImplicitInstantiation});
} else {
@@ -204,27 +204,38 @@ public:
// MemberPointerTypeLoc
private:
SymbolSlab& slab;
Indexer& slab;
clang::ASTContext& context;
clang::SourceManager& srcMgr;
};
} // namespace
CSIF SymbolSlab::index() {
CSIF Indexer::index() {
CSIF csif;
SymbolCollector collector(*this, sema.getASTContext());
collector.TraverseAST(sema.getASTContext());
for(std::size_t i = 0; i < relations.size(); ++i) {
llvm::sort(relations[i], [](const Relation& lhs, const Relation& rhs) {
return lhs.location < rhs.location;
});
symbols[i].relations = relations[i];
}
llvm::sort(symbols, [](const Symbol& lhs, const Symbol& rhs) {
return lhs.ID < rhs.ID;
});
llvm::sort(occurrences, [](const Occurrence& lhs, const Occurrence& rhs) {
return lhs.location < rhs.location;
});
csif.version = "0.1";
csif.language = "C++";
csif.symbols = symbols;
csif.occurrences = occurrences;
for(std::size_t i = 0; i < relations.size(); ++i) {
symbols[i].relations = relations[i];
}
return csif;
};

44
src/Index/SymbolSlab.cpp → src/Index/Indexer.cpp
View File

@@ -1,4 +1,4 @@
#include <Index/SymbolSlab.h>
#include <Index/Indexer.h>
#include <clang/Index/USRGeneration.h>
namespace clice {
@@ -17,7 +17,7 @@ Location toLocation(clang::SourceRange loc, clang::SourceManager& srcMgr) {
} // namespace
std::size_t SymbolSlab::lookup(const clang::NamedDecl* decl) {
std::size_t Indexer::lookup(const clang::NamedDecl* decl) {
auto iter = cache.find(decl);
if(iter != cache.end()) {
return iter->second;
@@ -27,7 +27,7 @@ std::size_t SymbolSlab::lookup(const clang::NamedDecl* decl) {
return symbols.size() - 1;
}
SymbolSlab& SymbolSlab::addSymbol(const clang::NamedDecl* decl) {
Indexer& Indexer::addSymbol(const clang::NamedDecl* decl) {
// Generate and save USR.
llvm::SmallString<128> USR;
clang::index::generateUSRForDecl(decl, USR);
@@ -35,6 +35,8 @@ SymbolSlab& SymbolSlab::addSymbol(const clang::NamedDecl* decl) {
if(!symbolIndex.contains(SymbolID::fromUSR(USR))) {
auto ID = SymbolID::fromUSR(saver.save(USR.str()));
symbols.emplace_back(ID);
symbols.back().document = saver.save(decl->getNameAsString());
cache.try_emplace(decl, symbols.size() - 1);
symbolIndex.try_emplace(ID, symbols.size() - 1);
relations.emplace_back();
@@ -45,18 +47,7 @@ SymbolSlab& SymbolSlab::addSymbol(const clang::NamedDecl* decl) {
return *this;
}
SymbolSlab& SymbolSlab::addOccurrence(const clang::NamedDecl* decl, clang::SourceLocation location) {
if(location.isInvalid()) {
return *this;
}
auto& srcMgr = sema.getSourceManager();
auto ID = symbols[lookup(decl)].ID;
occurrences.emplace_back(Occurrence{ID, toLocation(location, srcMgr)});
return *this;
}
SymbolSlab& SymbolSlab::addOccurrence(const clang::NamedDecl* decl, clang::SourceRange range) {
Indexer& Indexer::addOccurrence(const clang::NamedDecl* decl, clang::SourceRange range) {
if(range.isInvalid()) {
return *this;
}
@@ -67,7 +58,7 @@ SymbolSlab& SymbolSlab::addOccurrence(const clang::NamedDecl* decl, clang::Sourc
return *this;
}
SymbolSlab& SymbolSlab::addOccurrence(int Kind, clang::SourceLocation loc) {
Indexer& Indexer::addOccurrence(int Kind, clang::SourceLocation loc) {
if(loc.isInvalid()) {
return *this;
}
@@ -77,24 +68,9 @@ SymbolSlab& SymbolSlab::addOccurrence(int Kind, clang::SourceLocation loc) {
return *this;
}
SymbolSlab& SymbolSlab::addRelation(const clang::NamedDecl* from,
clang::SourceLocation loc,
std::initializer_list<Role> roles) {
if(loc.isInvalid()) {
return *this;
}
auto index = lookup(from);
auto& relations = this->relations[index];
for(auto role: roles) {
relations.emplace_back(Relation{role, toLocation(loc, sema.getSourceManager())});
}
return *this;
}
SymbolSlab& SymbolSlab::addRelation(const clang::NamedDecl* from,
clang::SourceRange range,
std::initializer_list<Role> roles) {
Indexer& Indexer::addRelation(const clang::NamedDecl* from,
clang::SourceRange range,
std::initializer_list<Role> roles) {
if(range.isInvalid()) {
return *this;
}

81
src/Support/SourceCode.cpp
View File

@@ -2,18 +2,83 @@
namespace clice {
std::size_t length(llvm::StringRef text, Encoding encoding) {
switch(encoding) {
case Encoding::UTF8: return text.size();
case Encoding::UTF16: {
// TODO:
return text.size() / 2;
}
namespace {
// Here be dragons. LSP positions use columns measured in *UTF-16 code units*!
// Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial.
// Iterates over unicode codepoints in the (UTF-8) string. For each,
// invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true.
// Returns true if CB returned true, false if we hit the end of string.
//
// If the string is not valid UTF-8, we log this error and "decode" the
// text in some arbitrary way. This is pretty sad, but this tends to happen deep
// within indexing of headers where clang misdetected the encoding, and
// propagating the error all the way back up is (probably?) not be worth it.
template <typename Callback>
static bool iterateCodepoints(llvm::StringRef u8string, const Callback& callback) {
bool LoggedInvalid = false;
// A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
// Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx.
for(size_t I = 0; I < u8string.size();) {
unsigned char C = static_cast<unsigned char>(u8string[I]);
if(LLVM_LIKELY(!(C & 0x80))) { // ASCII character.
if(callback(1, 1))
return true;
++I;
continue;
}
// This convenient property of UTF-8 holds for all non-ASCII characters.
size_t UTF8Length = llvm::countl_one(C);
// 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here.
// 11111xxx is not valid UTF-8 at all, maybe some ISO-8859-*.
if(LLVM_UNLIKELY(UTF8Length < 2 || UTF8Length > 4)) {
if(!LoggedInvalid) {
std::terminate();
LoggedInvalid = true;
}
// We can't give a correct result, but avoid returning something wild.
// Pretend this is a valid ASCII byte, for lack of better options.
// (Too late to get ISO-8859-* right, we've skipped some bytes already).
if(callback(1, 1))
return true;
++I;
continue;
}
I += UTF8Length; // Skip over all trailing bytes.
// A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
// Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...)
if(callback(UTF8Length, UTF8Length == 4 ? 2 : 1))
return true;
}
return false;
}
} // namespace
std::size_t length(llvm::StringRef u8string, Encoding encoding) {
size_t count = 0;
switch(encoding) {
case Encoding::UTF8: {
count = u8string.size();
break;
}
case Encoding::UTF16: {
iterateCodepoints(u8string, [&](int U8Len, int U16Len) {
count += U16Len;
return false;
});
break;
}
case Encoding::UTF32: {
return text.size() / 4;
iterateCodepoints(u8string, [&](int U8Len, int U16Len) {
++count;
return false;
});
break;
}
}
return count;
}
} // namespace clice

5
tests/Index/Index.cpp
View File

@@ -1,5 +1,5 @@
#include <gtest/gtest.h>
#include <Index/SymbolSlab.h>
#include <Index/Indexer.h>
#include <Support/JSON.h>
#include <Compiler/Compiler.h>
#include <Support/FileSystem.h>
@@ -18,13 +18,12 @@ TEST(clice, Index) {
foreachFile("Index", [](llvm::StringRef filepath, llvm::StringRef content) {
Compiler compiler("main.cpp", content, compileArgs);
compiler.buildAST();
SymbolSlab slab(compiler.sema(), compiler.tokBuf());
Indexer slab(compiler.sema(), compiler.tokBuf());
auto csif = slab.index();
auto value = json::serialize(csif);
std::error_code EC;
llvm::raw_fd_ostream fileStream("output.json", EC);
fileStream << value << "\n";
llvm::outs() << value << "\n";
// compiler.context().getTranslationUnitDecl()->dump();
});
}