caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

some update.

Browse Source
This commit is contained in:
ykiko
2024-09-16 11:14:14 +08:00
parent b2fc464d86
commit 53aeb39f16
11 changed files with 217 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@@ -4,7 +4,7 @@ project(CLICE)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE} -w -fno-rtti -g -O0")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti -g -O0")
set(LLVM_INSTALL_PATH "${CMAKE_SOURCE_DIR}/deps/llvm/build-install")

0
docs/semantic-tokens.md Normal file
View File

5
include/AST/Directive.h
View File

@@ -5,9 +5,10 @@ namespace clice {
// TODO:
struct Directive {
clang::SourceManager& sourceManager;
llvm::StringSet<> includes;
std::vector<clang::SourceRange> comments;
llvm::StringMap<std::vector<clang::SourceRange>> comments;
clang::CommentHandler* handler();
std::unique_ptr<clang::PPCallbacks> callback();
};

27
include/Protocol/SemanticTokens.def
View File

@@ -72,17 +72,6 @@ SEMANTIC_TOKEN_TYPE(Method, "method")
/// C/C++ function/method parameter name.
SEMANTIC_TOKEN_TYPE(Parameter, "parameter")
/// C++ dependent name in a template context (e.g., `name` in `auto y = T::name`
/// where T is a template parameter).
/// Note: This includes `T::template name(...)`; it's not possible to distinguish whether a name
/// is a function or a functor in a dependent context.
SEMANTIC_TOKEN_TYPE(DependentName, "dependentName")
/// C++ dependent type name (e.g., `type` in `typename std::vector<T>::type` where
/// T is a template parameter).
/// Note: This includes `typename T::template type<...>`.
SEMANTIC_TOKEN_TYPE(DependentType, "dependentType")
/// C++ attribute name(e.g., `nodiscard`, `likely`, `fallthrough`).
SEMANTIC_TOKEN_TYPE(Attribute, "attribute")
@@ -124,3 +113,19 @@ SEMANTIC_TOKEN_MODIFIER(InType, "inType")
/// for overloaded function or operator.
SEMANTIC_TOKEN_MODIFIER(Overloaded, "overloaded")
/// for name around template arguments(e.g., `foo` in `foo<int>`).
SEMANTIC_TOKEN_MODIFIER(Templated, "templated")
/// C++ dependent name in a template context (e.g., `name` in `auto y = T::name`
/// where T is a template parameter).
/// Note: This includes `T::template name(...)`; it's not possible to distinguish whether a name
/// is a function or a functor in a dependent context.
/// C++ dependent type name (e.g., `type` in `typename std::vector<T>::type` where
/// T is a template parameter).
/// Note: This includes `typename T::template type<...>`.
SEMANTIC_TOKEN_MODIFIER(Dependent, "dependent")
#undef SEMANTIC_TOKEN_TYPE
#undef SEMANTIC_TOKEN_MODIFIER

2
include/Protocol/SemanticTokens.h
View File

@@ -23,13 +23,11 @@ struct SemanticTokensLegend {
std::array<std::string_view, SemanticTokenType::LAST_TYPE> tokenTypes = {
#define SEMANTIC_TOKEN_TYPE(name, value) value,
#include "SemanticTokens.def"
#undef SEMANTIC_TOKEN_TYPE
};
std::array<std::string_view, SemanticTokenModifier::LAST_MODIFIER> tokenModifiers = {
#define SEMANTIC_TOKEN_MODIFIER(name, value) value,
#include "SemanticTokens.def"
#undef SEMANTIC_TOKEN_MODIFIER
};
}; // clang-format on

39
include/Support/Reflection.h
View File

@@ -1,6 +1,7 @@
#pragma once
// support basic reflection through template meta programming
#include <bit>
#include <tuple>
#include <string_view>
#include <source_location>
@@ -38,7 +39,7 @@ struct Wrapper {
};
template <Wrapper T>
constexpr auto member_name() {
consteval auto member_name() {
std::string_view name = std::source_location::current().function_name();
#if __GNUC__ && (!__clang__) && (!_MSC_VER)
std::size_t start = name.rfind("::") + 2;
@@ -163,6 +164,32 @@ struct Storage {
inline static T value;
};
template <auto value>
consteval auto enum_name() {
std::string_view name = std::source_location::current().function_name();
#if __GNUC__ || __clang__
std::size_t start = name.find('=') + 2;
std::size_t end = name.size() - 1;
#elif _MSC_VER
std::size_t start = name.find('<') + 1;
std::size_t end = name.rfind(">(");
#else
static_assert(false, "Not supported compiler");
#endif
name = name.substr(start, end - start);
start = name.rfind("::");
return start == std::string_view::npos ? name : name.substr(start + 2);
}
template <typename T, std::size_t N = 0>
consteval auto enum_max() {
constexpr auto value = std::bit_cast<T>(static_cast<std::underlying_type_t<T>>(N));
if constexpr(enum_name<value>().find(")") == std::string_view::npos)
return enum_max<T, N + 1>();
else
return N;
}
template <typename Source, typename Target>
struct replace_cv_ref;
@@ -234,4 +261,14 @@ struct Record : Ts... {
}
};
template <typename T>
requires std::is_enum_v<T>
constexpr auto enum_name(T value) {
constexpr auto count = impl::enum_max<T>();
constexpr auto names = []<std::size_t... Is>(std::index_sequence<Is...>) {
return std::array{impl::enum_name<static_cast<T>(Is)>()...};
}(std::make_index_sequence<count>{});
return names[static_cast<std::size_t>(value)];
}
}; // namespace clice

87
src/AST/Directive.cpp
View File

@@ -1,5 +1,5 @@
#include "AST/Directive.h"
#include <Support/Reflection.h>
namespace clice {
struct CommentHandler : public clang::CommentHandler {
@@ -8,40 +8,87 @@ struct CommentHandler : public clang::CommentHandler {
CommentHandler(Directive& directive) : directive(directive) {}
virtual bool HandleComment(clang::Preprocessor& preproc, clang::SourceRange Comment) {
directive.comments.push_back(Comment);
// directive.comments.push_back(Comment);
return false;
}
};
struct PPCallback : clang::PPCallbacks {
virtual void InclusionDirective(clang::SourceLocation HashLoc,
const clang::Token& IncludeTok,
llvm::StringRef FileName,
bool IsAngled,
clang::CharSourceRange FilenameRange,
clang::OptionalFileEntryRef File,
clang::StringRef SearchPath,
llvm::StringRef RelativePath,
const clang::Module* SuggestedModule,
bool ModuleImported,
clang::SrcMgr::CharacteristicKind FileType) {
PPCallback(Directive& directive) : directive(directive) {}
Directive& directive;
void FileChanged(clang::SourceLocation loc,
clang::PPCallbacks::FileChangeReason reason,
clang::SrcMgr::CharacteristicKind fileType,
clang::FileID id) override {
llvm::outs() << "FileChanged, reason: " << enum_name(reason) << "\n";
loc.dump(directive.sourceManager);
}
void InclusionDirective(clang::SourceLocation HashLoc,
const clang::Token& IncludeTok,
llvm::StringRef FileName,
bool IsAngled,
clang::CharSourceRange FilenameRange,
clang::OptionalFileEntryRef File,
clang::StringRef SearchPath,
llvm::StringRef RelativePath,
const clang::Module* SuggestedModule,
bool ModuleImported,
clang::SrcMgr::CharacteristicKind FileType) override {
// TODO: record all include files
namespace fs = llvm::sys::fs;
namespace path = llvm::sys::path;
llvm::SmallVector<char> RealPath;
// fs::make_absolute(SearchPath + "/" + RelativePath, RealPath);
// path::remove_dots(RealPath, /*remove_dot_dot=*/true);
// llvm::outs() << RealPath << "\n";
}
// virtual void
// moduleImport(clang::SourceLocation ImportLoc, clang::ModuleIdPath Path, const clang::Module* Imported) {
// // store for highlight
// }
void PragmaDirective(clang::SourceLocation Loc, clang::PragmaIntroducerKind Introducer) override {
// llvm::outs() << "PragmaDirective\n";
}
void If(clang::SourceLocation Loc,
clang::SourceRange ConditionRange,
clang::PPCallbacks::ConditionValueKind ConditionValue) override {
// llvm::outs() << "If\n";
}
void Elif(clang::SourceLocation loc,
clang::SourceRange conditionRange,
clang::PPCallbacks::ConditionValueKind conditionValue,
clang::SourceLocation ifLoc) override {}
void Ifdef(clang::SourceLocation loc, const clang::Token& name, const clang::MacroDefinition& definition) override {
}
void Elifdef(clang::SourceLocation loc,
const clang::Token& name,
const clang::MacroDefinition& definition) override {}
void Elifdef(clang::SourceLocation loc, clang::SourceRange conditionRange, clang::SourceLocation ifLoc) override {}
void Ifndef(clang::SourceLocation loc,
const clang::Token& name,
const clang::MacroDefinition& definition) override {}
// invoke when #elifndef is taken
void Elifndef(clang::SourceLocation loc,
const clang::Token& name,
const clang::MacroDefinition& definition) override {}
// invoke when #elifndef is skipped
void Elifndef(clang::SourceLocation loc, clang::SourceRange conditionRange, clang::SourceLocation ifLoc) override {}
void Else(clang::SourceLocation loc, clang::SourceLocation ifLoc) override {}
void Endif(clang::SourceLocation loc, clang::SourceLocation ifLoc) override {}
void MacroDefined(const clang::Token& MacroNameTok, const clang::MacroDirective* MD) override {}
};
clang::CommentHandler* Directive::handler() { return new CommentHandler(*this); }
std::unique_ptr<clang::PPCallbacks> Directive::callback() { return std::make_unique<PPCallback>(); }
std::unique_ptr<clang::PPCallbacks> Directive::callback() { return std::make_unique<PPCallback>(*this); }
} // namespace clice

2
src/AST/ParsedAST.cpp
View File

@@ -65,7 +65,7 @@ std::unique_ptr<ParsedAST> ParsedAST::build(llvm::StringRef filename,
auto& preproc = instance->getPreprocessor();
clang::syntax::TokenCollector collector(preproc);
auto directive = std::make_unique<Directive>();
auto directive = std::make_unique<Directive>(instance->getSourceManager());
preproc.addCommentHandler(directive->handler());
preproc.addPPCallbacks(directive->callback());

95
src/Feature/SemanticTokens.cpp
View File

@@ -160,6 +160,8 @@ static bool isKeyword(clang::tok::TokenKind kind, llvm::StringRef text, const cl
case clang::tok::kw_co_return: {
return option.CPlusPlus20;
}
default: break;
}
return false;
}
@@ -178,6 +180,7 @@ public:
if(token) {
return addToken(type, token->location(), token->length());
}
}
// FIXME: source range can be a multi-line range, split it into multiple tokens
@@ -191,7 +194,34 @@ public:
// return result.back();
//}
void addAngle(clang::SourceLocation left, clang::SourceLocation right) {}
void addAngle(clang::SourceLocation left, clang::SourceLocation right) {
if(left.isInvalid() || right.isInvalid()) {
return;
}
llvm::outs() << "is macro?: " << left.isMacroID() << " ";
left.dump(AST.sourceManager);
llvm::outs() << "is macro?: " << right.isMacroID() << " ";
right.dump(AST.sourceManager);
if(auto token = AST.tokenBuffer.spelledTokenContaining(left)) {
addToken(protocol::SemanticTokenType::Angle, token->location(), token->length())
.addModifier(protocol::SemanticTokenModifier::Left);
}
// RLoc might be pointing at a virtual buffer when it's part of a `>>` token.
auto loc = AST.sourceManager.getFileLoc(right);
if(auto token = AST.tokenBuffer.spelledTokenContaining(loc)) {
if(token->kind() == clang::tok::greater) {
addToken(protocol::SemanticTokenType::Angle, loc, 1)
.addModifier(protocol::SemanticTokenModifier::Right);
} else if(token->kind() == clang::tok::greatergreater) {
// TODO: split `>>` into two tokens
addToken(protocol::SemanticTokenType::Angle, loc, 2)
.addModifier(protocol::SemanticTokenModifier::Right);
}
}
}
std::vector<SemanticToken> build();
@@ -225,13 +255,20 @@ public:
// WalkUpFrom(NamespaceDecl) {}
VISIT(ImportDecl) {}
VISIT(ImportDecl) { return true; }
VISIT(NamedDecl) { return true; }
VISIT(NamespaceDecl) {
builder.addToken(protocol::SemanticTokenType::Namespace, node->getLocation());
return true;
}
VISIT(VarDecl) {
builder.addToken(protocol::SemanticTokenType::Variable, node->getLocation());
return true;
}
VISIT(DeclaratorDecl) {
for(unsigned i = 0; i < node->getNumTemplateParameterLists(); ++i) {
if(auto params = node->getTemplateParameterList(i)) {
@@ -293,13 +330,30 @@ public:
return true;
}
VISIT(OverloadExpr) {
builder.addAngle(node->getLAngleLoc(), node->getRAngleLoc());
return true;
}
VISIT(DeclRefExpr) {
builder.addToken(protocol::SemanticTokenType::Variable, node->getLocation());
node->getEnumConstantDecl();
builder.addAngle(node->getLAngleLoc(), node->getRAngleLoc());
return true;
}
VISIT(CXXNamedCastExpr) {
builder.addAngle(node->getAngleBrackets().getBegin(), node->getAngleBrackets().getEnd());
return true;
}
VISIT(OverloadExpr) {
VISIT(DependentScopeDeclRefExpr) {
// `T::value<...>`
// ^ ^^^^^~~~ Angles
// ^~~~ DependentValue
builder.addAngle(node->getLAngleLoc(), node->getRAngleLoc());
builder.addToken(protocol::SemanticTokenType::Variable, node->getLocation())
.addModifier(protocol::SemanticTokenModifier::Dependent);
return true;
}
@@ -308,21 +362,35 @@ public:
return true;
}
VISIT(DependentScopeDeclRefExpr) {
builder.addAngle(node->getLAngleLoc(), node->getRAngleLoc());
VISIT_TYPE(RecordTypeLoc) {
// `struct X x;`
// ^ Type
builder.addToken(protocol::SemanticTokenType::Type, node.getNameLoc());
return true;
}
VISIT_TYPE(DependentNameTypeLoc) {
// DependentNameType: `typename T::type`
// ^~~~ highlight this
// `typename T::type`
// ^~~~ DependentType
builder.addToken(protocol::SemanticTokenType::Type, node.getNameLoc())
.addModifier(protocol::SemanticTokenModifier::Dependent);
return true;
}
VISIT_TYPE(TemplateTypeParmTypeLoc) {
// `typename T::type`
// ^~~~ Type
builder.addToken(protocol::SemanticTokenType::Type, node.getNameLoc());
return true;
}
VISIT_TYPE(TemplateSpecializationTypeLoc) {
node.dump();
// `Template<...>`
// ^ ^~~~ Angles
// ^~~~ Type
builder.addAngle(node.getLAngleLoc(), node.getRAngleLoc());
builder.addToken(protocol::SemanticTokenType::Type, node.getTemplateNameLoc())
.addModifier(protocol::SemanticTokenModifier::Templated);
return true;
}
@@ -334,9 +402,10 @@ public:
bool TraverseNestedNameSpecifierLoc(clang::NestedNameSpecifierLoc loc) {
if(clang::NestedNameSpecifier* NNS = loc.getNestedNameSpecifier()) {
if(NNS->getKind() == clang::NestedNameSpecifier::Identifier) {
// NestedNameSpecifier: `T::type::`
// ^~~~ highlight this
builder.addToken(protocol::SemanticTokenType::Type, loc.getLocalBeginLoc());
// `T::type::`
// ^~~~ DependentType
builder.addToken(protocol::SemanticTokenType::Type, loc.getLocalBeginLoc())
.addModifier(protocol::SemanticTokenModifier::Dependent);
}
}
return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(loc);
@@ -409,10 +478,6 @@ protocol::SemanticTokens semanticTokens(const ParsedAST& AST, llvm::StringRef fi
HighlightBuilder builder(AST, filename);
std::vector<SemanticToken> tokens = builder.build();
// for(auto& token: tokens) {
// spdlog::info("{}", token.dump(AST.sourceManager));
// }
protocol::SemanticTokens result;
unsigned int last_line = 0;
unsigned int last_column = 0;

31
tests/Feature/SemanticTokens.cpp
View File

@@ -12,33 +12,24 @@ auto f() {
return T::value;
}
std::vector<const char*> compileArgs = {
"clang++",
"-std=c++20",
"main.cpp",
"-resource-dir=/home/ykiko/C++/clice2/build/lib/clang/20",
};
TEST(test, test) {
std::vector<const char*> compileArgs = {
"clang++",
"-std=c++20",
"main.cpp",
"-resource-dir=/home/ykiko/C++/clice2/build/lib/clang/20",
};
#include <cstdio>
const char* code = R"(
template<typename T>
struct X {
using type = T;
static constexpr bool value = true;
};
template <typename T>
auto f() {
using type = typename X<T>::type::type;
return T::value;
}
#include <cstdio>
)";
auto AST = clice::ParsedAST::build("main.cpp", code, compileArgs);
auto fileID = AST->getFileID("main.cpp");
AST->context.getTranslationUnitDecl()->dump();
auto semanticTokens = clice::feature::semanticTokens(*AST, "main.cpp");
// AST->context.getTranslationUnitDecl()->dump();
// auto semanticTokens = clice::feature::semanticTokens(*AST, "main.cpp");
}
} // namespace

44
tests/test.cpp
View File

@@ -1,44 +0,0 @@
#include <gtest/gtest.h>
#include <AST/ParsedAST.h>
#include <Feature/SemanticTokens.h>
namespace {
using namespace clang;
template <typename T>
auto f() {
using type = typename T::type;
return T::value;
}
TEST(test, test) {
std::vector<const char*> compileArgs = {
"clang++",
"-std=c++20",
"main.cpp",
"-resource-dir=/home/ykiko/C++/clice2/build/lib/clang/20",
};
const char* code = R"(
template<typename T>
struct X {
using type = T;
static constexpr bool value = true;
};
template <typename T>
auto f() {
using type = typename X<T>::type::type;
return T::value;
}
)";
auto AST = clice::ParsedAST::build("main.cpp", code, compileArgs);
auto fileID = AST->getFileID("main.cpp");
AST->context.getTranslationUnitDecl()->dump();
auto semanticTokens = clice::feature::semanticTokens(*AST, "main.cpp");
}
} // namespace