From d030bc28009e8af57d01f9cc47b95c67ed0049c5 Mon Sep 17 00:00:00 2001 From: ykiko Date: Sat, 21 Sep 2024 11:48:55 +0800 Subject: [PATCH] some update. --- .vscode/launch.json | 6 ++-- docs/design.md | 16 +++++++++ docs/roadmap.md | 2 ++ include/AST/Compiler.h | 19 +++++++++++ include/AST/Directive.h | 1 + include/AST/Preamble.h | 5 +-- include/AST/Selection.h | 26 +++++++++++++-- include/Index/Index.h | 22 +++++++++++++ include/Index/Protocol.h | 52 +++++++++++++++++++++++++++++ include/Support/ADT.h | 17 ++++++++++ src/AST/Compiler.cpp | 56 ++++++++++++++++++++++++++++++++ src/AST/Directive.cpp | 36 ++++++++++++++++++-- src/AST/ParsedAST.cpp | 49 +++------------------------- src/AST/Preamble.cpp | 24 +++++++++++--- tests/Feature/SemanticTokens.cpp | 17 +++++++--- tests/Support/TokenBuffer.cpp | 3 ++ tests/Support/URI.cpp | 16 ++++----- 17 files changed, 296 insertions(+), 71 deletions(-) create mode 100644 docs/design.md create mode 100644 include/AST/Compiler.h create mode 100644 include/Index/Index.h create mode 100644 include/Index/Protocol.h create mode 100644 include/Support/ADT.h create mode 100644 src/AST/Compiler.cpp create mode 100644 tests/Support/TokenBuffer.cpp diff --git a/.vscode/launch.json b/.vscode/launch.json index ff9c6735..f62389b5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,18 +8,18 @@ "type": "lldb", "request": "launch", "name": "clice_socket", - "program": "./build/bin/clice" + "program": "${workspaceFolder}/build/bin/clice" }, { "type": "lldb", "request": "attach", - "name": "clice", + "name": "clice_attach", "program": "clice" }, { "type": "lldb", "request": "launch", - "name": "DependentNameResolver", + "name": "clice_test", "program": "${workspaceFolder}/build/bin/clice_test", "cwd": "${workspaceFolder}" } diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 00000000..d35bce0e --- /dev/null +++ b/docs/design.md @@ -0,0 +1,16 @@ +clice 充分吸取了 clangd 教训，用一些全新的设计来解决问题。 + + +- 默认索引所有文件 +- 对于只读文件，使用 LSIF 返回请求结果，speed speed speed !!!! +- 对于可变文件，为它们构建 preamble，以加速后续的构建，然后跑代码补全之类的行为 + +- clice 如何区分只读文件和可变文件？一般来说，当您第一次尝试编辑这个文件的时候，clice 就会认为这是一个可变的文件，并为其构建 preamble。但是很多情况下可能只是不小心碰到了键盘，并不是真的想要修改，可以在设置里面配置选项，使得第一次更改并报错之后才会让 clice 认为这是一个活跃文件。另外，使用 code action 也会触发。 + +# non self contained + +clice 解决了一个 clangd 里面长期存在的问题，non self contained 的头文件，这种技巧常见于 glibc 或者 libstdc++ 这样的代码库中。non self-contained file 指的是那些不能单独编译的源文件。只有把他们一同拼接到一个文件中，才能编译通过，而且对于这种文件，往往 include 顺序也十分重要，建议关闭头文件顺序重排。 + +对于 non self contained 的头文件来说，只要生成的编译数据库中，有包含它的源文件，并且能正常编译，那么 clice 就可以正确为它生成索引，从而支持转到定义等等等操作。 + +那么 non self contained 的文件对于可变操作如何呢？TODO: ... 初步设想，手动通过字符串拼接的方式来补全缺少的头文件 ... \ No newline at end of file diff --git a/docs/roadmap.md b/docs/roadmap.md index ffe7747b..a1765b95 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -20,3 +20,5 @@ - Go to implementation 常用于跳转到虚函数实现，部分情况下。特别的，如果 clice 能发现你这个类型是一个 CRTP 类型，那么它也可以进行对应的跳转。 + +# diff --git a/include/AST/Compiler.h b/include/AST/Compiler.h new file mode 100644 index 00000000..83ca0bb3 --- /dev/null +++ b/include/AST/Compiler.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace clice { + +// TODO: + +class Preamble; + +std::unique_ptr createInvocation(StringRef filename, + StringRef content, + std::vector& args, + Preamble* preamble = nullptr); + +std::unique_ptr createInstance(std::shared_ptr invocation); + +} // namespace clice diff --git a/include/AST/Directive.h b/include/AST/Directive.h index ea8bbf74..7d5a3bb8 100644 --- a/include/AST/Directive.h +++ b/include/AST/Directive.h @@ -36,6 +36,7 @@ struct Directive { }; struct Directives { + clang::Preprocessor& preproc; clang::SourceManager& sourceManager; llvm::DenseMap x; diff --git a/include/AST/Preamble.h b/include/AST/Preamble.h index cbc997f5..95f4aa4b 100644 --- a/include/AST/Preamble.h +++ b/include/AST/Preamble.h @@ -13,8 +13,9 @@ namespace clice { struct Preamble { clang::PrecompiledPreamble data; - static std::unique_ptr - build(llvm::StringRef filename, llvm::StringRef content, std::vector& args); + static std::unique_ptr build(llvm::StringRef filename, + llvm::StringRef content, + std::vector& args); }; } // namespace clice diff --git a/include/AST/Selection.h b/include/AST/Selection.h index 196c295e..709074f2 100644 --- a/include/AST/Selection.h +++ b/include/AST/Selection.h @@ -1,9 +1,31 @@ #include "ParsedAST.h" +#include namespace clice { -class Selection {}; +class SelectionTree { +public: + enum Selection { + Unselected, + Partial, + Complete, + }; -clang::Decl* test(clang::syntax::Token* token, ParsedAST& AST, llvm::StringRef filename); + struct Node { + Node* parent; + llvm::SmallVector children; + clang::DynTypedNode ASTNode; + }; + + const Node* commonAncestor() const; + + const Node& root() const; + + SelectionTree(clang::ASTContext& context, const clang::syntax::TokenBuffer& tokens, unsigned start, unsigned end); + +private: + std::deque m_Nodes; + const Node* m_Root; +}; } // namespace clice diff --git a/include/Index/Index.h b/include/Index/Index.h new file mode 100644 index 00000000..c7db5e11 --- /dev/null +++ b/include/Index/Index.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace clice { + +class IndexConsumer : public clang::index::IndexDataConsumer { +public: + void initialize(clang::ASTContext& Ctx) override {} + + void setPreprocessor(std::shared_ptr PP) override {} + + bool handleDeclOccurrence(const clang::Decl* D, + clang::index::SymbolRoleSet Roles, + llvm::ArrayRef Relations, + clang::SourceLocation Loc, + clang::index::IndexDataConsumer::ASTNodeInfo ASTNode) override { + return true; + } +}; + +} // namespace clice diff --git a/include/Index/Protocol.h b/include/Index/Protocol.h new file mode 100644 index 00000000..f831c868 --- /dev/null +++ b/include/Index/Protocol.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +namespace clice { + +struct SymbolID {}; + +struct Range {}; + +enum class SymbolKind : uint8_t { + +}; + +struct Relation { + SymbolID target; + bool isReference = false; + bool isImplementation = false; + bool isTypeDefinition = false; + bool isBase = false; + bool isOverride = false; +}; + +struct Symbol { + SymbolID symbol; + SymbolKind kind; + StringRef displayName; + StringRef document; + std::vector relations; +}; + +enum class SymbolRole : uint8_t { + Definition, + Import, + WriteAccess, + ReadAccess, + ForwardDeclaration, +}; + +struct Occurrence { + SymbolID symbol; + Range range; + SymbolRole role; +}; + +struct Document { + StringRef uri; + std::vector symbols; + std::vector occurrences; +}; + +}; // namespace clice diff --git a/include/Support/ADT.h b/include/Support/ADT.h new file mode 100644 index 00000000..b966f542 --- /dev/null +++ b/include/Support/ADT.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace clice { +using llvm::ArrayRef; +using llvm::StringRef; +using llvm::SmallVector; +using llvm::SmallString; +} // namespace clice diff --git a/src/AST/Compiler.cpp b/src/AST/Compiler.cpp new file mode 100644 index 00000000..6f428cec --- /dev/null +++ b/src/AST/Compiler.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include + +namespace clice { + +static void setInvocation(clang::CompilerInvocation& invocation) { + clang::LangOptions& langOpts = invocation.getLangOpts(); + langOpts.CommentOpts.ParseAllComments = true; + langOpts.RetainCommentsFromSystemHeaders = true; +} + +std::unique_ptr + createInvocation(StringRef filename, StringRef content, std::vector& args, Preamble* preamble) { + clang::CreateInvocationOptions options; + // FIXME: explore VFS + auto vfs = llvm::vfs::getRealFileSystem(); + + auto invocation = clang::createInvocation(args, options); + + setInvocation(*invocation); + + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content, filename); + if(preamble) { + auto bounds = clang::ComputePreambleBounds(invocation->getLangOpts(), *buffer, false); + // check if the preamble can be reused + if(preamble->data.CanReuse(*invocation, *buffer, bounds, *vfs)) { + llvm::outs() << "Resued preamble\n"; + // reuse preamble + preamble->data.AddImplicitPreamble(*invocation, vfs, buffer.release()); + } + } else { + invocation->getPreprocessorOpts().addRemappedFile(invocation->getFrontendOpts().Inputs[0].getFile(), + buffer.release()); + } + + return invocation; +} + +std::unique_ptr createInstance(std::shared_ptr invocation) { + auto instance = std::make_unique(); + + instance->setInvocation(std::move(invocation)); + // FIXME: resolve diagnostics + instance->createDiagnostics(new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), true); + + if(!instance->createTarget()) { + llvm::errs() << "Failed to create target\n"; + std::terminate(); + } + + return instance; +} + +} // namespace clice diff --git a/src/AST/Directive.cpp b/src/AST/Directive.cpp index 0245206f..a56df0d0 100644 --- a/src/AST/Directive.cpp +++ b/src/AST/Directive.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace clice { @@ -12,7 +13,7 @@ struct CommentHandler : public clang::CommentHandler { // directive.comments.push_back(Comment); auto start = directive.sourceManager.getCharacterData(Comment.getBegin()); auto end = directive.sourceManager.getCharacterData(Comment.getEnd()); - llvm::outs() << "Comment: " << llvm::StringRef(start, end - start) << "\n"; + // llvm::outs() << "Comment: " << llvm::StringRef(start, end - start) << "\n"; return false; } }; @@ -58,7 +59,7 @@ struct PPCallback : clang::PPCallbacks { void If(clang::SourceLocation Loc, clang::SourceRange ConditionRange, clang::PPCallbacks::ConditionValueKind ConditionValue) override { - llvm::outs() << "If\n"; + // llvm::outs() << "If\n"; } void Elif(clang::SourceLocation loc, @@ -92,6 +93,37 @@ struct PPCallback : clang::PPCallbacks { void Endif(clang::SourceLocation loc, clang::SourceLocation ifLoc) override {} void MacroDefined(const clang::Token& MacroNameTok, const clang::MacroDirective* MD) override {} + + void MacroExpands(const clang::Token& MacroNameTok, + const clang::MacroDefinition& MD, + clang::SourceRange Range, + const clang::MacroArgs* Args) override { + // llvm::outs() << "------------------------\n"; + // auto info = MD.getMacroInfo(); + // if(info->isObjectLike()) { + // Range = clang::SourceRange(MacroNameTok.getLocation(), MacroNameTok.getEndLoc()); + // } else if(info->isFunctionLike()) { + // // MacroNameTok.getLocation().dump(directive.sourceManager); + // } + // Range.getBegin().dump(directive.sourceManager); + // Range.getEnd().dump(directive.sourceManager); + // auto s = directive.sourceManager.getCharacterData(Range.getBegin()); + // auto e = directive.sourceManager.getCharacterData(Range.getEnd()); + //// llvm::outs() << llvm::StringRef(s, e - s) << "\n"; + // + // llvm::outs() << directive.preproc.getSpelling(MacroNameTok) << " "; + // if(Args) { + // auto size = Args->getNumMacroArguments(); + // for(int i = 0; i < size; i++) { + // auto arg = Args->getUnexpArgument(i); + // auto len = Args->getArgLength(arg); + // for(int j = 0; j < len; j++) { + // llvm::outs() << directive.preproc.getSpelling(arg[j]) << " "; + // } + // } + //} + // llvm::outs() << "\n"; + } }; clang::CommentHandler* Directives::handler() { return new CommentHandler(*this); } diff --git a/src/AST/ParsedAST.cpp b/src/AST/ParsedAST.cpp index 4b6ee4a7..d5d64715 100644 --- a/src/AST/ParsedAST.cpp +++ b/src/AST/ParsedAST.cpp @@ -1,14 +1,8 @@ #include -#include "clang/Lex/PreprocessorOptions.h" +#include namespace clice { -static void setInvocation(clang::CompilerInvocation& invocation) { - clang::LangOptions& langOpts = invocation.getLangOpts(); - langOpts.CommentOpts.ParseAllComments = true; - langOpts.RetainCommentsFromSystemHeaders = true; -} - std::unique_ptr ParsedAST::build(llvm::StringRef filename, llvm::StringRef content, std::vector& args, @@ -18,43 +12,8 @@ std::unique_ptr ParsedAST::build(llvm::StringRef filename, clang::CreateInvocationOptions options; // options.VFS = vfs; - auto invocation = std::make_shared(); - invocation = clang::createInvocation(args, options); - - auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content, filename); - if(preamble) { - - auto bounds = clang::ComputePreambleBounds(invocation->getLangOpts(), *buffer, false); - - // check if the preamble can be reused - if(preamble->data.CanReuse(*invocation, *buffer, bounds, *vfs)) { - llvm::outs() << "Resued preamble\n"; - // reuse preamble - preamble->data.AddImplicitPreamble(*invocation, vfs, buffer.release()); - } - } else { - invocation->getPreprocessorOpts().addRemappedFile(invocation->getFrontendOpts().Inputs[0].getFile(), - buffer.release()); - } - - invocation->getLangOpts().CommentOpts.ParseAllComments = true; - - auto instance = std::make_unique(); - instance->setInvocation(std::move(invocation)); - - instance->createDiagnostics(new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), true); - - // if(auto remappingVSF = - // createVFSFromCompilerInvocation(instance->getInvocation(), instance->getDiagnostics(), vfs)) { - // vfs = remappingVSF; - // } - // instance->createFileManager(vfs); - - if(!instance->createTarget()) { - llvm::errs() << "Failed to create target\n"; - std::terminate(); - } - + auto invocation = createInvocation(filename, content, args, preamble); + auto instance = createInstance(std::move(invocation)); auto action = std::make_unique(); if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { @@ -65,7 +24,7 @@ std::unique_ptr ParsedAST::build(llvm::StringRef filename, auto& preproc = instance->getPreprocessor(); clang::syntax::TokenCollector collector(preproc); - auto directive = std::make_unique(instance->getSourceManager()); + auto directive = std::make_unique(instance->getPreprocessor(), instance->getSourceManager()); preproc.addCommentHandler(directive->handler()); preproc.addPPCallbacks(directive->callback()); diff --git a/src/AST/Preamble.cpp b/src/AST/Preamble.cpp index 94325538..397d6347 100644 --- a/src/AST/Preamble.cpp +++ b/src/AST/Preamble.cpp @@ -2,10 +2,26 @@ namespace clice { -std::unique_ptr - Preamble::build(llvm::StringRef filename, llvm::StringRef content, std::vector& args) { +class PreambleCallback : public clang::PreambleCallbacks { +public: + std::optional collector; + +public: + void BeforeExecute(clang::CompilerInstance& CI) override { collector.emplace(CI.getPreprocessor()); } + + void AfterExecute(clang::CompilerInstance& CI) override { + auto tokens = std::move(collector.value()).consume(); + for(auto& token: tokens.expandedTokens()) { + llvm::outs() << token.text(CI.getSourceManager()) << "\n"; + } + } +}; + +std::unique_ptr Preamble::build(llvm::StringRef filename, + llvm::StringRef content, + std::vector& args) { auto invocation = clang::createInvocation(args, {}); - auto buffer = llvm::MemoryBuffer::getMemBuffer(content, filename); + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content, filename); // compute preamble bounds, i.e. the range of the preamble. // if MaxLines set to false(0), i.e. limit the number of lines. @@ -19,7 +35,7 @@ std::unique_ptr // use to collect information in the process of building preamble, such as include files and macros // TODO: inherit from clang::PreambleCallbacks and collect the information - clang::PreambleCallbacks callbacks = {}; + PreambleCallback callbacks = {}; auto preamble = clang::PrecompiledPreamble::Build(*invocation, buffer.get(), diff --git a/tests/Feature/SemanticTokens.cpp b/tests/Feature/SemanticTokens.cpp index 55e9a995..1c9600dd 100644 --- a/tests/Feature/SemanticTokens.cpp +++ b/tests/Feature/SemanticTokens.cpp @@ -23,14 +23,21 @@ TEST(test, test) { #include const char* code = R"( -// 123 -#if x -#endif -// 333 +void f(); + +void f() {} )"; - auto AST = clice::ParsedAST::build("main.cpp", code, compileArgs); + auto preamble = clice::Preamble::build("main.cpp", code, compileArgs); + auto AST = clice::ParsedAST::build("main.cpp", code, compileArgs, preamble.get()); auto fileID = AST->getFileID("main.cpp"); + auto tokens = AST->tokenBuffer.spelledTokens(fileID); + AST->context.getTranslationUnitDecl()->dump(); + + // for(auto& token: tokens) { + // llvm::outs() << token.text(AST->sourceManager) << "\n"; + // } + // AST->context.getTranslationUnitDecl()->dump(); // auto semanticTokens = clice::feature::semanticTokens(*AST, "main.cpp"); } diff --git a/tests/Support/TokenBuffer.cpp b/tests/Support/TokenBuffer.cpp new file mode 100644 index 00000000..f733c9b7 --- /dev/null +++ b/tests/Support/TokenBuffer.cpp @@ -0,0 +1,3 @@ +#include + +TEST(clang, TokenBuffer) {} diff --git a/tests/Support/URI.cpp b/tests/Support/URI.cpp index 8a664171..e494ab5c 100644 --- a/tests/Support/URI.cpp +++ b/tests/Support/URI.cpp @@ -6,25 +6,25 @@ namespace { using namespace clice; TEST(URITest, ConstructorAndAccessors) { - URI uri("https", "reviews.lvm.org", "/D41946"); + URI uri("https", "reviews.llvm.org", "/D41946"); EXPECT_EQ(uri.scheme(), "https"); - EXPECT_EQ(uri.authority(), "reviews.lvm.org"); + EXPECT_EQ(uri.authority(), "reviews.llvm.org"); EXPECT_EQ(uri.body(), "/D41946"); } TEST(URITest, CopyConstructor) { - URI uri1("https", "reviews.lvm.org", "/D41946"); + URI uri1("https", "reviews.llvm.org", "/D41946"); URI uri2(uri1); EXPECT_EQ(uri2.scheme(), "https"); - EXPECT_EQ(uri2.authority(), "reviews.lvm.org"); + EXPECT_EQ(uri2.authority(), "reviews.llvm.org"); EXPECT_EQ(uri2.body(), "/D41946"); } TEST(URITest, EqualityOperator) { - URI uri1("https", "reviews.lvm.org", "/D41946"); - URI uri2("https", "reviews.lvm.org", "/D41946"); + URI uri1("https", "reviews.llvm.org", "/D41946"); + URI uri2("https", "reviews.llvm.org", "/D41946"); URI uri3("http", "example.com", "/index.html"); EXPECT_TRUE(uri1 == uri2); @@ -32,12 +32,12 @@ TEST(URITest, EqualityOperator) { } TEST(URITest, ParseFunction) { - auto expectedUri = URI::parse("https://reviews.lvm.org/D41946"); + auto expectedUri = URI::parse("https://reviews.llvm.org/D41946"); ASSERT_TRUE(static_cast(expectedUri)); URI uri = expectedUri.get(); EXPECT_EQ(uri.scheme(), "https"); - EXPECT_EQ(uri.authority(), "reviews.lvm.org"); + EXPECT_EQ(uri.authority(), "reviews.llvm.org"); EXPECT_EQ(uri.body(), "/D41946"); }