diff --git a/include/Basic/Document.h b/include/Basic/Document.h index 03f2e9d7..91e4c19e 100644 --- a/include/Basic/Document.h +++ b/include/Basic/Document.h @@ -4,6 +4,22 @@ namespace clice::proto { +struct TextDocumentSyncKind : refl::Enum { + using Enum::Enum; + + enum Kind : std::uint8_t { + /// Documents should not be synced at all. + None = 0, + + /// Documents are synced by always sending the full content of the document. + Full = 1, + + /// Documents are synced by sending the full content on open. After that only + /// incremental updates to the document are sent. + Incremental = 2, + }; +}; + struct TextDocumentItem { /// The text document's URI. DocumentUri uri; @@ -34,11 +50,38 @@ struct VersionedTextDocumentIdentifier { integer version; }; +/// An event describing a change to a text document. If only a text is provided +/// it is considered to be the full content of the document. struct TextDocumentContentChangeEvent { - /// The new text of the whole document. + /// The range of the document that changed. + Range range; + + /// The new text for the provided range. string text; }; +struct DidChangeTextDocumentParams { + /// The document that did change. The version number points + /// to the version after all provided content changes have + /// been applied. + VersionedTextDocumentIdentifier textDocument; + + /// The actual content changes. The content changes describe single state + /// changes to the document. So if there are two content changes c1 (at + /// array index 0) and c2 (at array index 1) for a document in state S then + /// c1 moves the document from S to S' and c2 from S' to S''. So c1 is + /// computed on the state S and c2 is computed on the state S'. + /// + /// To mirror the content of a document using change events use the following + /// approach: + /// - start with the same initial content + /// - apply the 'textDocument/didChange' notifications in the order you + /// receive them. + /// - apply the `TextDocumentContentChangeEvent`s in a single notification + /// in the order you receive them. + std::vector contentChanges; +}; + struct TextDocumentPositionParams { /// The text document. TextDocumentIdentifier textDocument; @@ -62,16 +105,6 @@ struct DidOpenTextDocumentParams { TextDocumentItem textDocument; }; -struct DidChangeTextDocumentParams { - /// The document that did change. The version number points - /// to the version after all provided content changes have - /// been applied. - VersionedTextDocumentIdentifier textDocument; - - /// The actual content changes. - std::vector contentChanges; -}; - struct DidSaveTextDocumentParams { /// The document that was saved. TextDocumentIdentifier textDocument; diff --git a/include/Basic/Lifecycle.h b/include/Basic/Lifecycle.h index 768e48bb..268a24ac 100644 --- a/include/Basic/Lifecycle.h +++ b/include/Basic/Lifecycle.h @@ -69,6 +69,15 @@ struct ServerCapabilities { /// /// If omitted it defaults to 'utf-16'. PositionEncodingKind positionEncoding = PositionEncodingKind::UTF16; + + /// Defines how text documents are synced. Is either a detailed structure + /// defining each notification or for backwards compatibility the + /// TextDocumentSyncKind number. If omitted it defaults to + /// `TextDocumentSyncKind.None`. + TextDocumentSyncKind textDocumentSync = TextDocumentSyncKind::Incremental; + + /// The server provides semantic tokens support. + SemanticTokensOptions semanticTokensProvider; }; struct InitializeResult { diff --git a/include/Basic/SourceCode.h b/include/Basic/SourceCode.h new file mode 100644 index 00000000..03223dc5 --- /dev/null +++ b/include/Basic/SourceCode.h @@ -0,0 +1,19 @@ +#pragma once + +#include "clang/Basic/SourceLocation.h" + +namespace clice { + +/// Get the content of the file with the given file ID. +llvm::StringRef getFileContent(const clang::SourceManager& SM, clang::FileID fid); + +/// Get the length of the token at the given location. All SourceLocation instances in the clang +/// AST originate from the start position of tokens, which helps reduce memory usage. When token +/// length information is needed, a simple lexing operation based on the start position can be +/// performed. +std::uint32_t getTokenLength(const clang::SourceManager& SM, clang::SourceLocation location); + +/// Get the spelling of the token at the given location. +llvm::StringRef getTokenSpelling(const clang::SourceManager& SM, clang::SourceLocation location); + +} // namespace clice diff --git a/include/Basic/SourceConverter.h b/include/Basic/SourceConverter.h index 12405f16..dada2ce1 100644 --- a/include/Basic/SourceConverter.h +++ b/include/Basic/SourceConverter.h @@ -26,21 +26,17 @@ public: /// Measure the length (character count) of the content with the specified encoding kind. std::size_t remeasure(llvm::StringRef content) const; + /// Same as above, but input is raw offset to the content beginning. + proto::Position toPosition(llvm::StringRef content, std::uint32_t offset) const; + /// Convert a clang::SourceLocation to a proto::Position according to the /// specified encoding kind. Note that `SourceLocation` in clang is 1-based and /// is always encoded in UTF-8. - proto::Position toPosition(llvm::StringRef content, - clang::SourceLocation location, - const clang::SourceManager& SM) const; - - /// Same as above, but content is retrieved from the `SourceManager`. proto::Position toPosition(clang::SourceLocation location, const clang::SourceManager& SM) const; /// Convert a clang::SourceRange to a proto::Range according to the specified encoding kind. - proto::Range toRange(clang::SourceRange range, const clang::SourceManager& SM) const { - return {toPosition(range.getBegin(), SM), toPosition(range.getEnd(), SM)}; - } + proto::Range toRange(clang::SourceRange range, const clang::SourceManager& SM) const; /// Convert a proto::Position to a file offset in the content with the specified /// encoding kind. diff --git a/include/Compiler/AST.h b/include/Compiler/AST.h new file mode 100644 index 00000000..f5be28de --- /dev/null +++ b/include/Compiler/AST.h @@ -0,0 +1,107 @@ +#pragma once + +#include "Resolver.h" +#include "Directive.h" + +#include "clang/Frontend/CompilerInstance.h" + +namespace clice { + +/// All AST related information needed for language server. +class ASTInfo { +public: + ASTInfo(clang::FileID interested, + std::unique_ptr action, + std::unique_ptr instance, + std::optional resolver, + std::optional buffer, + llvm::DenseMap directives) : + interested(interested), action(std::move(action)), instance(std::move(instance)), + m_resolver(std::move(resolver)), buffer(std::move(buffer)), + m_directives(std::move(directives)) {} + + ASTInfo(const ASTInfo&) = delete; + + ASTInfo(ASTInfo&&) = default; + + ~ASTInfo() { + if(action) { + action->EndSourceFile(); + } + } + +public: + auto& srcMgr() { + return instance->getSourceManager(); + } + + auto& pp() { + return instance->getPreprocessor(); + } + + auto& context() { + return instance->getASTContext(); + } + + auto& sema() { + return instance->getSema(); + } + + auto& tokBuf() { + assert(buffer && "Token buffer is not available"); + return *buffer; + } + + auto& resolver() { + assert(m_resolver && "Template resolver is not available"); + return *m_resolver; + } + + auto& directives() { + return m_directives; + } + + auto tu() { + return instance->getASTContext().getTranslationUnitDecl(); + } + + /// ============================================================================ + /// Utility Functions + /// ============================================================================ + + /// @brief Get the length of the token at the given location. + /// All SourceLocation instances in the Clang AST originate from the start position of tokens, + /// which helps reduce memory usage. When token length information is needed, a simple lexing + /// operation based on the start position can be performed. + auto getTokenLength(clang::SourceLocation loc) { + return clang::Lexer::MeasureTokenLength(loc, srcMgr(), instance->getLangOpts()); + } + + /// @brief Get the spelling of the token at the given location. + llvm::StringRef getTokenSpelling(clang::SourceLocation loc) { + return llvm::StringRef(srcMgr().getCharacterData(loc), getTokenLength(loc)); + } + +private: + /// The interested file ID. For file without header context, it is the main file ID. + /// For file with header context, it is the file ID of header file. + clang::FileID interested; + + /// The frontend action used to build the AST. + std::unique_ptr action; + + /// Compiler instance, responsible for performing the actual compilation and managing the + /// lifecycle of all objects during the compilation process. + std::unique_ptr instance; + + /// The template resolver used to resolve dependent name. + std::optional m_resolver; + + /// Token information collected during the preprocessing. + std::optional buffer; + + /// All diretive information collected during the preprocessing. + llvm::DenseMap m_directives; +}; + +} // namespace clice diff --git a/include/Compiler/Compilation.h b/include/Compiler/Compilation.h new file mode 100644 index 00000000..dd816d00 --- /dev/null +++ b/include/Compiler/Compilation.h @@ -0,0 +1,70 @@ +#pragma once + +#include "AST.h" +#include "Module.h" +#include "Preamble.h" + +namespace clice { + +struct CompilationParams { + /// Source file content. + llvm::StringRef content; + + /// Source file path. + llvm::SmallString<128> srcPath; + + /// Output file path. + llvm::SmallString<128> outPath; + + /// Responsible for storing the arguments. + llvm::SmallString<1024> command; + + /// - If we are building PCH, we need a size to verify the bounds of preamble. That is + /// which source code range the PCH will cover. + /// - If we are building main file AST for header, we need a size to cut off code after the + /// `#include` directive that includes the header to speed up the parsing. + std::optional bounds; + + llvm::IntrusiveRefCntPtr vfs = new ThreadSafeFS(); + + /// Remapped files. Currently, this is only used for testing. + llvm::SmallVector> remappedFiles; + + /// Information about reuse PCH. + std::string pch; + clang::PreambleBounds pchBounds = {0, false}; + + /// Information about reuse PCM(name, path). + llvm::StringMap pcms; + + /// Code completion file:line:column. + llvm::StringRef file = ""; + uint32_t line = 0; + uint32_t column = 0; +}; + +namespace impl { + +/// Create a compiler invocation from the given compilation parameters. +std::unique_ptr createInvocation(CompilationParams& params); + +/// Create a compiler instance from the given compilation parameters. +std::unique_ptr createInstance(CompilationParams& params); + +} // namespace impl + +/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. +/// Note this function will not check whether we need to update the PCH or PCM, caller should check +/// their reusability and update in time. +llvm::Expected compile(CompilationParams& params); + +/// Run code completion at the given location. +llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer); + +/// Build PCH from given file path and content. +llvm::Expected compile(CompilationParams& params, PCHInfo& out); + +/// Build PCM from given file path and content. +llvm::Expected compile(CompilationParams& params, PCMInfo& out); + +} // namespace clice diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h deleted file mode 100644 index 7967934b..00000000 --- a/include/Compiler/Compiler.h +++ /dev/null @@ -1,175 +0,0 @@ -#pragma once - -#include "Clang.h" -#include "Module.h" -#include "Preamble.h" -#include "Resolver.h" -#include "Directive.h" - -#include "Basic/Location.h" -#include "Support/Error.h" - -#include "llvm/ADT/StringSet.h" - -namespace clice { - -struct CompilationParams; - -/// All information about AST. -class ASTInfo { -public: - ASTInfo() = default; - - ASTInfo(std::unique_ptr action, - std::unique_ptr instance, - std::unique_ptr tokBuf, - llvm::DenseMap&& directives, - std::vector deps) : - action(std::move(action)), m_Instance(std::move(instance)), m_TokBuf(std::move(tokBuf)), - m_Directives(std::move(directives)), m_Deps(std::move(deps)) { - m_Resolver = std::make_unique(this->m_Instance->getSema()); - } - - ASTInfo(const ASTInfo&) = delete; - - ASTInfo(ASTInfo&&) = default; - ASTInfo& operator= (ASTInfo&&) = default; - - ~ASTInfo() { - if(action) { - action->EndSourceFile(); - } - } - - auto& sema() { - return m_Instance->getSema(); - } - - auto& context() { - return m_Instance->getASTContext(); - } - - auto& srcMgr() { - return m_Instance->getSourceManager(); - } - - auto& pp() { - return m_Instance->getPreprocessor(); - } - - clang::TranslationUnitDecl* tu() { - return m_Instance->getASTContext().getTranslationUnitDecl(); - } - - auto& tokBuf() { - assert(m_TokBuf && "Token buffer is not available"); - return *m_TokBuf; - } - - auto& resolver() { - return *m_Resolver; - } - - auto& directives() { - return m_Directives; - } - - auto& directive(clang::FileID id) { - return m_Directives[id]; - } - - auto& deps() { - return m_Deps; - } - - auto& instance() { - return *m_Instance; - } - - /// Get the length of the token at the given location. - auto getTokenLength(clang::SourceLocation loc) { - return clang::Lexer::MeasureTokenLength(loc, srcMgr(), m_Instance->getLangOpts()); - } - - /// Get the spelling of the token at the given location. - llvm::StringRef getTokenSpelling(clang::SourceLocation loc) { - return llvm::StringRef(srcMgr().getCharacterData(loc), getTokenLength(loc)); - } - - auto getLocation(clang::SourceLocation loc) { - return srcMgr().getPresumedLoc(loc); - } - -private: - std::unique_ptr action; - std::unique_ptr m_Instance; - std::unique_ptr m_TokBuf; - std::unique_ptr m_Resolver; - llvm::DenseMap m_Directives; - std::vector m_Deps; -}; - -/// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. -/// Note this function will not check whether we need to update the PCH or PCM, caller should check -/// their reusability and update in time. -llvm::Expected compile(CompilationParams& params); - -/// Run code completion at the given location. -llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer); - -struct CompilationParams { - /// Source file content. - llvm::StringRef content; - - /// Source file path. - llvm::SmallString<128> srcPath; - - /// Output file path. - llvm::SmallString<128> outPath; - - /// Responsible for storing the arguments. - llvm::SmallString<1024> command; - - /// - If we are building PCH, we need a size to verify the bounds of preamble. That is - /// which source code range the PCH will cover. - /// - If we are building main file AST for header, we need a size to cut off code after the - /// `#include` directive that includes the header to speed up the parsing. - std::optional bounds; - - /// Computes the preamble bounds for the given content. - /// If the bounds are not provided explicitly, they will be calculated based on the content. - /// - /// - If the header is empty, the bounds can be determined by lexing the source file. - /// - If the header is not empty, the preprocessor must be executed to compute the bounds. - void computeBounds(llvm::StringRef header = ""); - - llvm::IntrusiveRefCntPtr vfs = new ThreadSafeFS(); - - /// Remapped files. Currently, this is only used for testing. - llvm::SmallVector> remappedFiles; - - /// Information about reuse PCH. - std::string pch; - clang::PreambleBounds pchBounds = {0, false}; - - /// Information about reuse PCM(name, path). - llvm::StringMap pcms; - - /// Code completion file:line:column. - llvm::StringRef file = ""; - uint32_t line = 0; - uint32_t column = 0; - - void addPCH(const PCHInfo& info) { - pch = info.path; - /// pchBounds = info.bounds(); - } - - void addPCM(const PCMInfo& info) { - assert((!pcms.contains(info.name) || pcms[info.name] == info.path) && - "Add a different PCM with the same name"); - pcms[info.name] = info.path; - } -}; - -} // namespace clice diff --git a/include/Compiler/Directive.h b/include/Compiler/Directive.h index ef016aa1..a86767ba 100644 --- a/include/Compiler/Directive.h +++ b/include/Compiler/Directive.h @@ -4,17 +4,27 @@ namespace clice { +/// Information about `#include` directive. struct Include { + /// The file id of the included file. If the file is skipped because of + /// include guard, or `#pragma once`, this will be invalid. + clang::FileID fid; + + /// Location of the `include`. + clang::SourceLocation location; +}; + +/// Information about `__has_include` directive. +struct HasInclude { /// The path of the included file. llvm::StringRef path; - /// Location of the directive identifier. - clang::SourceLocation loc; - - /// Range of the filename. - clang::SourceRange range; + /// Location of the filename token start. + clang::SourceLocation location; }; +/// Information about `#if`, `#ifdef`, `#ifndef`, `#elif`, +/// `#elifdef`, `#else`, `#endif` directive. struct Condition { enum class BranchKind : uint8_t { If = 0, @@ -51,6 +61,7 @@ struct Condition { clang::SourceRange conditionRange; }; +/// Information about macro definition, reference and undef. struct MacroRef { enum class Kind : uint8_t { Def = 0, @@ -75,8 +86,10 @@ struct Pragma {}; struct Directive { std::vector includes; + std::vector hasIncludes; std::vector conditions; std::vector macros; + std::vector pragmas; /// Tell preprocessor to collect directives information and store them in `directives`. static void attach(clang::Preprocessor& pp, diff --git a/include/Compiler/Module.h b/include/Compiler/Module.h index 10c740ab..c182c09b 100644 --- a/include/Compiler/Module.h +++ b/include/Compiler/Module.h @@ -43,7 +43,6 @@ std::string scanModuleName(CompilationParams& params); /// collect its module name and dependencies. llvm::Expected scanModule(CompilationParams& params); -/// Build PCM from given file path and content. -llvm::Expected compile(CompilationParams& params, PCMInfo& out); + } // namespace clice diff --git a/include/Compiler/Preamble.h b/include/Compiler/Preamble.h index 9abcf885..3c3081b4 100644 --- a/include/Compiler/Preamble.h +++ b/include/Compiler/Preamble.h @@ -25,10 +25,16 @@ struct PCHInfo { std::vector deps; }; -/// Compute the preamble to build PCH with the given content. -std::string computePreamble(CompilationParams& params); -/// Build PCH from given file path and content. -llvm::Expected compile(CompilationParams& params, PCHInfo& out); +std::uint32_t computePreambleBound(llvm::StringRef content); + +/// Computes the preamble bounds for the given content. +/// If the bounds are not provided explicitly, they will be calculated based on the content. +/// +/// - If the header is empty, the bounds can be determined by lexing the source file. +/// - If the header is not empty, the preprocessor must be executed to compute the bounds. +std::uint32_t computeBounds(CompilationParams& params); + + } // namespace clice diff --git a/include/Compiler/Semantic.h b/include/Compiler/Semantic.h index 76073a75..5072ac13 100644 --- a/include/Compiler/Semantic.h +++ b/include/Compiler/Semantic.h @@ -1,6 +1,6 @@ #pragma once -#include "Compiler.h" +#include "Compilation.h" #include "Resolver.h" #include "Utility.h" diff --git a/include/Feature/FoldingRange.h b/include/Feature/FoldingRange.h index 068bd414..d43c327c 100644 --- a/include/Feature/FoldingRange.h +++ b/include/Feature/FoldingRange.h @@ -1,6 +1,6 @@ #include "Basic/Document.h" #include "Basic/SourceConverter.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" namespace clice { diff --git a/include/Index/SymbolIndex.h b/include/Index/SymbolIndex.h index 7d27427e..eb306b3a 100644 --- a/include/Index/SymbolIndex.h +++ b/include/Index/SymbolIndex.h @@ -3,7 +3,7 @@ #include "ArrayView.h" #include "Basic/RelationKind.h" #include "Basic/SymbolKind.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" #include "Support/JSON.h" namespace clice::index { diff --git a/include/Server/Cache.h b/include/Server/Cache.h index 606d24da..11e3115b 100644 --- a/include/Server/Cache.h +++ b/include/Server/Cache.h @@ -3,9 +3,9 @@ #include #include "Async.h" +#include "Database.h" #include "Compiler/Module.h" #include "Compiler/Preamble.h" -#include "Server/Database.h" #include "llvm/ADT/StringMap.h" diff --git a/include/Server/Config.h b/include/Server/Config.h new file mode 100644 index 00000000..1cff2a1f --- /dev/null +++ b/include/Server/Config.h @@ -0,0 +1,48 @@ +#pragma once + +#include "Support/Support.h" + +namespace clice::config { + +/// Read the config file, call when the program starts. +void load(llvm::StringRef execute, llvm::StringRef filename); + +/// Initialize the config, replace all predefined variables in the config file. +/// called in `Server::initialize`. +void init(std::string_view workplace); + +struct ServerOptions { + std::vector compile_commands_dirs; +}; + +struct CacheOptions { + std::string dir; + uint32_t limit = 0; +}; + +struct IndexOptions { + std::string dir; + bool implicitInstantiation = true; +}; + +struct Rule { + std::string pattern; + std::vector append; + std::vector remove; + std::string readonly; + std::string header; + std::vector context; +}; + +extern llvm::StringRef version; +extern llvm::StringRef binary; +extern llvm::StringRef llvm_version; +extern llvm::StringRef workspace; + +extern const ServerOptions& server; +extern const CacheOptions& cache; +extern const IndexOptions& index; +extern llvm::ArrayRef rules; + +}; // namespace clice::config + diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index 3816f684..ecc33df7 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -22,9 +22,9 @@ struct Rule { }; /// This class is responsible for managing all opened files. -class FileController { +class Scheduler { public: - FileController(CompilationDatabase& database, llvm::ArrayRef rules) : + Scheduler(CompilationDatabase& database, llvm::ArrayRef rules) : database(database), rules(rules) {} async::Task<> open(llvm::StringRef path); @@ -35,7 +35,12 @@ public: private: CompilationDatabase& database; + llvm::ArrayRef rules; + + struct File {}; + + llvm::StringMap files; }; } // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index 6eea47af..99215133 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -1,7 +1,10 @@ #pragma once #include "Async.h" +#include "Config.h" #include "Protocol.h" +#include "Database.h" +#include "Scheduler.h" namespace clice { @@ -147,6 +150,10 @@ private: async::Task<> onContextAll(const proto::TextDocumentIdentifier& params); async::Task<> onContextSwitch(const proto::TextDocumentIdentifier& params); + + SourceConverter converter; + CompilationDatabase database; + Scheduler scheduler; }; } // namespace clice diff --git a/include/Test/Annotation.h b/include/Test/Annotation.h deleted file mode 100644 index 115e456b..00000000 --- a/include/Test/Annotation.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "Basic/Location.h" - -namespace clice { - -class Annotation { -public: - Annotation(llvm::StringRef source) : m_source() { - m_source.reserve(source.size()); - - uint32_t line = 0; - uint32_t column = 0; - - for(uint32_t i = 0; i < source.size();) { - auto c = source[i]; - - if(c == '@') { - i += 1; - auto key = source.substr(i).take_until([](char c) { return c == ' '; }); - assert(!locations.contains(key) && "duplicate key"); - locations.try_emplace(key, line, column); - continue; - } - - if(c == '$') { - assert(i + 1 < source.size() && source[i + 1] == '(' && "expect $(name)"); - i += 2; - auto key = source.substr(i).take_until([](char c) { return c == ')'; }); - i += key.size() + 1; - assert(!locations.contains(key) && "duplicate key"); - locations.try_emplace(key, line, column); - continue; - } - - if(c == '\n') { - line += 1; - column = 0; - } else { - column += 1; - } - - i += 1; - m_source.push_back(c); - } - } - - llvm::StringRef source() const { - return m_source; - } - - proto::Position position(llvm::StringRef key) const { - return locations.lookup(key); - } - -private: - std::string m_source; - llvm::StringMap locations; -}; - -} // namespace clice diff --git a/include/Test/CTest.h b/include/Test/CTest.h index f2260dc6..8ca50809 100644 --- a/include/Test/CTest.h +++ b/include/Test/CTest.h @@ -2,7 +2,7 @@ #include "gtest/gtest.h" #include "Basic/Location.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" #include "Support/Support.h" namespace clice::test { @@ -72,23 +72,28 @@ inline void EXPECT_NE(const LHS& lhs, } } -class Tester { -public: +struct Tester { CompilationParams params; - std::unique_ptr vfs; - ASTInfo info; + std::optional info; /// Annoated locations. - std::vector sources; - llvm::StringMap locations; llvm::StringMap offsets; + llvm::StringMap locations; + std::vector sources; public: + Tester() = default; + Tester(llvm::StringRef file, llvm::StringRef content) { params.srcPath = file; params.content = annoate(content); } + void addMain(llvm::StringRef file, llvm::StringRef content) { + params.srcPath = file; + params.content = annoate(content); + } + void addFile(llvm::StringRef name, llvm::StringRef content) { params.remappedFiles.emplace_back(name, content); } @@ -141,7 +146,6 @@ public: } Tester& run(const char* standard = "-std=c++20") { - params.vfs = std::move(vfs); params.command = std::format("clang++ {} {}", standard, params.srcPath); auto info = compile(params); @@ -150,7 +154,7 @@ public: std::terminate(); } - this->info = std::move(*info); + this->info.emplace(std::move(*info)); return *this; } diff --git a/include/Test/IndexTester.h b/include/Test/IndexTester.h index d024a1a1..2e309410 100644 --- a/include/Test/IndexTester.h +++ b/include/Test/IndexTester.h @@ -11,7 +11,7 @@ struct IndexTester : Tester { IndexTester& run(const char* standard = "-std=c++20") { Tester::run(standard); - indices = index::test(info); + indices = index::test(*info); return *this; } diff --git a/include/Test/Test.h b/include/Test/Test.h index 22c549b4..27190da9 100644 --- a/include/Test/Test.h +++ b/include/Test/Test.h @@ -1,9 +1,134 @@ #pragma once #include "gtest/gtest.h" +#include "Basic/Location.h" +#include "llvm/ADT/StringMap.h" +#include "Support/Support.h" namespace clice::testing { +#undef EXPECT_EQ +#undef EXPECT_NE +inline void EXPECT_FAILURE(std::string msg, + std::source_location current = std::source_location::current()) { + ::testing::internal::AssertHelper(::testing ::TestPartResult ::kNonFatalFailure, + current.file_name(), + current.line(), + msg.c_str()) = ::testing ::Message(); +} -} \ No newline at end of file +template +inline void EXPECT_EQ(const LHS& lhs, + const RHS& rhs, + std::source_location current = std::source_location::current()) { + if(!refl::equal(lhs, rhs)) { + std::string expect; + if constexpr(requires { sizeof(json::Serde::serialize); }) { + llvm::raw_string_ostream(expect) << json::serialize(lhs); + } else { + expect = "cannot dump value"; + } + + std::string actual; + if constexpr(requires { sizeof(json::Serde::serialize); }) { + llvm::raw_string_ostream(actual) << json::serialize(rhs); + } else { + actual = "cannot dump value"; + } + + EXPECT_FAILURE(std::format("expect: {}, actual: {}\n", expect, actual), current); + } +} + +template +inline void EXPECT_NE(const LHS& lhs, + const RHS& rhs, + std::source_location current = std::source_location::current()) { + if(refl::equal(lhs, rhs)) { + std::string expect; + if constexpr(requires { sizeof(json::Serde); }) { + llvm::raw_string_ostream(expect) << json::serialize(lhs); + } else { + expect = "cannot dump value"; + } + + std::string actual; + if constexpr(requires { sizeof(json::Serde); }) { + llvm::raw_string_ostream(actual) << json::serialize(rhs); + } else { + actual = "cannot dump value"; + } + + EXPECT_FAILURE(std::format("expect: {}, actual: {}\n", expect, actual), current); + } +} + +class Annotation { +public: + Annotation(llvm::StringRef source) : m_source() { + m_source.reserve(source.size()); + + uint32_t line = 0; + uint32_t column = 0; + + for(uint32_t i = 0; i < source.size();) { + auto c = source[i]; + + if(c == '@') { + i += 1; + auto key = source.substr(i).take_until([](char c) { return c == ' '; }); + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '$') { + assert(i + 1 < source.size() && source[i + 1] == '(' && "expect $(name)"); + i += 2; + auto key = source.substr(i).take_until([](char c) { return c == ')'; }); + i += key.size() + 1; + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '\n') { + line += 1; + column = 0; + } else { + column += 1; + } + + i += 1; + m_source.push_back(c); + } + } + + Annotation(const Annotation&) = delete; + + Annotation(Annotation&& other) noexcept : + m_source(std::move(other.m_source)), locations(std::move(other.locations)) {} + + Annotation& operator= (const Annotation&) = delete; + + Annotation& operator= (Annotation&& other) noexcept { + m_source = std::move(other.m_source); + locations = std::move(other.locations); + return *this; + } + + llvm::StringRef source() const { + return m_source; + } + + proto::Position pos(llvm::StringRef key) const { + return locations.lookup(key); + } + +private: + std::string m_source; + llvm::StringMap locations; +}; + +} // namespace clice::testing diff --git a/src/Basic/SourceCode.cpp b/src/Basic/SourceCode.cpp new file mode 100644 index 00000000..73a23f3d --- /dev/null +++ b/src/Basic/SourceCode.cpp @@ -0,0 +1,17 @@ +#include "Basic/SourceCode.h" + +namespace clice { + +llvm::StringRef getFileContent(const clang::SourceManager& SM, clang::FileID fid) { + return SM.getBufferData(fid); +} + +std::uint32_t getTokenLength(const clang::SourceManager& SM, clang::SourceLocation location) { + return clang::Lexer::MeasureTokenLength(location, SM, {}); +} + +llvm::StringRef getTokenSpelling(const clang::SourceManager& SM, clang::SourceLocation location) { + return llvm::StringRef(SM.getCharacterData(location), getTokenLength(SM, location)); +} + +} // namespace clice diff --git a/src/Basic/SourceConverter.cpp b/src/Basic/SourceConverter.cpp index 259f7987..664c3eb5 100644 --- a/src/Basic/SourceConverter.cpp +++ b/src/Basic/SourceConverter.cpp @@ -1,6 +1,8 @@ #include "Basic/Location.h" +#include "Basic/SourceCode.h" #include "Basic/SourceConverter.h" #include "Support/Support.h" + #include "clang/Basic/SourceManager.h" namespace clice { @@ -89,34 +91,55 @@ std::size_t SourceConverter::remeasure(llvm::StringRef content) const { std::unreachable(); } -proto::Position SourceConverter::toPosition(llvm::StringRef content, clang::SourceLocation location, - const clang::SourceManager& SM) const { - assert(location.isValid() && location.isFileID() && - "SourceLocation must be valid and not a macro location"); - auto [fileID, offset] = SM.getDecomposedSpellingLoc(location); +proto::Position SourceConverter::toPosition(llvm::StringRef content, std::uint32_t offset) const { + assert(offset <= content.size() && "Offset is out of range"); + proto::Position position = {0, 0}; - /// Line and column in LSP are 0-based but clang's SourceLocation is 1-based. - auto line = SM.getLineNumber(fileID, offset) - 1; - auto column = SM.getColumnNumber(fileID, offset) - 1; + std::uint32_t line = 0; + std::uint32_t column = 0; + for(std::uint32_t i = 0; i < offset; i++) { + auto c = content[i]; + if(c == '\n') { + line += 1; + column = 0; + } else { + column += 1; + } + } - proto::Position position; - /// Line doesn't need to be adjusted. It is encoding-dependent. + /// Line doesn't need to be adjusted. position.line = line; /// Column needs to be adjusted based on the encoding. - if(auto word = content.substr(offset - column, column); !word.empty()) + if(column > 0) { + auto word = content.substr(offset - column, column); position.character = remeasure(word); - else - position.character = column; // word is the last column of that line. + } + return position; } proto::Position SourceConverter::toPosition(clang::SourceLocation location, const clang::SourceManager& SM) const { - bool isInvalid = false; - llvm::StringRef content = SM.getCharacterData(location, &isInvalid); - assert(!isInvalid && "Invalid SourceLocation"); - return toPosition(content, location, SM); + assert(location.isValid() && location.isFileID() && + "SourceLocation must be valid and not a macro location"); + auto [fid, offset] = SM.getDecomposedSpellingLoc(location); + auto content = getFileContent(SM, fid); + return toPosition(content, offset); +} + +proto::Range SourceConverter::toRange(clang::SourceRange range, + const clang::SourceManager& SM) const { + auto [begin, end] = range; + assert(begin.isValid() && end.isValid() && "Invalid SourceRange"); + assert(begin.isFileID() && end.isFileID() && "SourceRange must be FileID"); + + auto [fileID, offset] = SM.getDecomposedSpellingLoc(end); + auto content = getFileContent(SM, fileID); + return proto::Range{ + toPosition(begin, SM), + toPosition(content, offset + getTokenLength(SM, end)), + }; } std::size_t SourceConverter::toOffset(llvm::StringRef content, proto::Position position) const { @@ -224,16 +247,16 @@ std::string SourceConverter::toPath(llvm::StringRef uri) { llvm::StringRef cloned = uri; #if defined(_WIN32) - if (cloned.starts_with("file:///")) { - cloned = cloned.drop_front(8); + if(cloned.starts_with("file:///")) { + cloned = cloned.drop_front(8); } else { - std::terminate(); + std::terminate(); } #elif defined(__unix__) - if (cloned.starts_with("file://")) { - cloned = cloned.drop_front(7); + if(cloned.starts_with("file://")) { + cloned = cloned.drop_front(7); } else { - std::terminate(); + std::terminate(); } #else #error "Unsupported platform" @@ -242,7 +265,7 @@ std::string SourceConverter::toPath(llvm::StringRef uri) { auto decoded = decodePercent(cloned); llvm::SmallString<128> result; - if(auto err = fs::real_path(decoded, result)){ + if(auto err = fs::real_path(decoded, result)) { print("Failed to get real path: {}, Input is {}\n", err.message(), decoded); std::terminate(); } diff --git a/src/Basic/SymbolKind.cpp b/src/Basic/SymbolKind.cpp index cb5e3152..a4d6a996 100644 --- a/src/Basic/SymbolKind.cpp +++ b/src/Basic/SymbolKind.cpp @@ -1,5 +1,5 @@ #include "Basic/SymbolKind.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" namespace clice { diff --git a/src/Compiler/Compilation.cpp b/src/Compiler/Compilation.cpp new file mode 100644 index 00000000..9daac340 --- /dev/null +++ b/src/Compiler/Compilation.cpp @@ -0,0 +1,236 @@ +#include "Compiler/Command.h" +#include "Compiler/Compilation.h" + +#include "clang/Lex/PreprocessorOptions.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" + +namespace clice { + +namespace impl { + +std::unique_ptr createInvocation(CompilationParams& params) { + llvm::SmallString<1024> buffer; + llvm::SmallVector args; + + if(auto error = mangleCommand(params.command, args, buffer)) { + std::terminate(); + } + + clang::CreateInvocationOptions options = {}; + options.VFS = params.vfs; + + auto invocation = clang::createInvocation(args, options); + if(!invocation) { + std::terminate(); + } + + auto& frontOpts = invocation->getFrontendOpts(); + frontOpts.DisableFree = false; + + clang::LangOptions& langOpts = invocation->getLangOpts(); + langOpts.CommentOpts.ParseAllComments = true; + langOpts.RetainCommentsFromSystemHeaders = true; + + return invocation; +} + +std::unique_ptr createInstance(CompilationParams& params) { + auto instance = std::make_unique(); + + instance->setInvocation(createInvocation(params)); + + /// TODO: use a thread safe filesystem and our customized `DiagnosticConsumer`. + instance->createDiagnostics( + *params.vfs, + new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), + true); + + instance->createFileManager(params.vfs); + + /// Add remapped files, if bounds is provided, cut off the content. + std::size_t size = params.bounds.has_value() ? params.bounds.value() : params.content.size(); + + assert(!instance->getPreprocessorOpts().RetainRemappedFileBuffers && + "RetainRemappedFileBuffers should be false"); + if(!params.content.empty()) { + instance->getPreprocessorOpts().addRemappedFile( + params.srcPath, + llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath) + .release()); + } + + for(auto& [file, content]: params.remappedFiles) { + instance->getPreprocessorOpts().addRemappedFile( + file, + llvm::MemoryBuffer::getMemBufferCopy(content, file).release()); + } + + if(!instance->createTarget()) { + /// FIXME: add error handle here. + std::terminate(); + } + + auto& PPOpts = instance->getPreprocessorOpts(); + auto& pch = params.pch; + auto& bounds = params.pchBounds; + + if(bounds.Size != 0) { + PPOpts.UsePredefines = false; + PPOpts.ImplicitPCHInclude = std::move(pch); + PPOpts.PrecompiledPreambleBytes.first = bounds.Size; + PPOpts.PrecompiledPreambleBytes.second = bounds.PreambleEndsAtStartOfLine; + PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; + } + + auto& pcms = params.pcms; + for(auto& [name, path]: pcms) { + auto& HSOpts = instance->getHeaderSearchOpts(); + HSOpts.PrebuiltModuleFiles.try_emplace(name.str(), std::move(path)); + } + + return instance; +} + +} // namespace impl + +namespace { + +/// Execute given action with the on the given instance. `callback` is called after +/// `BeginSourceFile`. Beacuse `BeginSourceFile` may create new preprocessor. +llvm::Error ExecuteAction(clang::CompilerInstance& instance, + clang::FrontendAction& action, + auto&& callback) { + if(!action.BeginSourceFile(instance, instance.getFrontendOpts().Inputs[0])) { + return error("Failed to begin source file"); + } + + callback(); + + if(auto error = action.Execute()) { + return error; + } + + return llvm::Error::success(); +} + +llvm::Expected ExecuteAction(std::unique_ptr instance, + std::unique_ptr action) { + + if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { + return error("Failed to begin source file"); + } + + auto& pp = instance->getPreprocessor(); + // FIXME: clang-tidy, include-fixer, etc? + + // `BeginSourceFile` may create new preprocessor, so all operations related to preprocessor + // should be done after `BeginSourceFile`. + + /// Collect directives. + llvm::DenseMap directives; + Directive::attach(pp, directives); + + /// Collect tokens. + std::optional tokCollector; + + /// It is not necessary to collect tokens if we are running code completion. + /// And in fact will cause assertion failure. + if(!instance->hasCodeCompletionConsumer()) { + tokCollector.emplace(pp); + } + + if(auto error = action->Execute()) { + return clice::error("Failed to execute action, because {} ", error); + } + + std::optional tokBuf; + if(tokCollector) { + tokBuf = std::move(*tokCollector).consume(); + } + + /// FIXME: getDependencies currently return ArrayRef, which actually results in + /// extra copy. It would be great to avoid this copy. + + std::optional resolver; + if(instance->hasSema()) { + resolver.emplace(instance->getSema()); + } + + return ASTInfo(pp.getSourceManager().getMainFileID(), + std::move(action), + std::move(instance), + std::move(resolver), + std::move(tokBuf), + std::move(directives)); +} + +} // namespace + +llvm::Expected compile(CompilationParams& params) { + auto instance = impl::createInstance(params); + + return ExecuteAction(std::move(instance), std::make_unique()); +} + +llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer) { + auto instance = impl::createInstance(params); + + /// Set options to run code completion. + instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str(); + instance->getFrontendOpts().CodeCompletionAt.Line = params.line; + instance->getFrontendOpts().CodeCompletionAt.Column = params.column; + instance->setCodeCompletionConsumer(consumer); + + return ExecuteAction(std::move(instance), std::make_unique()); +} + +llvm::Expected compile(CompilationParams& params, PCHInfo& out) { + assert(params.bounds.has_value() && "Preamble bounds is required to build PCH"); + + auto instance = impl::createInstance(params); + + /// Set options to generate PCH. + instance->getFrontendOpts().OutputFile = params.outPath.str(); + instance->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH; + instance->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false}; + instance->getPreprocessorOpts().GeneratePreamble = true; + instance->getLangOpts().CompilingPCH = true; + + if(auto info = + ExecuteAction(std::move(instance), std::make_unique())) { + auto& bounds = *params.bounds; + out.preamble = params.content.substr(0, bounds).str(); + out.path = params.outPath.str(); + /// TODO: collect files involved in building this PCH. + return std::move(*info); + } else { + return info.takeError(); + } +} + +llvm::Expected compile(CompilationParams& params, PCMInfo& out) { + auto instance = impl::createInstance(params); + + /// Set options to generate PCM. + instance->getFrontendOpts().OutputFile = params.outPath.str(); + instance->getFrontendOpts().ProgramAction = clang::frontend::GenerateReducedModuleInterface; + + ; + if(auto info = ExecuteAction(std::move(instance), + std::make_unique())) { + assert(info->pp().isInNamedInterfaceUnit() && + "Only module interface unit could be built as PCM"); + out.isInterfaceUnit = true; + out.name = info->pp().getNamedModuleName(); + for(auto& [name, path]: params.pcms) { + out.mods.emplace_back(name); + } + out.path = params.outPath.str(); + out.srcPath = params.srcPath.str(); + return std::move(*info); + } else { + return info.takeError(); + } +} + +} // namespace clice diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp deleted file mode 100644 index 096f2f95..00000000 --- a/src/Compiler/Compiler.cpp +++ /dev/null @@ -1,412 +0,0 @@ -#include -#include - -#include -#include - -namespace clice { - -namespace { - -auto createInvocation(CompilationParams& params) { - llvm::SmallString<1024> buffer; - llvm::SmallVector args; - - if(auto error = mangleCommand(params.command, args, buffer)) { - std::terminate(); - } - - clang::CreateInvocationOptions options = {}; - options.VFS = params.vfs; - - auto invocation = clang::createInvocation(args, options); - if(!invocation) { - std::terminate(); - } - - auto& frontOpts = invocation->getFrontendOpts(); - frontOpts.DisableFree = false; - - clang::LangOptions& langOpts = invocation->getLangOpts(); - langOpts.CommentOpts.ParseAllComments = true; - langOpts.RetainCommentsFromSystemHeaders = true; - - return invocation; -} - -auto createInstance(CompilationParams& params) { - auto instance = std::make_unique(); - - instance->setInvocation(createInvocation(params)); - - /// TODO: use a thread safe filesystem and our customized `DiagnosticConsumer`. - instance->createDiagnostics( - *params.vfs, - new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), - true); - - instance->createFileManager(params.vfs); - - /// Add remapped files, if bounds is provided, cut off the content. - std::size_t size = - params.bounds.has_value() ? params.bounds.value().Size : params.content.size(); - - assert(!instance->getPreprocessorOpts().RetainRemappedFileBuffers && - "RetainRemappedFileBuffers should be false"); - if(!params.content.empty()) { - instance->getPreprocessorOpts().addRemappedFile( - params.srcPath, - llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, size), params.srcPath) - .release()); - } - - for(auto& [file, content]: params.remappedFiles) { - instance->getPreprocessorOpts().addRemappedFile( - file, - llvm::MemoryBuffer::getMemBufferCopy(content, file).release()); - } - - if(!instance->createTarget()) { - /// FIXME: add error handle here. - std::terminate(); - } - - return instance; -} - -void applyPreamble(clang::CompilerInstance& instance, CompilationParams& params) { - auto& PPOpts = instance.getPreprocessorOpts(); - auto& pch = params.pch; - auto& bounds = params.pchBounds; - - if(bounds.Size != 0) { - PPOpts.UsePredefines = false; - PPOpts.ImplicitPCHInclude = std::move(pch); - PPOpts.PrecompiledPreambleBytes.first = bounds.Size; - PPOpts.PrecompiledPreambleBytes.second = bounds.PreambleEndsAtStartOfLine; - PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; - } - - auto& pcms = params.pcms; - for(auto& [name, path]: pcms) { - auto& HSOpts = instance.getHeaderSearchOpts(); - HSOpts.PrebuiltModuleFiles.try_emplace(name.str(), std::move(path)); - } -} - -/// Execute given action with the on the given instance. `callback` is called after -/// `BeginSourceFile`. Beacuse `BeginSourceFile` may create new preprocessor. -llvm::Error ExecuteAction(clang::CompilerInstance& instance, - clang::FrontendAction& action, - auto&& callback) { - if(!action.BeginSourceFile(instance, instance.getFrontendOpts().Inputs[0])) { - return error("Failed to begin source file"); - } - - callback(); - - if(auto error = action.Execute()) { - return error; - } - - return llvm::Error::success(); -} - -llvm::Expected ExecuteAction(std::unique_ptr instance, - std::unique_ptr action) { - - if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { - return error("Failed to begin source file"); - } - - auto& pp = instance->getPreprocessor(); - // FIXME: clang-tidy, include-fixer, etc? - - // `BeginSourceFile` may create new preprocessor, so all operations related to preprocessor - // should be done after `BeginSourceFile`. - - /// Collect directives. - llvm::DenseMap directives; - Directive::attach(pp, directives); - - /// Collect tokens. - std::optional tokCollector; - - /// It is not necessary to collect tokens if we are running code completion. - /// And in fact will cause assertion failure. - if(!instance->hasCodeCompletionConsumer()) { - tokCollector.emplace(pp); - } - - if(auto error = action->Execute()) { - return clice::error("Failed to execute action, because {} ", error); - } - - std::unique_ptr tokBuf; - if(tokCollector) { - tokBuf = std::make_unique(std::move(*tokCollector).consume()); - } - - /// FIXME: getDependencies currently return ArrayRef, which actually results in - /// extra copy. It would be great to avoid this copy. - - return ASTInfo(std::move(action), - std::move(instance), - std::move(tokBuf), - std::move(directives), - {}); -} - -} // namespace - -void CompilationParams::computeBounds(llvm::StringRef header) { - assert(!bounds.has_value() && "Bounds is already computed"); - assert(!content.empty() && "Source content is required to compute bounds"); - - if(header.empty()) { - auto invocation = createInvocation(*this); - bounds = clang::Lexer::ComputePreamble(content, invocation->getLangOpts()); - return; - } - - auto instance = createInstance(*this); - - instance->getFrontendOpts().ProgramAction = clang::frontend::RunPreprocessorOnly; - - struct SearchBoundary : public clang::PPCallbacks { - llvm::StringRef header; - clang::SourceLocation& hashLoc; - - SearchBoundary(llvm::StringRef header, clang::SourceLocation& hashLoc) : - header(header), hashLoc(hashLoc) {} - - void InclusionDirective(clang::SourceLocation hashLoc, - const clang::Token& includeTok, - llvm::StringRef filename, - bool isAngled, - clang::CharSourceRange filenameRange, - clang::OptionalFileEntryRef file, - llvm::StringRef searchPath, - llvm::StringRef relativePath, - const clang::Module* suggestedModule, - bool moduleImported, - clang::SrcMgr::CharacteristicKind fileType) override { - llvm::SmallString<128> path; - if(searchPath != ".") { - path::append(path, searchPath); - } - path::append(path, relativePath); - if(path == header) { - this->hashLoc = hashLoc; - } - } - }; - - /// The hash location of the include directive that includes the header. - clang::SourceLocation hashLoc; - - clang::PreprocessOnlyAction action; - - /// FIXME: merge the logic to `ExecuteAction`. - if(!instance->createTarget()) { - llvm::errs() << "Failed to create target\n"; - std::terminate(); - } - - if(!action.BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { - llvm::errs() << "Failed to begin source file\n"; - std::terminate(); - } - - instance->getPreprocessor().addPPCallbacks(std::make_unique(header, hashLoc)); - - if(auto error = action.Execute()) { - llvm::errs() << "Failed to execute action, because " << error << "\n"; - std::terminate(); - } - - if(hashLoc.isInvalid()) { - llvm::errs() << "Failed to find the boundary\n"; - std::terminate(); - } - - /// Find the top level file. - auto& srcMgr = instance->getSourceManager(); - while(srcMgr.getIncludeLoc(srcMgr.getFileID(hashLoc)).isValid()) { - hashLoc = srcMgr.getIncludeLoc(srcMgr.getFileID(hashLoc)); - } - auto offset = srcMgr.getFileOffset(hashLoc); - - action.EndSourceFile(); - - /// We need to move to next line to get the correct bounds. - for(auto i = offset; i < content.size(); ++i) { - if(content[i] == '\n') { - bounds = {i + 2, true}; - break; - } - } - - if(!bounds.has_value()) { - llvm::errs() << "Failed to compute bounds\n"; - std::terminate(); - } -} - -/// Scan the module name. This will not run the preprocessor. -std::string scanModuleName(llvm::StringRef content) { - clang::LangOptions langOpts; - langOpts.Modules = true; - langOpts.CPlusPlus20 = true; - clang::Lexer lexer(clang::SourceLocation(), - langOpts, - content.begin(), - content.begin(), - content.end()); - - clang::Token token; - lexer.Lex(token); - - while(!token.is(clang::tok::eof)) { - llvm::outs() << token.getName() << "\n"; - if(token.is(clang::tok::kw_module)) { - lexer.Lex(token); - - if(token.is(clang::tok::coloncolon)) { - lexer.Lex(token); - } - - if(token.is(clang::tok::identifier)) { - return token.getIdentifierInfo()->getName().str(); - } - } - - lexer.Lex(token); - } - - return ""; -} - -llvm::Expected scanModule(CompilationParams& params) { - struct ModuleCollector : public clang::PPCallbacks { - ModuleInfo& info; - - ModuleCollector(ModuleInfo& info) : info(info) {} - - void moduleImport(clang::SourceLocation importLoc, - clang::ModuleIdPath path, - const clang::Module* imported) override { - assert(path.size() == 1); - info.mods.emplace_back(path[0].first->getName()); - } - }; - - ModuleInfo info; - clang::PreprocessOnlyAction action; - auto instance = createInstance(params); - - if(!action.BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { - return error("Failed to begin source file"); - } - - auto& pp = instance->getPreprocessor(); - - pp.addPPCallbacks(std::make_unique(info)); - - if(auto error = action.Execute()) { - return error; - } - - if(pp.isInNamedModule()) { - info.isInterfaceUnit = pp.isInNamedInterfaceUnit(); - info.name = pp.getNamedModuleName(); - } - - return info; -} - -llvm::Expected compile(CompilationParams& params) { - auto instance = createInstance(params); - - applyPreamble(*instance, params); - - return ExecuteAction(std::move(instance), std::make_unique()); -} - -llvm::Expected compile(CompilationParams& params, clang::CodeCompleteConsumer* consumer) { - auto instance = createInstance(params); - - /// Set options to run code completion. - instance->getFrontendOpts().CodeCompletionAt.FileName = params.srcPath.str(); - instance->getFrontendOpts().CodeCompletionAt.Line = params.line; - instance->getFrontendOpts().CodeCompletionAt.Column = params.column; - instance->setCodeCompletionConsumer(consumer); - - applyPreamble(*instance, params); - - return ExecuteAction(std::move(instance), std::make_unique()); -} - -llvm::Expected compile(CompilationParams& params, PCHInfo& out) { - assert(params.bounds.has_value() && "Preamble bounds is required to build PCH"); - - auto instance = createInstance(params); - - /// Set options to generate PCH. - instance->getFrontendOpts().OutputFile = params.outPath.str(); - instance->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH; - instance->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false}; - instance->getPreprocessorOpts().GeneratePreamble = true; - instance->getLangOpts().CompilingPCH = true; - - auto info = ExecuteAction(std::move(instance), std::make_unique()); - if(!info) { - return info.takeError(); - } - - out.path = params.outPath.str(); - - auto& bounds = *params.bounds; - out.preamble = params.content.substr(0, bounds.Size).str(); - out.deps = info->deps(); - if(bounds.PreambleEndsAtStartOfLine) { - out.preamble.append("@"); - } - - /// TODO: collect files involved in building this PCH. - - return std::move(*info); -} - -llvm::Expected compile(CompilationParams& params, PCMInfo& out) { - auto instance = createInstance(params); - - /// Set options to generate PCM. - instance->getFrontendOpts().OutputFile = params.outPath.str(); - instance->getFrontendOpts().ProgramAction = clang::frontend::GenerateReducedModuleInterface; - - applyPreamble(*instance, params); - - auto info = ExecuteAction(std::move(instance), - std::make_unique()); - if(!info) { - return info.takeError(); - } - - assert(info->pp().isInNamedInterfaceUnit() && - "Only module interface unit could be built as PCM"); - - out.isInterfaceUnit = true; - out.name = info->pp().getNamedModuleName(); - for(auto& [name, path]: params.pcms) { - out.mods.emplace_back(name); - } - - out.path = params.outPath.str(); - out.srcPath = params.srcPath.str(); - out.deps = info->deps(); - - return std::move(*info); -} - -} // namespace clice diff --git a/src/Compiler/Directive.cpp b/src/Compiler/Directive.cpp index cc37496a..a116d74d 100644 --- a/src/Compiler/Directive.cpp +++ b/src/Compiler/Directive.cpp @@ -7,22 +7,21 @@ namespace clice { namespace { -class PPCallback : public clang::PPCallbacks { -public: - PPCallback(clang::Preprocessor& pp, llvm::DenseMap& directives) : - pp(pp), directives(directives) {} +struct PPCallback : public clang::PPCallbacks { + clang::FileID prevFID; + clang::Preprocessor& PP; + clang::SourceManager& SM; + llvm::DenseMap& directives; + llvm::DenseMap macroCache; -private: - void addInclude(llvm::StringRef path, clang::SourceLocation loc, clang::SourceRange range) { - auto& directive = directives[pp.getSourceManager().getFileID(loc)]; - directive.includes.emplace_back(Include{path, loc, range}); - } + PPCallback(clang::Preprocessor& PP, llvm::DenseMap& directives) : + PP(PP), SM(PP.getSourceManager()), directives(directives) {} void addCondition(clang::SourceLocation loc, Condition::BranchKind kind, Condition::ConditionValue value, clang::SourceRange conditionRange) { - auto& directive = directives[pp.getSourceManager().getFileID(loc)]; + auto& directive = directives[PP.getSourceManager().getFileID(loc)]; directive.conditions.emplace_back(Condition{kind, value, loc, conditionRange}); } @@ -61,33 +60,57 @@ private: } void addMacro(const clang::MacroInfo* def, MacroRef::Kind kind, clang::SourceLocation loc) { - if(pp.getSourceManager().isWrittenInBuiltinFile(loc) || - pp.getSourceManager().isWrittenInCommandLineFile(loc)) { + if(PP.getSourceManager().isWrittenInBuiltinFile(loc) || + PP.getSourceManager().isWrittenInCommandLineFile(loc)) { return; } - auto& directive = directives[pp.getSourceManager().getFileID(loc)]; + auto& directive = directives[PP.getSourceManager().getFileID(loc)]; directive.macros.emplace_back(MacroRef{def, kind, loc}); } -private: - void FileChanged(clang::SourceLocation loc, - clang::PPCallbacks::FileChangeReason reason, - clang::SrcMgr::CharacteristicKind fileType, - clang::FileID) override {} + /// ============================================================================ + /// Rewritten Preprocessor Callbacks + /// ============================================================================ + + void LexedFileChanged(clang::FileID currFID, + LexedFileChangeReason reason, + clang::SrcMgr::CharacteristicKind, + clang::FileID prevFID, + clang::SourceLocation) override { + if(reason == LexedFileChangeReason::EnterFile && currFID.isValid() && prevFID.isValid() && + this->prevFID.isValid() && prevFID == this->prevFID) { + directives[prevFID].includes.back().fid = currFID; + } + } void InclusionDirective(clang::SourceLocation hashLoc, const clang::Token& includeTok, - llvm::StringRef filename, - bool isAngled, - clang::CharSourceRange filenameRange, - clang::OptionalFileEntryRef file, - llvm::StringRef searchPath, - llvm::StringRef relativePath, - const clang::Module* suggestedModule, - bool moduleImported, - clang::SrcMgr::CharacteristicKind fileType) override { - addInclude(filename, includeTok.getLocation(), filenameRange.getAsRange()); + llvm::StringRef, + bool, + clang::CharSourceRange, + clang::OptionalFileEntryRef, + llvm::StringRef, + llvm::StringRef, + const clang::Module*, + bool, + clang::SrcMgr::CharacteristicKind) override { + prevFID = SM.getFileID(hashLoc); + directives[prevFID].includes.emplace_back(Include{ + {}, + includeTok.getLocation(), + }); + } + + void HasInclude(clang::SourceLocation location, + llvm::StringRef, + bool, + clang::OptionalFileEntryRef file, + clang::SrcMgr::CharacteristicKind) override { + directives[SM.getFileID(location)].hasIncludes.emplace_back(clice::HasInclude{ + file ? file->getName() : "", + location, + }); } void PragmaDirective(clang::SourceLocation Loc, @@ -178,11 +201,6 @@ private: addMacro(def, MacroRef::Undef, name.getLocation()); } } - -private: - clang::Preprocessor& pp; - llvm::DenseMap& directives; - llvm::DenseMap macroCache; }; } // namespace diff --git a/src/Compiler/Module.cpp b/src/Compiler/Module.cpp index 921070eb..67e6e3ca 100644 --- a/src/Compiler/Module.cpp +++ b/src/Compiler/Module.cpp @@ -1,5 +1,5 @@ #include "Compiler/Module.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" namespace clice { @@ -27,9 +27,17 @@ std::string scanModuleName(CompilationParams& params) { std::string name; clang::Token token; - while(!lexer.LexFromRawLexer(token)) { - auto kind = token.getKind(); - if(kind == clang::tok::hash) { + while(true) { + lexer.LexFromRawLexer(token); + if(token.is(clang::tok::eof)) { + break; + } + + if(!token.isAtStartOfLine()) { + continue; + } + + if(token.is(clang::tok::hash)) { lexer.LexFromRawLexer(token); auto diretive = token.getRawIdentifier(); if(diretive == "if" || diretive == "ifdef" || diretive == "ifndef") { @@ -37,7 +45,7 @@ std::string scanModuleName(CompilationParams& params) { } else if(diretive == "endif") { isInDirective = false; } - } else if(kind == clang::tok::raw_identifier) { + } else if(token.is(clang::tok::raw_identifier)) { if(token.getRawIdentifier() != "export") [[likely]] { continue; } @@ -57,7 +65,7 @@ std::string scanModuleName(CompilationParams& params) { /// Otherwise, we can determine the module name directly. while(!lexer.LexFromRawLexer(token)) { - kind = token.getKind(); + auto kind = token.getKind(); if(kind == clang::tok::raw_identifier) { name += token.getRawIdentifier(); } else if(kind == clang::tok::colon) { @@ -88,4 +96,42 @@ std::string scanModuleName(CompilationParams& params) { return info->name; } +llvm::Expected scanModule(CompilationParams& params) { + struct ModuleCollector : public clang::PPCallbacks { + ModuleInfo& info; + + ModuleCollector(ModuleInfo& info) : info(info) {} + + void moduleImport(clang::SourceLocation importLoc, + clang::ModuleIdPath path, + const clang::Module* imported) override { + assert(path.size() == 1); + info.mods.emplace_back(path[0].first->getName()); + } + }; + + ModuleInfo info; + clang::PreprocessOnlyAction action; + auto instance = impl::createInstance(params); + + if(!action.BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { + return error("Failed to begin source file"); + } + + auto& pp = instance->getPreprocessor(); + + pp.addPPCallbacks(std::make_unique(info)); + + if(auto error = action.Execute()) { + return error; + } + + if(pp.isInNamedModule()) { + info.isInterfaceUnit = pp.isInNamedInterfaceUnit(); + info.name = pp.getNamedModuleName(); + } + + return info; +} + } // namespace clice diff --git a/src/Compiler/Preamble.cpp b/src/Compiler/Preamble.cpp index 9830746d..0850004a 100644 --- a/src/Compiler/Preamble.cpp +++ b/src/Compiler/Preamble.cpp @@ -1,3 +1,65 @@ #include "Compiler/Preamble.h" -namespace clice {} +namespace clice { + +std::uint32_t computePreambleBound(llvm::StringRef content) { + clang::LangOptions langOpts; + langOpts.CPlusPlus = true; + langOpts.CPlusPlus26 = true; + + // Create a lexer starting at the beginning of the file. Note that we use a + // "fake" file source location at offset 1 so that the lexer will track our + // position within the file. + auto beginLoc = clang::SourceLocation::getFromRawEncoding(1); + clang::Lexer lexer(beginLoc, langOpts, content.begin(), content.begin(), content.end()); + + bool isInDirective = false; + + clang::Token token; + clang::Token end; + + while(true) { + lexer.LexFromRawLexer(token); + if(token.is(clang::tok::eof)) { + break; + } + + if(isInDirective) { + /// If we are in a directive, we should skip the rest of the line. + if(!token.isAtStartOfLine()) { + end = token; + continue; + } else { + isInDirective = false; + } + } + + if(token.isAtStartOfLine() && token.is(clang::tok::hash)) { + /// If we encounter a `#` at the start of a line, it must be a directive. + isInDirective = true; + continue; + } else if(token.isAtStartOfLine() && token.is(clang::tok::raw_identifier) && + token.getRawIdentifier() == "module") { + if(!lexer.LexFromRawLexer(token) && token.is(clang::tok::semi)) { + /// If we encounter a `module` followed by a `;`, it must be + /// a global module fragment. It should be a part of the preamble. + end = token; + continue; + } + } + + break; + } + + if(auto endLocation = end.getLocation(); endLocation.isValid()) { + return endLocation.getRawEncoding() - beginLoc.getRawEncoding() + end.getLength(); + } else { + return 0; + } +} + +std::uint32_t computeBounds(CompilationParams& params) { + return 0; +} + +} // namespace clice diff --git a/src/Driver/clice.cc b/src/Driver/clice.cc index 846b614a..040098e3 100644 --- a/src/Driver/clice.cc +++ b/src/Driver/clice.cc @@ -27,7 +27,7 @@ int main(int argc, const char** argv) { if(cl::config.empty()) { log::warn("No config file specified; using default configuration."); } else { - /// config::load(argv[0], cl::config.getValue()); + config::load(argv[0], cl::config.getValue()); log::info("Successfully loaded configuration file from {0}.", cl::config.getValue()); } diff --git a/src/Feature/DocumentSymbol.cpp b/src/Feature/DocumentSymbol.cpp index ae01d856..507ddd49 100644 --- a/src/Feature/DocumentSymbol.cpp +++ b/src/Feature/DocumentSymbol.cpp @@ -1,6 +1,6 @@ #include "Basic/SourceConverter.h" #include "Feature/DocumentSymbol.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" namespace clice { diff --git a/src/Feature/FoldingRange.cpp b/src/Feature/FoldingRange.cpp index 4fb1ef1a..3ac6d21d 100644 --- a/src/Feature/FoldingRange.cpp +++ b/src/Feature/FoldingRange.cpp @@ -1,5 +1,5 @@ #include "Feature/FoldingRange.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" /// Clangd's FoldingRange Implementation: /// https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/clangd/SemanticSelection.cpp diff --git a/src/Feature/InlayHint.cpp b/src/Feature/InlayHint.cpp index 959c4e68..a6c337cb 100644 --- a/src/Feature/InlayHint.cpp +++ b/src/Feature/InlayHint.cpp @@ -1,5 +1,5 @@ #include "Basic/SourceConverter.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" #include "Feature/InlayHint.h" namespace clice { diff --git a/src/Feature/SignatureHelp.cpp b/src/Feature/SignatureHelp.cpp index 1380fcdb..6a51595f 100644 --- a/src/Feature/SignatureHelp.cpp +++ b/src/Feature/SignatureHelp.cpp @@ -1,5 +1,5 @@ -#include -#include +#include "Compiler/Compilation.h" +#include "Feature/SignatureHelp.h" namespace clice::feature { diff --git a/src/Server/Config.cpp b/src/Server/Config.cpp new file mode 100644 index 00000000..003819bc --- /dev/null +++ b/src/Server/Config.cpp @@ -0,0 +1,144 @@ +#define TOML_EXCEPTIONS 0 +#include + +#include +#include +#include + +namespace clice::config { + +static llvm::StringMap predefined = { + /// the directory of the workplace. + {"workplace", "" }, + /// the directory of the executable. + {"binary", "" }, + /// the version of the clice. + {"version", "0.0.1"}, + /// the version of dependent llvm. + {"llvm_version", "20" }, +}; + +/// predefined variables. +llvm::StringRef version = predefined["version"]; +llvm::StringRef binary = predefined["binary"]; +llvm::StringRef llvm_version = predefined["llvm_version"]; +llvm::StringRef workspace = predefined["workplace"]; + +struct Config { + ServerOptions server; + CacheOptions cache; + IndexOptions index; + std::vector rules; +}; + +/// global config instance. +static Config config = {}; + +const ServerOptions& server = config.server; +const CacheOptions& cache = config.cache; +const IndexOptions& index = config.index; +llvm::ArrayRef rules = config.rules; + +template +static void parse(Object& object, auto&& value) { + if constexpr(std::is_same_v) { + if(auto v = value.as_boolean()) { + object = v->get(); + } + } else if constexpr(clice::integral) { + if(auto v = value.as_integer()) { + object = v->get(); + } + } else if constexpr(std::is_same_v) { + if(auto v = value.as_string()) { + object = v->get(); + } + } else if constexpr(clice::is_specialization_of) { + if(auto v = value.as_array()) { + for(auto& item: *v) { + object.emplace_back(); + parse(object.back(), item); + } + } + } else if constexpr(refl::reflectable_struct) { + if(auto table = value.as_table()) { + refl::foreach(object, [&](std::string_view key, auto& member) { + if(auto v = (*table)[key]) { + parse(member, v); + } + }); + } + } else { + static_assert(dependent_false, "Unsupported type"); + } +} + +void load(llvm::StringRef execute, llvm::StringRef filename) { + predefined["version"] = "0.0.1"; + predefined["binary"] = execute; + predefined["llvm_version"] = "20"; + + auto toml = toml::parse_file(filename); + if(toml.failed()) { + log::fatal("Failed to parse config file: {0}. Because: {1}", + filename, + toml.error().description()); + } + + parse(config, toml.table()); +} + +/// replace all predefined variables in the text. +static void resolve(std::string& input) { + std::string_view text = input; + llvm::SmallString<128> path; + std::size_t pos = 0; + while((pos = text.find("${", pos)) != std::string::npos) { + path.append(text.substr(0, pos)); + + auto end = text.find('}', pos + 2); + if(end == std::string::npos) { + path.append(text.substr(pos)); + break; + } + + auto variable = text.substr(pos + 2, end - (pos + 2)); + + if(auto iter = predefined.find(variable); iter != predefined.end()) { + path.append(iter->second); + } else { + path.append(text.substr(pos, end - pos + 1)); + } + + text.remove_prefix(end + 1); + pos = 0; + } + + path.append(text); + path::remove_dots(path, true); + input = path.str(); +} + +template +static void replace(Object& object) { + if constexpr(std::is_same_v) { + resolve(object); + } else if constexpr(clice::is_specialization_of) { + for(auto& item: object) { + replace(item); + } + } else if constexpr(refl::reflectable_struct) { + refl::foreach(object, [&](auto, auto& member) { replace(member); }); + } +} + +void init(std::string_view workplace) { + predefined["workspace"] = workplace; + + replace(config); + + log::info("Config initialized successfully: {0}", json::serialize(config)); + return; +} + +} // namespace clice::config diff --git a/src/Server/Database.cpp b/src/Server/Database.cpp index 4f541540..1e88978e 100644 --- a/src/Server/Database.cpp +++ b/src/Server/Database.cpp @@ -1,7 +1,7 @@ #include "Server/Logger.h" #include "Server/Database.h" #include "Support/Support.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" namespace clice { diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index ee9daa1c..085e4b19 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -8,7 +8,37 @@ async::Task<> Server::onInitialize(json::Value id, const proto::InitializeParams result.serverInfo.name = "clice"; result.serverInfo.version = "0.0.1"; + /// Set `SemanticTokensOptions` + result.capabilities.semanticTokensProvider.legend.tokenTypes = { + "keyword", "class", "interface", "enum", "struct", "type", "parameter", + "variable", "property", "enumMember", "event", "function", "method", "macro", + "keyword", "modifier", "comment", "string", "number", "regexp", "operator", + }; + + result.capabilities.semanticTokensProvider.legend.tokenModifiers = { + "declaration", + "definition", + "readonly", + "static", + "deprecated", + "abstract", + "async", + "modification", + "documentation", + "defaultLibrary", + "local", + }; + co_await response(std::move(id), json::serialize(result)); + + auto workplace = SourceConverter::toPath(params.workspaceFolders[0].uri); + config::init(workplace); + + for(auto& dir: config::server.compile_commands_dirs) { + llvm::SmallString<128> path = {dir}; + path::append(path, "compile_commands.json"); + database.update(path); + } } async::Task<> Server::onInitialized(const proto::InitializedParams& params) { diff --git a/src/Server/File.cpp b/src/Server/Scheduler.cpp similarity index 93% rename from src/Server/File.cpp rename to src/Server/Scheduler.cpp index 25ab55dc..1e840dd5 100644 --- a/src/Server/File.cpp +++ b/src/Server/Scheduler.cpp @@ -2,7 +2,7 @@ namespace clice { -async::Task<> FileController::open(llvm::StringRef file) { +async::Task<> Scheduler::open(llvm::StringRef file) { /// 首先根据 rule 来对 file 进行判别 /// 查看一下这 file 属于什么模式 diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index ed10887e..12050a66 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -3,7 +3,7 @@ namespace clice { -Server::Server() { +Server::Server() : scheduler(database, {}) { addMethod("initialize", &Server::onInitialize); addMethod("initialized", &Server::onInitialized); addMethod("shutdown", &Server::onShutdown); diff --git a/unittests/Basic/SourceConverter.cpp b/unittests/Basic/SourceConverter.cpp index d09dcd0c..231ea18c 100644 --- a/unittests/Basic/SourceConverter.cpp +++ b/unittests/Basic/SourceConverter.cpp @@ -30,8 +30,8 @@ TEST(SourceConverter, Position) { Tester txs("main.cpp", main); txs.run("-std=c++11"); - auto& src = txs.info.srcMgr(); - auto& tks = txs.info.tokBuf(); + auto& src = txs.info->srcMgr(); + auto& tks = txs.info->tokBuf(); auto mainid = src.getMainFileID(); auto tokens = @@ -50,14 +50,14 @@ TEST(SourceConverter, Position) { SourceConverter cvtr{proto::PositionEncodingKind::UTF16}; auto pos = cvtr.toPosition(eof, src); EXPECT_EQ(pos.line, 0); - EXPECT_EQ(pos.character, 19); + EXPECT_EQ(pos.character, 17); } { SourceConverter cvtr{proto::PositionEncodingKind::UTF32}; auto pos = cvtr.toPosition(eof, src); EXPECT_EQ(pos.line, 0); - EXPECT_EQ(pos.character, 19); + EXPECT_EQ(pos.character, 16); } } diff --git a/unittests/Compiler/Compiler.cpp b/unittests/Compiler/Compiler.cpp index 163207c5..66667c65 100644 --- a/unittests/Compiler/Compiler.cpp +++ b/unittests/Compiler/Compiler.cpp @@ -1,181 +1,16 @@ #include "Test/Test.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" #include "Support/FileSystem.h" namespace clice::testing { namespace { -TEST(Compiler, buildAST) { - const char* code = R"cpp( -#include +TEST(Compiler, buildAST) {} -int main(){ - printf("Hello world"); - return 0; -} -)cpp"; +TEST(Compiler, buildPCM) {} - CompilationParams params; - params.content = code; - params.srcPath = "main.cpp"; - params.command = "clang++ -std=c++20 main.cpp"; - - auto info = compile(params); - ASSERT_TRUE(bool(info)); -} - -TEST(Compiler, ComputeBounds) { - const char* code = R"cpp( -#include -int main(){ - printf("Hello world"); - return 0; -})cpp"; - - /// Test in no header file. - CompilationParams params; - params.content = code; - params.srcPath = "main.cpp"; - params.command = "clang++ -std=c++20 main.cpp"; - params.computeBounds(); - - ASSERT_TRUE(params.bounds.has_value()); - ASSERT_EQ(params.bounds->Size, 19); - - params.bounds.reset(); - - std::unique_ptr vfs(new vfs::InMemoryFileSystem); - const char* header = R"cpp( -#include "target.h" -)cpp"; - - vfs->addFile("header.h", 0, llvm::MemoryBuffer::getMemBuffer(header)); - vfs->addFile("header2.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("target.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - - code = R"cpp( -#include "header2.h" -#include "header.h" -int main(){ - return 0; -})cpp"; - - params.srcPath = "main.cpp"; - params.content = code; - params.vfs = std::move(vfs); - params.computeBounds("target.h"); - - ASSERT_TRUE(params.bounds.has_value()); - ASSERT_EQ(params.bounds->Size, 43); -} - -TEST(Compiler, buildPCH) { - const char* code = R"cpp( -#include - -int main(){ - printf("Hello world"); - return 0; -} -)cpp"; - - llvm::SmallString<128> outpath; - if(auto error = llvm::sys::fs::createTemporaryFile("main", "pch", outpath)) { - llvm::errs() << error.message() << "\n"; - return; - } - - if(auto error = fs::remove(outpath)) { - llvm::errs() << error.message() << "\n"; - return; - } - - CompilationParams params; - params.content = code; - params.srcPath = "main.cpp"; - params.outPath = outpath; - params.command = "clang++ -std=c++20 main.cpp"; - params.computeBounds(); - - PCHInfo pch; - ASSERT_TRUE(bool(clice::compile(params, pch))); - - params.bounds.reset(); - params.addPCH(pch); - - auto ast = compile(params); - ASSERT_TRUE(bool(ast)); -} - -TEST(Compiler, buildPCM) { - const char* code = R"cpp( -export module A; - -export int foo() { - return 0; -} -)cpp"; - - llvm::SmallString<128> outpath; - if(auto error = llvm::sys::fs::createTemporaryFile("main", "pcm", outpath)) { - llvm::errs() << error.message() << "\n"; - return; - } - - if(auto error = fs::remove(outpath)) { - llvm::errs() << error.message() << "\n"; - return; - } - - CompilationParams params; - params.srcPath = "main.cppm"; - params.content = code; - params.outPath = outpath; - params.command = "clang++ -std=c++20 main.cppm"; - - PCMInfo pcm; - ASSERT_TRUE(bool(clice::compile(params, pcm))); - ASSERT_EQ(pcm.name, "A"); - - const char* code2 = R"cpp( -import A; - -int main(){ - foo(); - return 0; -} -)cpp"; - - params.srcPath = "main.cpp"; - params.content = code2; - params.command = "clang++ -std=c++20 main.cpp"; - params.addPCM(pcm); - - auto info = compile(params); - ASSERT_TRUE(bool(info)); -} - -TEST(Compiler, codeCompleteAt) { - const char* code = R"cpp( -export module A; -export int foo = 1; -)cpp"; - - CompilationParams params; - params.content = code; - params.srcPath = "main.cppm"; - params.command = "clang++ -std=c++20 main.cppm"; - params.file = "main.cppm"; - params.line = 3; - params.column = 10; - - auto consumer = new clang::PrintingCodeCompleteConsumer({}, llvm::outs()); - auto info = compile(params, consumer); - ASSERT_TRUE(bool(info)); - - /// TODO: add tests in the case of PCH, PCM and override file. -} +TEST(Compiler, codeCompleteAt) {} } // namespace diff --git a/unittests/Compiler/Directive.cpp b/unittests/Compiler/Directive.cpp index 05f44196..912293e6 100644 --- a/unittests/Compiler/Directive.cpp +++ b/unittests/Compiler/Directive.cpp @@ -5,44 +5,130 @@ namespace clice::testing { namespace { -TEST(Directive, Include) { +struct Directive : ::testing::Test, Tester { + clang::SourceManager* SM; + llvm::ArrayRef includes; + llvm::ArrayRef hasIncludes; + llvm::ArrayRef conditions; + llvm::ArrayRef macros; + + void run(const char* standard = "-std=c++20") { + Tester::run("-std=c++23"); + SM = &info->srcMgr(); + auto fid = SM->getMainFileID(); + includes = info->directives()[fid].includes; + hasIncludes = info->directives()[fid].hasIncludes; + conditions = info->directives()[fid].conditions; + macros = info->directives()[fid].macros; + } + + void EXPECT_INCLUDE(std::size_t index, + llvm::StringRef position, + llvm::StringRef path, + std::source_location current = std::source_location::current()) { + auto& include = includes[index]; + auto entry = SM->getFileEntryRefForID(include.fid); + EXPECT_EQ(SourceConverter().toPosition(include.location, *SM), pos(position), current); + EXPECT_EQ(entry ? entry->getName() : "", path, current); + } + + void EXPECT_HAS_INCLUDE(std::size_t index, + llvm::StringRef position, + llvm::StringRef path, + std::source_location current = std::source_location::current()) { + auto& hasInclude = hasIncludes[index]; + EXPECT_EQ(SourceConverter().toPosition(hasInclude.location, *SM), pos(position), current); + EXPECT_EQ(hasInclude.path, path, current); + } + + void EXPECT_CON(std::size_t index, + Condition::BranchKind kind, + llvm::StringRef position, + std::source_location current = std::source_location::current()) { + auto& condition = conditions[index]; + EXPECT_EQ(condition.kind, kind, current); + EXPECT_EQ(SourceConverter().toPosition(condition.loc, *SM), pos(position), current); + } + + void EXPECT_MACRO(std::size_t index, + MacroRef::Kind kind, + llvm::StringRef position, + std::source_location current = std::source_location::current()) { + auto& macro = macros[index]; + EXPECT_EQ(macro.kind, kind, current); + EXPECT_EQ(SourceConverter().toPosition(macro.loc, *SM), pos(position), current); + } +}; + +TEST_F(Directive, Include) { const char* test = ""; const char* test2 = R"cpp( #include "test.h" )cpp"; - const char* main = R"cpp( -#$(0)include "test.h" -#$(1)include "test2.h" -#$(2)include "test3.h" + const char* pragma_once = R"cpp( +#pragma once )cpp"; - Tester tester("main.cpp", main); - tester.addFile("./test.h", test); - tester.addFile("./test2.h", test2); - tester.addFile("./test3.h", ""); - tester.run(); + const char* guard_macro = R"cpp( +#ifndef TEST3_H +#define TEST3_H +#endif +)cpp"; - auto& info = tester.info; - auto& includes = info.directive(info.srcMgr().getMainFileID()).includes; + const char* main = R"cpp( +#$(0)include "test.h" +#$(1)include "test.h" +#$(2)include "pragma_once.h" +#$(3)include "pragma_once.h" +#$(4)include "guard_macro.h" +#$(5)include "guard_macro.h" +)cpp"; - auto EXPECT_INCLUDE = [&](std::size_t index, - llvm::StringRef position, - llvm::StringRef path, - std::source_location current = std::source_location::current()) { - auto& include = includes[index]; - EXPECT_EQ(SourceConverter().toPosition(include.loc, info.srcMgr()), tester.pos(position)); - EXPECT_EQ(include.path, path); - }; + addMain("main.cpp", main); - EXPECT_EQ(includes.size(), 3); - EXPECT_INCLUDE(0, "0", "test.h"); - EXPECT_INCLUDE(1, "1", "test2.h"); - EXPECT_INCLUDE(2, "2", "test3.h"); + using Path = llvm::SmallString<128>; + Path ptest, ppragma_once, pguard_macro; + path::append(ptest, ".", "test.h"); + path::append(ppragma_once, ".", "pragma_once.h"); + path::append(pguard_macro, ".", "guard_macro.h"); + + addFile(ptest, test); + addFile(ppragma_once, pragma_once); + addFile(pguard_macro, guard_macro); + run(); + + EXPECT_EQ(includes.size(), 6); + EXPECT_INCLUDE(0, "0", ptest); + EXPECT_INCLUDE(1, "1", ptest); + EXPECT_INCLUDE(2, "2", ppragma_once); + EXPECT_INCLUDE(3, "3", ""); + EXPECT_INCLUDE(4, "4", pguard_macro); + EXPECT_INCLUDE(5, "5", ""); } -TEST(Directive, Condition) { +TEST_F(Directive, HasInclude) { + const char* test = ""; + + const char* main = R"cpp( +#if __has_include($(0)"test.h") +#endif +)cpp"; + + addMain("main.cpp", main); + + llvm::SmallString<128> path; + path::append(path, ".", "test.h"); + addFile(path, test); + + run(); + + EXPECT_EQ(hasIncludes.size(), 1); + EXPECT_HAS_INCLUDE(0, "0", path); +} + +TEST_F(Directive, Condition) { const char* code = R"cpp( #$(0)if 0 @@ -61,34 +147,21 @@ TEST(Directive, Condition) { #$(7)endif )cpp"; - Tester tester("main.cpp", code); - tester.run("-std=c++23"); - auto& info = tester.info; - auto& conditions = info.directive(info.srcMgr().getMainFileID()).conditions; - - auto EPXECT_CON = [&](std::size_t index, - Condition::BranchKind kind, - llvm::StringRef position, - std::source_location current = std::source_location::current()) { - auto& condition = conditions[index]; - EXPECT_EQ(condition.kind, kind, current); - EXPECT_EQ(SourceConverter().toPosition(condition.loc, info.srcMgr()), - tester.pos(position), - current); - }; + addMain("main.cpp", code); + run("-std=c++23"); EXPECT_EQ(conditions.size(), 8); - EPXECT_CON(0, Condition::BranchKind::If, "0"); - EPXECT_CON(1, Condition::BranchKind::Elif, "1"); - EPXECT_CON(2, Condition::BranchKind::Else, "2"); - EPXECT_CON(3, Condition::BranchKind::EndIf, "3"); - EPXECT_CON(4, Condition::BranchKind::Ifdef, "4"); - EPXECT_CON(5, Condition::BranchKind::Elifdef, "5"); - EPXECT_CON(6, Condition::BranchKind::Else, "6"); - EPXECT_CON(7, Condition::BranchKind::EndIf, "7"); + EXPECT_CON(0, Condition::BranchKind::If, "0"); + EXPECT_CON(1, Condition::BranchKind::Elif, "1"); + EXPECT_CON(2, Condition::BranchKind::Else, "2"); + EXPECT_CON(3, Condition::BranchKind::EndIf, "3"); + EXPECT_CON(4, Condition::BranchKind::Ifdef, "4"); + EXPECT_CON(5, Condition::BranchKind::Elifdef, "5"); + EXPECT_CON(6, Condition::BranchKind::Else, "6"); + EXPECT_CON(7, Condition::BranchKind::EndIf, "7"); } -TEST(Directive, Macro) { +TEST_F(Directive, Macro) { const char* code = R"cpp( #define $(0)expr(v) v @@ -108,21 +181,8 @@ int y = $(6)expr($(7)expr(1)); )cpp"; - Tester tester("main.cpp", code); - tester.run(); - auto& info = tester.info; - auto& macros = info.directive(info.srcMgr().getMainFileID()).macros; - - auto EXPECT_MACRO = [&](std::size_t index, - MacroRef::Kind kind, - llvm::StringRef position, - std::source_location current = std::source_location::current()) { - auto& macro = macros[index]; - EXPECT_EQ(macro.kind, kind, current); - EXPECT_EQ(SourceConverter().toPosition(macro.loc, info.srcMgr()), - tester.pos(position), - current); - }; + addMain("main.cpp", code); + run(); EXPECT_EQ(macros.size(), 9); EXPECT_MACRO(0, MacroRef::Kind::Def, "0"); diff --git a/unittests/Compiler/Module.cpp b/unittests/Compiler/Module.cpp index 6bd04ad3..f182f632 100644 --- a/unittests/Compiler/Module.cpp +++ b/unittests/Compiler/Module.cpp @@ -1,5 +1,5 @@ #include "Test/Test.h" -#include "Compiler/Compiler.h" +#include "Compiler/Compilation.h" #include "llvm/Support/ToolOutputFile.h" namespace clice::testing { diff --git a/unittests/Compiler/Preamble.cpp b/unittests/Compiler/Preamble.cpp index 029add70..34fd09ee 100644 --- a/unittests/Compiler/Preamble.cpp +++ b/unittests/Compiler/Preamble.cpp @@ -1,9 +1,57 @@ #include "Test/Test.h" +#include "Basic/SourceConverter.h" #include "Compiler/Preamble.h" +#include "Compiler/Compilation.h" namespace clice::testing { -namespace {} +namespace { + +TEST(Preamble, ComputePreambleBound) { + proto::Position pos; + SourceConverter converter; + + Annotation annotation = {"\n\n\nint x = 1;"}; + + auto compute = [&](llvm::StringRef source) { + annotation = {source}; + auto content = annotation.source(); + return converter.toPosition(content, computePreambleBound(content)); + }; + + pos = compute("#include $(end)"); + EXPECT_EQ(pos, annotation.pos("end")); + + pos = compute("#include $(end)\n"); + EXPECT_EQ(pos, annotation.pos("end")); + + pos = compute(R"cpp( +#ifdef TEST +#include +#define 1 +#endif$(end) + )cpp"); + EXPECT_EQ(pos, annotation.pos("end")); + + pos = compute(R"cpp( +#include $(end) +int x = 1; + )cpp"); + EXPECT_EQ(pos, annotation.pos("end")); + + pos = compute(R"cpp( +module; +#include $(end) +export module test; + )cpp"); + EXPECT_EQ(pos, annotation.pos("end")); +} + +TEST(Preamble, Build) { + CompilationParams params; +} + +} // namespace } // namespace clice::testing diff --git a/unittests/Compiler/Resolver.cpp b/unittests/Compiler/Resolver.cpp index b02c8915..a22cb86f 100644 --- a/unittests/Compiler/Resolver.cpp +++ b/unittests/Compiler/Resolver.cpp @@ -39,10 +39,10 @@ void run(llvm::StringRef code, std::source_location current = std::source_locati } }; - Run run{tester.info}; - run.TraverseAST(tester.info.context()); + Run run{*tester.info}; + run.TraverseAST(tester.info->context()); - auto input = tester.info.resolver().resolve(run.input); + auto input = tester.info->resolver().resolve(run.input); auto expect = run.expect; EXPECT_EQ(input.isNull(), false); diff --git a/unittests/Feature/DocumentSymbol.cpp b/unittests/Feature/DocumentSymbol.cpp index 1b0d0924..1a37e398 100644 --- a/unittests/Feature/DocumentSymbol.cpp +++ b/unittests/Feature/DocumentSymbol.cpp @@ -46,7 +46,7 @@ namespace _1::_2{ Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 8); } @@ -75,7 +75,7 @@ int main(int argc, char* argv[]) { Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 9); } @@ -100,7 +100,7 @@ struct x { Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 7); } @@ -120,7 +120,7 @@ struct S { Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 6); } @@ -142,7 +142,7 @@ struct _0 { Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 7); } @@ -167,7 +167,7 @@ enum B { Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 8); } @@ -181,7 +181,7 @@ int y = 2; Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); ASSERT_EQ(total_size(res), 2); } @@ -211,7 +211,7 @@ VAR(test) Tester txs("main.cpp", main); txs.run(); - auto res = feature::documentSymbol(txs.info, converter); + auto res = feature::documentSymbol(*txs.info, converter); // dbg(res); // clang-format off diff --git a/unittests/Feature/FoldingRange.cpp b/unittests/Feature/FoldingRange.cpp index 43306595..3287b55e 100644 --- a/unittests/Feature/FoldingRange.cpp +++ b/unittests/Feature/FoldingRange.cpp @@ -16,7 +16,7 @@ struct FoldingRange : public ::testing::Test { FoldingRangeParams param; SourceConverter converter; - result = feature::foldingRange(param, info, converter); + result = feature::foldingRange(param, *info, converter); } void EXPECT_RANGE(std::size_t index, diff --git a/unittests/Feature/InlayHint.cpp b/unittests/Feature/InlayHint.cpp index 23c9b763..b938e2e9 100644 --- a/unittests/Feature/InlayHint.cpp +++ b/unittests/Feature/InlayHint.cpp @@ -24,7 +24,7 @@ protected: tester->run(); auto& info = tester->info; SourceConverter converter; - result = feature::inlayHints({.range = range}, info, converter, option); + result = feature::inlayHints({.range = range}, *info, converter, option); } std::optional tester; diff --git a/unittests/Index/Template.cpp b/unittests/Index/Template.cpp index 137c7e18..57759ddf 100644 --- a/unittests/Index/Template.cpp +++ b/unittests/Index/Template.cpp @@ -14,7 +14,7 @@ void $(2)foo() {} IndexTester tester{"main.cpp", code}; tester.run(); - auto indices = index::test(tester.info); + auto indices = index::test(*tester.info); std::size_t total = 0; llvm::SmallVector symbols; diff --git a/unittests/Server/Database.cpp b/unittests/Server/Database.cpp new file mode 100644 index 00000000..727daa07 --- /dev/null +++ b/unittests/Server/Database.cpp @@ -0,0 +1,14 @@ +#include "Test/Test.h" +#include "Server/Database.h" + +namespace clice::testing { + +namespace { + +TEST(CompilationDatabase, Command) {} + +TEST(CompilationDatabase, Module) {} + +} // namespace + +} // namespace clice::testing diff --git a/xmake.lua b/xmake.lua index d501655c..8d19cbbc 100644 --- a/xmake.lua +++ b/xmake.lua @@ -20,7 +20,7 @@ if has_config("dev") then end end -add_requires("llvm", "libuv") +add_requires("llvm", "libuv", "toml++") add_rules("mode.release", "mode.debug") set_languages("c++23") @@ -33,6 +33,7 @@ target("clice-core") add_includedirs("include", {public = true}) add_packages("libuv", {public = true}) + add_packages("toml++", {public = true}) if is_mode("debug") then add_packages("llvm", { public = true,