diff --git a/include/Basic/Basic.h b/include/Basic/Basic.h index f7461325..4a849ed4 100644 --- a/include/Basic/Basic.h +++ b/include/Basic/Basic.h @@ -5,7 +5,7 @@ namespace clice { -class Compiler; +class ASTInfo; } diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h index dad9fb1f..2fd76e1b 100644 --- a/include/Compiler/Compiler.h +++ b/include/Compiler/Compiler.h @@ -6,102 +6,26 @@ namespace clice { -class Compiler { -public: - Compiler(llvm::StringRef filepath, - llvm::StringRef content, - llvm::ArrayRef args, - clang::DiagnosticConsumer* consumer = nullptr, - llvm::IntrusiveRefCntPtr vfs = llvm::vfs::getRealFileSystem()); - - Compiler(llvm::ArrayRef args, - clang::DiagnosticConsumer* consumer = nullptr, - llvm::IntrusiveRefCntPtr vfs = llvm::vfs::getRealFileSystem()) : - Compiler("", "", args, consumer, vfs) {} - - ~Compiler(); - - /// Is success, return true. - bool applyPCH(llvm::StringRef filepath, std::uint32_t bound, bool endAtStart = false); - - bool applyPCM(llvm::StringRef filepath, llvm::StringRef name); - - /// Build AST. - void buildAST(); - - /// Generate the PCH(PreCompiledHeader) to output path. Generally execute - /// `clang::GeneratePCHAction`. The Header part of the source file is stored in the PCH file. - /// Bound is the size of the header part. - void generatePCH(llvm::StringRef outpath, std::uint32_t bound, bool endAtStart = false); - - /// Generate the PCM(PreCompiledModule) to output path. Generally execute - /// `clang::GenerateReducedModuleInterfaceAction`. - void generatePCM(llvm::StringRef outpath); - - /// Run code complete in given file and location. - void codeCompletion(llvm::StringRef filepath, - std::uint32_t line, - std::uint32_t column, - clang::CodeCompleteConsumer* consumer); - - clang::Preprocessor& pp() { - return instance->getPreprocessor(); - } - - clang::Sema& sema() { - return instance->getSema(); - } - - clang::FileManager& fileMgr() { - return instance->getFileManager(); - } - - clang::SourceManager& srcMgr() { - return instance->getSourceManager(); - } - - clang::ASTContext& context() { - return instance->getASTContext(); - } - - clang::TranslationUnitDecl* tu() { - return instance->getASTContext().getTranslationUnitDecl(); - } - - clang::syntax::TokenBuffer& tokBuf() { - return *buffer; - } - - TemplateResolver& resolver() { - return *m_Resolver; - } - -private: - void ExecuteAction(); - -private: - std::string filepath; - std::string content; - std::unique_ptr action; - std::unique_ptr instance; - std::unique_ptr buffer; - std::unique_ptr m_Resolver; -}; - /// All information about AST. struct ASTInfo { std::unique_ptr action; std::unique_ptr instance; - std::unique_ptr tokBuf; + std::unique_ptr tokBuf_; + std::unique_ptr resolver_; + + ASTInfo() = default; ASTInfo(std::unique_ptr action, std::unique_ptr instance, std::unique_ptr tokBuf) : - action(std::move(action)), instance(std::move(instance)), tokBuf(std::move(tokBuf)) {} + action(std::move(action)), instance(std::move(instance)), tokBuf_(std::move(tokBuf)) { + resolver_ = std::make_unique(instance->getSema()); + } ASTInfo(const ASTInfo&) = delete; ASTInfo(ASTInfo&&) = default; + ASTInfo& operator= (ASTInfo&&) = default; ~ASTInfo() { if(action) { @@ -128,6 +52,15 @@ struct ASTInfo { clang::TranslationUnitDecl* tu() { return instance->getASTContext().getTranslationUnitDecl(); } + + clang::syntax::TokenBuffer& tokBuf() { + assert(tokBuf_ && "Token buffer is not available"); + return *tokBuf_; + } + + TemplateResolver& resolver() { + return *resolver_; + } }; struct PCHInfo : ASTInfo { @@ -145,7 +78,7 @@ struct PCHInfo : ASTInfo { llvm::StringRef content, llvm::StringRef mainpath, clang::PreambleBounds bounds) : - ASTInfo(std::move(info)), path(std::move(path)), mainpath(mainpath) { + ASTInfo(std::move(info)), path(path), mainpath(mainpath) { preamble = content.substr(0, bounds.Size).str(); if(bounds.PreambleEndsAtStartOfLine) { @@ -161,7 +94,16 @@ struct PCHInfo : ASTInfo { } }; -struct PCMInfo : ASTInfo {}; +struct PCMInfo : ASTInfo { + /// PCM file path. + std::string path; + /// Module name. + std::string name; + + PCMInfo(ASTInfo info, llvm::StringRef path) : ASTInfo(std::move(info)), path(path) { + name = context().getCurrentNamedModule()->Name; + } +}; /// Information about reuse PCH or PCM. This should be placed in stack. struct Preamble { @@ -178,22 +120,39 @@ struct Preamble { } }; +struct CompliationParams { + llvm::StringRef path; + llvm::StringRef content; + llvm::StringRef outpath; + llvm::StringRef mainpath; + llvm::ArrayRef args; + + /// Information about reuse PCH. + std::string pch; + clang::PreambleBounds bounds = {0, false}; + + /// Information about reuse PCM(name, path). + llvm::SmallVector> pcms; + + void addPCH(const PCHInfo& info) { + pch = info.path; + bounds = info.bounds(); + } +}; + /// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. /// Note this function will not check whether we need to update the PCH or PCM, caller should check /// their reusability and update in time. -llvm::Expected buildAST(llvm::StringRef path, - llvm::StringRef content, - llvm::ArrayRef args, - Preamble* preamble = nullptr); +llvm::Expected buildAST(CompliationParams& params); -llvm::Expected buildPCH(llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef outpath, - llvm::ArrayRef args); +llvm::Expected buildPCH(CompliationParams& params); -llvm::Expected buildPCM(llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef outpath, - llvm::ArrayRef args); +llvm::Expected buildPCM(CompliationParams& params); + +llvm::Expected codeCompleteAt(CompliationParams& params, + uint32_t line, + uint32_t column, + llvm::StringRef file, + clang::CodeCompleteConsumer* consumer); } // namespace clice diff --git a/include/Compiler/Semantic.h b/include/Compiler/Semantic.h index a5b2d5fa..6af3265f 100644 --- a/include/Compiler/Semantic.h +++ b/include/Compiler/Semantic.h @@ -58,7 +58,7 @@ class SemanticVisitor : public clang::RecursiveASTVisitor; - SemanticVisitor(Compiler& compiler, bool mainFileOnly = false) : + SemanticVisitor(ASTInfo& compiler, bool mainFileOnly = false) : sema(compiler.sema()), pp(compiler.pp()), resolver(compiler.resolver()), srcMgr(compiler.srcMgr()), tokBuf(compiler.tokBuf()), mainFileOnly(mainFileOnly) {} diff --git a/include/Feature/SemanticTokens.h b/include/Feature/SemanticTokens.h index f830fab9..1de97eb5 100644 --- a/include/Feature/SemanticTokens.h +++ b/include/Feature/SemanticTokens.h @@ -59,7 +59,7 @@ struct SemanticTokens { namespace clice::feature { /// FIXME: -proto::SemanticTokens semanticTokens(Compiler& compiler, llvm::StringRef filename); +proto::SemanticTokens semanticTokens(ASTInfo& compiler, llvm::StringRef filename); } // namespace clice::feature diff --git a/include/Index/Indexer.h b/include/Index/Indexer.h index a63fc13a..ca237c8e 100644 --- a/include/Index/Indexer.h +++ b/include/Index/Indexer.h @@ -9,7 +9,7 @@ #include "Index.h" namespace clice { -class Compiler; +class ASTInfo; } namespace clice::index { @@ -32,6 +32,6 @@ using Index = index::Index; } // namespace memory /// index the given translation unit. -memory::Index index(Compiler& compiler); +memory::Index index(ASTInfo& compiler); } // namespace clice::index diff --git a/include/Server/Config.h b/include/Server/Config.h index 9fb12251..9e6405ba 100644 --- a/include/Server/Config.h +++ b/include/Server/Config.h @@ -23,8 +23,10 @@ struct ServerOption { struct FrontendOption { std::vector append; std::vector remove; - std::string resource_dictionary = "${binary}/../lib/clang/${llvm_version}"; + std::string index_directory = "${workplace}/.clice/index"; + std::string cache_directory = "${workplace}/.clice/cache"; std::string compile_commands_directory = "${workplace}/build"; + std::string resource_dictionary = "${binary}/../lib/clang/${llvm_version}"; }; const ServerOption& server(); diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index 3efb7f7e..4cde983c 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -4,10 +4,11 @@ #include "Async.h" #include "llvm/ADT/StringMap.h" +#include "Compiler/Compiler.h" namespace clice { -class Compiler; +class ASTInfo; /// Information of building precompiled header. struct PCH { @@ -43,8 +44,6 @@ struct PCH { /// TODO: check whether deps changed through comparing timestamps. return false; } - - void apply(Compiler& compiler) const; }; /// Information of building precompiled module. @@ -63,7 +62,7 @@ struct File { bool isIdle = true; /// The compiler instance of this file. - std::unique_ptr compiler; + ASTInfo compiler; std::deque waitings; }; @@ -93,7 +92,7 @@ public: /// Otherwise, the task will be executed immediately. template auto schedule(llvm::StringRef path, Task&& task) - -> async::promise()))> { + -> async::promise()))> { auto& file = files[path]; if(!file.isIdle) { co_await async::suspend([&](auto handle) { @@ -102,7 +101,7 @@ public: } file.isIdle = false; - auto& compiler = *file.compiler; + auto& compiler = file.compiler; auto result = co_await async::schedule_task([&task, &compiler] { return task(compiler); }); diff --git a/scripts/calculate-line.sh b/scripts/calculate-line.sh index 4f04b5e7..fe0380c6 100755 --- a/scripts/calculate-line.sh +++ b/scripts/calculate-line.sh @@ -1 +1 @@ -cloc . --exclude-dir=build,deps \ No newline at end of file +cloc . --exclude-dir=build,build-release,deps \ No newline at end of file diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp index a16be973..cc308f36 100644 --- a/src/Compiler/Compiler.cpp +++ b/src/Compiler/Compiler.cpp @@ -15,151 +15,6 @@ static void adjustInvocation(clang::CompilerInvocation& invocation) { // FIXME: add more. } -Compiler::Compiler(llvm::StringRef filepath, - llvm::StringRef content, - llvm::ArrayRef args, - clang::DiagnosticConsumer* consumer, - llvm::IntrusiveRefCntPtr vfs) : - filepath(filepath), content(content) { - // FIXME: figure out should we use createInvocation? - clang::CreateInvocationOptions options; - auto invocation = clang::createInvocation(args, options); - - /// FIXME: use a thread safe for every thread. - instance = std::make_unique( - std::make_shared()); - adjustInvocation(*invocation); - - instance->setInvocation(std::move(invocation)); - - // FIXME: customize DiagnosticConsumer - if(consumer) { - instance->createDiagnostics(*vfs, consumer, true); - } else { - instance->createDiagnostics( - *vfs, - new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), - true); - } -} - -bool Compiler::applyPCH(llvm::StringRef filepath, std::uint32_t bound, bool endAtStart) { - // FIXME: check reuseable? - auto& preproc = instance->getPreprocessorOpts(); - preproc.UsePredefines = false; - preproc.ImplicitPCHInclude = filepath; - preproc.PrecompiledPreambleBytes.first = bound; - preproc.PrecompiledPreambleBytes.second = endAtStart; - preproc.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; - return true; -} - -bool Compiler::applyPCM(llvm::StringRef filepath, llvm::StringRef name) { - // FIXME: check reuseable? - instance->getHeaderSearchOpts().PrebuiltModuleFiles.try_emplace(name.str(), filepath); - return true; -} - -void Compiler::buildAST() { - action = std::make_unique(); - instance->getFrontendOpts().DisableFree = false; - ExecuteAction(); - m_Resolver = std::make_unique(instance->getSema()); -} - -void Compiler::generatePCH(llvm::StringRef outpath, std::uint32_t bound, bool endAtStart) { - content = content.substr(0, bound); - instance->getFrontendOpts().OutputFile = outpath; - instance->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH; - instance->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false}; - instance->getPreprocessorOpts().GeneratePreamble = true; - instance->getLangOpts().CompilingPCH = true; - action = std::make_unique(); - ExecuteAction(); -} - -void Compiler::generatePCM(llvm::StringRef outpath) { - instance->getFrontendOpts().OutputFile = outpath; - action = std::make_unique(); - ExecuteAction(); -} - -void Compiler::codeCompletion(llvm::StringRef filepath, - std::uint32_t line, - std::uint32_t column, - clang::CodeCompleteConsumer* consumer) { - auto& location = instance->getFrontendOpts().CodeCompletionAt; - location.FileName = filepath; - location.Line = line; - location.Column = column; - - auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content); - instance->getPreprocessorOpts().addRemappedFile(filepath, buffer.release()); - - instance->setCodeCompletionConsumer(consumer); - - action = std::make_unique(); - - if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { - llvm::errs() << "Failed to begin source file\n"; - std::terminate(); - } - - /// instance->getASTContext().setExternalSource(nullptr); - - if(auto error = action->Execute()) { - llvm::errs() << "Failed to execute action: " << error << "\n"; - std::terminate(); - } -} - -void Compiler::ExecuteAction() { - { - auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content); - instance->getPreprocessorOpts().addRemappedFile(filepath, buffer.release()); - } - - if(auto VFSWithRemapping = createVFSFromCompilerInvocation(instance->getInvocation(), - instance->getDiagnostics(), - llvm::vfs::getRealFileSystem())) { - instance->createFileManager(VFSWithRemapping); - } - - if(!instance->createTarget()) { - llvm::errs() << "Failed to create target\n"; - std::terminate(); - } - - if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { - llvm::errs() << "Failed to begin source file\n"; - std::terminate(); - } - - auto& preproc = instance->getPreprocessor(); - // FIXME: add PPCallbacks to collect information. - - // Beacuse CompilerInstance may create new Preprocessor in `BeginSourceFile`, - // So we must need to create TokenCollector here. - clang::syntax::TokenCollector collector{preproc}; - - // FIXME: clang-tidy, include-fixer, etc? - - if(auto error = action->Execute()) { - llvm::errs() << "Failed to execute action: " << error << "\n"; - std::terminate(); - } - - // Build TokenBuffer and index expanded tokens for improving performance. - buffer = std::make_unique(std::move(collector).consume()); - buffer->indexExpandedTokens(); -} - -Compiler::~Compiler() { - if(action) { - action->EndSourceFile(); - } -} - static auto createInstance(llvm::ArrayRef args) { auto instance = std::make_unique(); @@ -178,8 +33,28 @@ static auto createInstance(llvm::ArrayRef args) { return instance; } +static void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) { + auto& PPOpts = instance.getPreprocessorOpts(); + auto& pch = params.pch; + auto& bounds = params.bounds; + auto& pcms = params.pcms; + if(bounds.Size != 0) { + PPOpts.UsePredefines = false; + PPOpts.ImplicitPCHInclude = std::move(pch); + PPOpts.PrecompiledPreambleBytes.first = bounds.Size; + PPOpts.PrecompiledPreambleBytes.second = bounds.PreambleEndsAtStartOfLine; + PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; + } + + for(auto& [name, path]: pcms) { + auto& HSOpts = instance.getHeaderSearchOpts(); + HSOpts.PrebuiltModuleFiles.try_emplace(std::move(name), std::move(path)); + } +} + static llvm::Expected ExecuteAction(std::unique_ptr instance, - clang::frontend::ActionKind kind) { + clang::frontend::ActionKind kind, + bool collectPP = true) { std::unique_ptr action; if(kind == clang::frontend::ActionKind::ParseSyntaxOnly) { action = std::make_unique(); @@ -204,89 +79,108 @@ static llvm::Expected ExecuteAction(std::unique_ptrgetPreprocessor(); - clang::syntax::TokenCollector collector{PP}; + if(collectPP) { + auto& PP = instance->getPreprocessor(); + clang::syntax::TokenCollector collector{PP}; - if(auto error = action->Execute()) { - return clice::error("Failed to execute action, because {} ", error); + if(auto error = action->Execute()) { + return clice::error("Failed to execute action, because {} ", error); + } + + auto tokBuf = std::make_unique(std::move(collector).consume()); + tokBuf->indexExpandedTokens(); + + return ASTInfo(std::move(action), std::move(instance), std::move(tokBuf)); + } else { + if(auto error = action->Execute()) { + return clice::error("Failed to execute action, because {} ", error); + } + + return ASTInfo(std::move(action), std::move(instance), nullptr); } - - auto tokBuf = std::make_unique(std::move(collector).consume()); - tokBuf->indexExpandedTokens(); - - return ASTInfo(std::move(action), std::move(instance), std::move(tokBuf)); } -llvm::Expected buildAST(llvm::StringRef path, - llvm::StringRef content, - llvm::ArrayRef args, - Preamble* preamble) { - auto instance = createInstance(args); +llvm::Expected buildAST(CompliationParams& params) { + auto instance = createInstance(params.args); - auto& PPOpts = instance->getPreprocessorOpts(); + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content); + instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); - auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content); - /// FIXME: Check PPOpts.RetainRemappedFileBuffers. - PPOpts.addRemappedFile(path, buffer.release()); - - if(preamble) { - auto& [pch, bounds, pcms] = *preamble; - if(bounds.Size != 0) { - PPOpts.UsePredefines = false; - PPOpts.ImplicitPCHInclude = std::move(pch); - PPOpts.PrecompiledPreambleBytes.first = bounds.Size; - PPOpts.PrecompiledPreambleBytes.second = bounds.PreambleEndsAtStartOfLine; - PPOpts.DisablePCHOrModuleValidation = clang::DisableValidationForModuleKind::PCH; - } - - for(auto& [name, path]: pcms) { - auto& HSOpts = instance->getHeaderSearchOpts(); - HSOpts.PrebuiltModuleFiles.try_emplace(std::move(name), std::move(path)); - } - } + applyPreamble(*instance, params); return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly); } -llvm::Expected buildPCH(llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef outpath, - llvm::StringRef mainpath, - std::size_t index, - llvm::ArrayRef args) { - auto instance = createInstance(args); +llvm::Expected buildPCH(CompliationParams& params) { + auto instance = createInstance(params.args); clang::PreambleBounds bounds = {0, false}; - if(mainpath == path) { + if(params.mainpath == params.path) { /// If mainpath is equal to path, just tokenize the content to get preamble bounds. - bounds = clang::Lexer::ComputePreamble(content, {}, false); + bounds = clang::Lexer::ComputePreamble(params.content, {}, false); } else { /// FIXME: if the mainpath is not equal to path, we need to preprocess the mainpath to get /// the preamble bounds. std::terminate(); } - instance->getFrontendOpts().OutputFile = outpath; + /// Set options to generate PCH. + instance->getFrontendOpts().OutputFile = params.outpath; instance->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH; instance->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false}; instance->getPreprocessorOpts().GeneratePreamble = true; instance->getLangOpts().CompilingPCH = true; - auto buffer = llvm::MemoryBuffer::getMemBufferCopy(content.substr(0, bounds.Size)); - instance->getPreprocessorOpts().addRemappedFile(path, buffer.release()); + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content.substr(0, bounds.Size)); + instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); if(auto info = ExecuteAction(std::move(instance), clang::frontend::ActionKind::GeneratePCH)) { - return PCHInfo(std::move(*info), outpath, mainpath, content, bounds); + return PCHInfo(std::move(*info), params.outpath, params.content, params.mainpath, bounds); } else { return info.takeError(); } } -llvm::Expected buildPCH(llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef outpath, - llvm::ArrayRef args) { - return buildPCH(path, content, outpath, path, 0, args); +llvm::Expected buildPCM(CompliationParams& params) { + auto instance = createInstance(params.args); + + /// Set options to generate PCM. + instance->getFrontendOpts().OutputFile = params.outpath; + instance->getFrontendOpts().ProgramAction = clang::frontend::GenerateReducedModuleInterface; + + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content); + instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); + + applyPreamble(*instance, params); + + if(auto info = ExecuteAction(std::move(instance), + clang::frontend::ActionKind::GenerateReducedModuleInterface)) { + return PCMInfo(std::move(*info), params.outpath); + } else { + return info.takeError(); + } +} + +llvm::Expected codeCompleteAt(CompliationParams& params, + uint32_t line, + uint32_t column, + llvm::StringRef file, + clang::CodeCompleteConsumer* consumer) { + auto instance = createInstance(params.args); + + /// Set options to run code completion. + instance->getFrontendOpts().CodeCompletionAt.Line = line; + instance->getFrontendOpts().CodeCompletionAt.Column = column; + instance->getFrontendOpts().CodeCompletionAt.FileName = file; + instance->setCodeCompletionConsumer(consumer); + + auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content); + /// FIXME: Check PPOpts.RetainRemappedFileBuffers. + instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); + + applyPreamble(*instance, params); + + return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly, false); } } // namespace clice diff --git a/src/Feature/CodeCompletion.cpp b/src/Feature/CodeCompletion.cpp index b5398187..6d7b1f57 100644 --- a/src/Feature/CodeCompletion.cpp +++ b/src/Feature/CodeCompletion.cpp @@ -57,15 +57,17 @@ private: } // namespace -std::vector codeCompletion(Compiler& compiler, +std::vector codeCompletion(CompliationParams& params, llvm::StringRef filepath, proto::Position position, const config::CodeCompletionOption& option) { // TODO: decode here. - compiler.codeCompletion(filepath, - position.line, - position.character, - new CodeCompletionCollector({})); + auto result = codeCompleteAt(params, + position.line, + position.character, + filepath, + new CodeCompletionCollector({})); + if(result) {} return {}; } diff --git a/src/Feature/SemanticTokens.cpp b/src/Feature/SemanticTokens.cpp index c2584913..0b5f1a8d 100644 --- a/src/Feature/SemanticTokens.cpp +++ b/src/Feature/SemanticTokens.cpp @@ -513,7 +513,7 @@ struct SemanticToken { class HighlightBuilder : public SemanticVisitor { public: - HighlightBuilder(Compiler& compiler) : SemanticVisitor(compiler, true) {} + HighlightBuilder(ASTInfo& compiler) : SemanticVisitor(compiler, true) {} void handleOccurrence(const clang::Decl* decl, clang::SourceLocation location, @@ -710,7 +710,7 @@ public: } // namespace -proto::SemanticTokens semanticTokens(Compiler& compiler, llvm::StringRef filename) { +proto::SemanticTokens semanticTokens(ASTInfo& compiler, llvm::StringRef filename) { HighlightBuilder builder(compiler); return builder.build(); } diff --git a/src/Index/SymbolBuilder.h b/src/Index/SymbolBuilder.h index 107e0dfd..77259856 100644 --- a/src/Index/SymbolBuilder.h +++ b/src/Index/SymbolBuilder.h @@ -187,7 +187,7 @@ public: return symbolCache.contains(decl); } - void indexTU(memory::Index& result, Compiler& compiler); + void indexTU(memory::Index& result, ASTInfo& compiler); ~SymbolBuilder() { // for(auto [decl, _]: symbolCache) { diff --git a/src/Index/SymbolCollector.cpp b/src/Index/SymbolCollector.cpp index 21b8a74a..4c0a5939 100644 --- a/src/Index/SymbolCollector.cpp +++ b/src/Index/SymbolCollector.cpp @@ -616,7 +616,7 @@ void sortFiles(std::vector& files, std::vector& location }); } -void SymbolBuilder::indexTU(memory::Index& result, Compiler& compiler) { +void SymbolBuilder::indexTU(memory::Index& result, ASTInfo& compiler) { index = &result; SymbolCollector collector(*this, compiler.tu()->getASTContext()); collector.TraverseAST(compiler.context()); @@ -638,7 +638,7 @@ void SymbolBuilder::indexTU(memory::Index& result, Compiler& compiler) { } // namespace -memory::Index index(Compiler& compiler) { +memory::Index index(ASTInfo& compiler) { memory::Index result; SymbolBuilder builder(compiler.sema(), compiler.tokBuf()); builder.indexTU(result, compiler); diff --git a/src/Server/Config.cpp b/src/Server/Config.cpp index afab1c28..8dd338d3 100644 --- a/src/Server/Config.cpp +++ b/src/Server/Config.cpp @@ -1,4 +1,4 @@ -#define TOML_EXCEPTIONS 0 +#define TOML_EXCEPTIONS 0 #include #include @@ -66,6 +66,8 @@ std::string replace(std::string_view text) { } // namespace int parse(llvm::StringRef execute, llvm::StringRef filepath) { + predefined["binary"] = execute; + auto toml = toml::parse_file(filepath); if(toml.failed()) { log::fatal("Failed to parse config file: {0}, Beacuse {1}", @@ -100,6 +102,8 @@ void init(std::string_view workplace) { field = replace(field); } }); + + log::info("Config initialized successfully, result: {0}", json::serialize(config)); return; } diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index 8c5c3bdf..2d9ed3ec 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -3,15 +3,6 @@ namespace clice { -void PCH::apply(Compiler& compiler) const { - bool endAtStart = preamble.ends_with('@'); - auto size = preamble.size() - endAtStart; - - if(size != 0) { - compiler.applyPCH(path, size, endAtStart); - } -} - struct Tracer { std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); @@ -33,11 +24,19 @@ async::promise Scheduler::updatePCH(llvm::StringRef filepath, Tracer tracer; clang::PreambleBounds bounds = {0, 0}; + CompliationParams params; + params.path = filepath; + params.content = content; + params.args = args; + co_await async::schedule_task([&] { - Compiler compiler(filepath, content, args); bounds = clang::Lexer::ComputePreamble(content, {}, false); if(bounds.Size != 0) { - compiler.generatePCH(outpath, bounds.Size, bounds.PreambleEndsAtStartOfLine); + auto pch = buildPCH(params); + if(!pch) { + log::fatal("Failed to build PCH for {0}", filepath.str()); + return; + } } }); @@ -93,16 +92,23 @@ async::promise Scheduler::buildAST(llvm::StringRef filepath, llvm::StringR Tracer tracer; log::info("Start building AST for {0}", filepath.str()); - auto task = [&path, &content, &args, pch = pchs.at(filepath)] { + CompliationParams params; + params.path = path; + params.content = content; + params.args = args; + // params.addPCH(pchs.at(filepath)); + + auto task = [&] { /// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be /// Invalid Because other changed the `pchs` map. We also cannot to retrieve the `pch` from /// the `pchs` map in this task, beacuse it is called in thread pool which will result in /// data race. So temporarily copy the `pch` here. There must be a better way to solve this /// problem. - std::unique_ptr compiler = std::make_unique(path, content, args); - pch.apply(*compiler); - compiler->buildAST(); - return compiler; + auto info = clice::buildAST(params); + if(!info) { + log::fatal("Failed to build AST for {0}", filepath.str()); + } + return std::move(*info); }; auto compiler = co_await async::schedule_task(std::move(task)); diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 98f4e854..93eb174b 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -72,7 +72,8 @@ void Server::run(int argc, const char** argv) { } async::promise Server::onInitialize(json::Value id, const proto::InitializeParams& params) { - + auto workplace = URI::resolve(params.workspaceFolders[0].uri); + config::init(workplace); async::write(std::move(id), json::serialize(proto::InitializeResult())); co_return; } @@ -202,7 +203,7 @@ async::promise Server::onDocumentSymbol(json::Value id, async::promise Server::onSemanticTokens(json::Value id, const proto::SemanticTokensParams& params) { auto path = URI::resolve(params.textDocument.uri); - auto tokens = co_await scheduler.schedule(path, [&](Compiler& compiler) { + auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) { return feature::semanticTokens(compiler, ""); }); async::write(std::move(id), json::serialize(tokens)); diff --git a/unittests/AST/ASTVisitor.cpp b/unittests/AST/ASTVisitor.cpp deleted file mode 100644 index 69eb95d7..00000000 --- a/unittests/AST/ASTVisitor.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include -#include -#include -#include -#include -#include "../Test.h" -#include "clang/AST/DeclTemplate.h" -#include "Compiler/Semantic.h" -#include "Feature/SemanticTokens.h" - -namespace { - -using namespace clice; - -class ASTVisitor : public clang::RecursiveASTVisitor { -public: - Compiler& compiler; - clang::SourceManager& srcMgr; - clang::Sema& sema; - - ASTVisitor(Compiler& compiler, clang::SourceManager& srcMgr, clang::Sema& sema) : - compiler(compiler), srcMgr(srcMgr), sema(sema) {} - - using Base = clang::RecursiveASTVisitor; - - void dump(clang::SourceLocation loc) { - if(loc.isMacroID()) { - llvm::outs() << "expansion: "; - srcMgr.getExpansionLoc(loc).print(llvm::outs(), srcMgr); - llvm::outs() << "\n"; - - llvm::outs() << "spelling: "; - srcMgr.getSpellingLoc(loc).print(llvm::outs(), srcMgr); - llvm::outs() << "\n"; - - llvm::outs() << "file: "; - srcMgr.getFileLoc(loc).print(llvm::outs(), srcMgr); - llvm::outs() << "\n"; - - auto [begin, end] = srcMgr.getImmediateExpansionRange(loc).getAsRange(); - if(begin.isValid() && end.isValid()) { - llvm::outs() << "expansion Range: \n"; - dump(begin); - dump(end); - llvm::outs() << "\n"; - } - - llvm::outs() << "\n"; - } - } - - bool VisitUnresolvedLookupExpr(clang::UnresolvedLookupExpr* expr) { - if(srcMgr.isInMainFile(expr->getNameLoc())) { - expr->dump(); - } - return true; - } - - bool VisitTemplateSpecializationTypeLoc(clang::TemplateSpecializationTypeLoc loc) { - if(srcMgr.isInMainFile(loc.getLAngleLoc())) { - loc.dump(); - } - return true; - } - - bool VisitDependentScopeDeclRefExpr(clang::DependentScopeDeclRefExpr* expr) { - expr->dump(); - for(auto member: compiler.resolver().lookup(expr)) { - member->dump(); - } - clang::UserDefinedLiteral* literal; - auto x = "x\n"; - return true; - } - - bool VisitParmVarDecl(clang::ParmVarDecl* decl) { - // decl->getTypeSourceInfo()->getTypeLoc().dump(); - return true; - } - - bool VisitTypeLoc(clang::TypeLoc loc) { - loc.dump(); - loc.getBeginLoc().dump(srcMgr); - return true; - } -}; - -TEST(clice, ASTVisitor) { - foreachFile("ASTVisitor", [](std::string filepath, llvm::StringRef content) { - if(filepath.ends_with("test.cpp")) { - std::vector compileArgs = { - "clang++", - "-std=c++23", - filepath.c_str(), - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - - auto bounds = clang::Lexer::ComputePreamble(content, {}, false); - // auto start1 = std::chrono::steady_clock::now(); - //{ - // Compiler compiler(filepath, content, compileArgs); - // compiler.generatePCH("/home/ykiko/C++/clice2/build/cache/xxx.pch", - // bounds.Size, - // bounds.PreambleEndsAtStartOfLine); - // } - // auto end1 = std::chrono::steady_clock::now(); - - auto start2 = std::chrono::steady_clock::now(); - Compiler compiler(compileArgs); - // compiler.applyPCH("/home/ykiko/C++/clice2/build/cache/xxx.pch", - // bounds.Size, - // bounds.PreambleEndsAtStartOfLine); - compiler.buildAST(); - auto end2 = std::chrono::steady_clock::now(); - - // ASTVisitor visitor(compiler, compiler.srcMgr(), compiler.sema()); - // visitor.TraverseAST(compiler.sema().getASTContext()); - - auto start3 = std::chrono::steady_clock::now(); - auto r = feature::semanticTokens(compiler, filepath); - auto end3 = std::chrono::steady_clock::now(); - - // llvm::outs() - // << "Build PCH: " - // << std::chrono::duration_cast(end1 - start1).count() - // << "ms\n"; - llvm::outs() - << "Build AST: " - << std::chrono::duration_cast(end2 - start2).count() - << "ms\n"; - llvm::outs() - << "Semantic Tokens: " - << std::chrono::duration_cast(end3 - start3).count() - << "ms\n"; - } - }); -} - -} // namespace - diff --git a/unittests/AST/Compiler.cpp b/unittests/AST/Compiler.cpp deleted file mode 100644 index 6b95a6b7..00000000 --- a/unittests/AST/Compiler.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include "../Test.h" -#include -#include - -namespace { - -using namespace clice; - -TEST(clice, ModuleScanner) { - - using namespace clang::tooling::dependencies; - - DependencyScanningService service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::P1689); - DependencyScanningTool tool(service); - - foreachFile("ModuleScanner", [&](std::string file, llvm::StringRef content) { - clang::tooling::CompileCommand command; - command.Filename = file; - command.CommandLine = {"clang++", "-std=c++20", file}; - auto rule = tool.getP1689ModuleDependencyFile(command, "CWD"); - - if(rule) { - llvm::outs() << "Module: " << rule->Provides->ModuleName << "\n"; - llvm::outs() << "Module: " << rule->Provides->IsStdCXXModuleInterface << "\n"; - for(auto& info: rule->Requires) { - llvm::outs() << info.ModuleName << " -> " << info.SourcePath << "\n"; - } - } - }); -} - -TEST(clice, PCH) { - const char* code = R"( -#include - -int main(){ - printf("Hello world"); - return 0; -} - -)"; - - std::vector compileArgs = { - "clang++", - "-std=c++20", - "main.cpp", - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - - auto bounds = clang::Lexer::ComputePreamble(code, {}, false); - { - Compiler compiler("main.cpp", code, compileArgs); - compiler.generatePCH("/home/ykiko/C++/clice2/build/cache/xxx.pch", - bounds.Size, - bounds.PreambleEndsAtStartOfLine); - } - - Compiler compiler("main.cpp", code, compileArgs); - compiler.applyPCH("/home/ykiko/C++/clice2/build/cache/xxx.pch", bounds.Size, bounds.PreambleEndsAtStartOfLine); - compiler.buildAST(); - compiler.tu()->dump(); -} - -TEST(clice, PCM) { - const char* mod = R"( -export module M; - -export constexpr int f() { - return 42; -} - -export void x; -)"; - - const char* code = R"( -module; - -export module Main; - -import M; - -module: private; - -int main() { - constexpr int x = f(); - return x; -} -)"; - - std::vector compileArgs = { - "clang++", - "-std=c++20", - "main.cpp", - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - - { - Compiler compiler("main.cpp", mod, compileArgs); - compiler.generatePCM("/home/ykiko/C++/clice2/build/cache/M.pcm"); - } - - llvm::outs() << "=====================\n"; - - static Compiler compiler("main.cpp", code, compileArgs); - compiler.applyPCM("/home/ykiko/C++/clice2/build/cache/M.pcm", "M"); - compiler.buildAST(); - compiler.tu()->dump(); - - class ModuleVisitor : public clang::RecursiveASTVisitor { - public: - bool VisitImportDecl(clang::ImportDecl* decl) { - for(auto loc: decl->getIdentifierLocs()) { - loc.dump(compiler.srcMgr()); - } - auto mod = decl->getImportedModule(); - mod->DefinitionLoc.dump(compiler.srcMgr()); - // llvm::outs() << "Module: " << decl->getImportedModule()->getFullModuleName() << "\n"; - - return true; - } - }; - - ModuleVisitor visitor; - visitor.TraverseAST(compiler.context()); - - // instance->getASTContext().getTranslationUnitDecl()->dump(); -} - -} // namespace - diff --git a/unittests/AST/Diagnostic.cpp b/unittests/AST/Diagnostic.cpp index 2fcf29db..56003fde 100644 --- a/unittests/AST/Diagnostic.cpp +++ b/unittests/AST/Diagnostic.cpp @@ -7,17 +7,17 @@ namespace { using namespace clice; TEST(clice, Diagnostic) { - foreachFile("Diagnostic", [](std::string file, llvm::StringRef content) { - std::vector compileArgs = { - "clang++", - "-std=c++20", - file.c_str(), - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - Compiler compiler(file, content, compileArgs, new DiagnosticCollector()); - compiler.buildAST(); - }); + // foreachFile("Diagnostic", [](std::string file, llvm::StringRef content) { + // std::vector compileArgs = { + // "clang++", + // "-std=c++20", + // file.c_str(), + // "-resource-dir", + // "/home/ykiko/C++/clice2/build/lib/clang/20", + // }; + // Compiler compiler(file, content, compileArgs, new DiagnosticCollector()); + // compiler.buildAST(); + // }); } } // namespace diff --git a/unittests/AST/Resolver.cpp b/unittests/AST/Resolver.cpp index 81032e7a..b197604b 100644 --- a/unittests/AST/Resolver.cpp +++ b/unittests/AST/Resolver.cpp @@ -17,8 +17,17 @@ struct TemplateResolverTester : public clang::RecursiveASTVisitor("main.cpp", code, compileArgs); - compiler->buildAST(); + CompliationParams params; + params.path = "main.cpp"; + params.content = code; + params.args = compileArgs; + auto info = buildAST(params); + if(!info) { + llvm::errs() << info.takeError() << "\n"; + return; + } + + compiler = std::move(*info); test(); } @@ -35,11 +44,11 @@ struct TemplateResolverTester : public clang::RecursiveASTVisitortu()); + TraverseDecl(compiler.tu()); EXPECT_FALSE(input.isNull()); EXPECT_FALSE(expect.isNull()); - auto& resolver = compiler->resolver(); + auto& resolver = compiler.resolver(); clang::QualType result = resolver.resolve(input); EXPECT_EQ(result.getCanonicalType(), expect.getCanonicalType()); if(result.getCanonicalType() != expect.getCanonicalType()) { @@ -56,7 +65,7 @@ struct TemplateResolverTester : public clang::RecursiveASTVisitor compileArgs; - std::unique_ptr compiler; + ASTInfo compiler; }; TEST(TemplateResolver, TypeParameterType) { diff --git a/unittests/AST/Selection.cpp b/unittests/AST/Selection.cpp deleted file mode 100644 index 81b96721..00000000 --- a/unittests/AST/Selection.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "../Test.h" -#include -#include - -namespace { - -std::vector compileArgs = { - "clang++", - "-std=c++20", - "main.cpp", - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", -}; - -using namespace clice; - -TEST(clice, SelectionTree) { - foreachFile("SelectionTree", [](std::string file, llvm::StringRef content) { - auto compiler = Compiler("main.cpp", content, compileArgs); - compiler.buildAST(); - auto& fileMgr = compiler.fileMgr(); - auto entry = fileMgr.getFileRef("main.cpp"); - if(!entry) { - llvm::outs() << "Failed to get file id\n"; - std::terminate(); - } - auto& srcMgr = compiler.srcMgr(); - auto id = srcMgr.translateFile(*entry); - auto begin = srcMgr.translateLineCol(id, 7, 17); - auto end = srcMgr.translateLineCol(id, 7, 17); - SelectionTree tree(srcMgr.getFileOffset(begin), - srcMgr.getFileOffset(end), - compiler.context(), - compiler.tokBuf()); - }); -} - -} // namespace - diff --git a/unittests/Compiler/Compiler.cpp b/unittests/Compiler/Compiler.cpp index 0c6be0bd..f63a6ec0 100644 --- a/unittests/Compiler/Compiler.cpp +++ b/unittests/Compiler/Compiler.cpp @@ -24,7 +24,12 @@ int main(){ "/home/ykiko/C++/clice2/build/lib/clang/20", }; - auto info = buildAST("main.cpp", code, compileArgs); + CompliationParams params; + params.path = "main.cpp"; + params.content = code; + params.args = compileArgs; + + auto info = buildAST(params); ASSERT_TRUE(bool(info)); } @@ -57,14 +62,101 @@ int main(){ return; } - auto pch = clice::buildPCH("main.cpp", code, outpath, compileArgs); + CompliationParams params; + params.path = "main.cpp"; + params.content = code; + params.outpath = outpath; + params.args = compileArgs; + + auto pch = clice::buildPCH(params); ASSERT_TRUE(bool(pch)); - Preamble preamble; - preamble.addPCH(*pch); + params.addPCH(*pch); - auto ast = buildAST("main.cpp", code, compileArgs, &preamble); + auto ast = buildAST(params); ASSERT_TRUE(bool(ast)); } +TEST(Compiler, buildPCM) { + const char* code = R"cpp( +export module A; + +export int foo() { + return 0; +} +)cpp"; + + llvm::SmallString<128> outpath; + if(auto error = llvm::sys::fs::createTemporaryFile("main", "pcm", outpath)) { + llvm::errs() << error.message() << "\n"; + return; + } + + if(auto error = fs::remove(outpath)) { + llvm::errs() << error.message() << "\n"; + return; + } + + llvm::SmallVector compileArgs = { + "clang++", + "-std=c++20", + "main.cppm", + }; + + CompliationParams params; + params.path = "main.cppm"; + params.content = code; + params.outpath = outpath; + params.args = compileArgs; + + auto pcm = clice::buildPCM(params); + ASSERT_TRUE(bool(pcm)); + ASSERT_EQ(pcm->name, "A"); + + const char* code2 = R"cpp( +import A; + +int main(){ + foo(); + return 0; +} +)cpp"; + + compileArgs = { + "clang++", + "-std=c++20", + "main.cpp", + }; + + params.path = "main.cpp"; + params.content = code2; + params.args = compileArgs; + auto info = buildAST(params); + ASSERT_TRUE(bool(info)); +} + +TEST(Compiler, codeCompleteAt) { + const char* code = R"cpp( +export module A; +export int foo = 1; +)cpp"; + + llvm::SmallVector compileArgs = { + "clang++", + "-std=c++20", + "main.cppm", + }; + + CompliationParams params; + params.path = "main.cppm"; + params.content = code; + params.args = compileArgs; + + auto consumer = new clang::PrintingCodeCompleteConsumer({}, llvm::outs()); + auto info = codeCompleteAt(params, 3, 10, "main.cppm", consumer); + ASSERT_TRUE(bool(info)); + + /// TODO: add tests in the case of PCH, PCM and override file. +} + } // namespace diff --git a/unittests/Feature/CodeCompletion.cpp b/unittests/Feature/CodeCompletion.cpp deleted file mode 100644 index c94ee134..00000000 --- a/unittests/Feature/CodeCompletion.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "../Test.h" -#include -#include - -namespace { - -using namespace clice; - -TEST(CodeCompletion, non_self_contain) { - llvm::SmallString<64> main, header; - path::append(main, test_dir(), "CodeCompletion", "non-self-contain.cpp"); - path::append(header, test_dir(), "CodeCompletion", "non-self-contain.h"); - - std::vector compileArgs = { - "clang++", - "-std=c++20", - main.c_str(), - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - - Compiler compiler(compileArgs); - proto::Position position; - position.line = 2; - position.character = 6; - auto completionItems = feature::codeCompletion(compiler, header, position, {}); -} - -} // namespace diff --git a/unittests/Index/Tester.h b/unittests/Index/Tester.h index c26b075e..1151ff0e 100644 --- a/unittests/Index/Tester.h +++ b/unittests/Index/Tester.h @@ -34,7 +34,7 @@ void testEqual(const Loader& loader, const In& in, const Out& out) { struct IndexerTester { Annotation annotation; std::unique_ptr loader; - std::unique_ptr compiler; + std::unique_ptr compiler; FileRef mainFile; IndexerTester(llvm::StringRef source, bool json = false) : annotation(source) { @@ -46,8 +46,9 @@ struct IndexerTester { "/home/ykiko/C++/clice2/build/lib/clang/20", }; - compiler = std::make_unique("main.cpp", annotation.source(), args); - compiler->buildAST(); + /// FIXME: + // compiler = std::make_unique("main.cpp", annotation.source(), args); + // compiler->buildAST(); auto index = index::index(*compiler);