From d6b0c424c2f83719a2d61fc9fec5b853be330eb6 Mon Sep 17 00:00:00 2001 From: ykiko Date: Sun, 1 Dec 2024 14:52:24 +0800 Subject: [PATCH] Update `Server`. --- include/Compiler/Compiler.h | 68 +++++++++++---------------------- include/Server/Config.h | 2 + include/Server/Scheduler.h | 41 +------------------- include/Support/Format.h | 17 ++++++++- src/Compiler/Compiler.cpp | 51 ++++++++++++++++++------- src/Server/Config.cpp | 4 ++ src/Server/Scheduler.cpp | 52 +++++++++++++------------ unittests/Compiler/Compiler.cpp | 14 ++++--- 8 files changed, 120 insertions(+), 129 deletions(-) diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h index 2fd76e1b..92689d5b 100644 --- a/include/Compiler/Compiler.h +++ b/include/Compiler/Compiler.h @@ -7,19 +7,15 @@ namespace clice { /// All information about AST. -struct ASTInfo { - std::unique_ptr action; - std::unique_ptr instance; - std::unique_ptr tokBuf_; - std::unique_ptr resolver_; - +class ASTInfo { +public: ASTInfo() = default; ASTInfo(std::unique_ptr action, std::unique_ptr instance, std::unique_ptr tokBuf) : action(std::move(action)), instance(std::move(instance)), tokBuf_(std::move(tokBuf)) { - resolver_ = std::make_unique(instance->getSema()); + resolver_ = std::make_unique(this->instance->getSema()); } ASTInfo(const ASTInfo&) = delete; @@ -61,9 +57,15 @@ struct ASTInfo { TemplateResolver& resolver() { return *resolver_; } + +private: + std::unique_ptr action; + std::unique_ptr instance; + std::unique_ptr tokBuf_; + std::unique_ptr resolver_; }; -struct PCHInfo : ASTInfo { +struct PCHInfo { /// PCM file path. std::string path; /// Source file path. @@ -73,58 +75,30 @@ struct PCHInfo : ASTInfo { /// Files involved in building this PCM. std::vector deps; - PCHInfo(ASTInfo info, - llvm::StringRef path, - llvm::StringRef content, - llvm::StringRef mainpath, - clang::PreambleBounds bounds) : - ASTInfo(std::move(info)), path(path), mainpath(mainpath) { - - preamble = content.substr(0, bounds.Size).str(); - if(bounds.PreambleEndsAtStartOfLine) { - preamble.append("@"); - } - } - clang::PreambleBounds bounds() const { /// We use '@' to mark the end of the preamble. bool endAtStart = preamble.ends_with('@'); unsigned int size = preamble.size() - endAtStart; return {size, endAtStart}; } + + bool needUpdate(llvm::StringRef content); }; -struct PCMInfo : ASTInfo { +struct PCMInfo { /// PCM file path. std::string path; /// Module name. std::string name; - PCMInfo(ASTInfo info, llvm::StringRef path) : ASTInfo(std::move(info)), path(path) { - name = context().getCurrentNamedModule()->Name; - } -}; - -/// Information about reuse PCH or PCM. This should be placed in stack. -struct Preamble { - /// Information about reuse PCH. - std::string pch; - clang::PreambleBounds bounds = {0, false}; - - /// Information about reuse PCM(name, path). - llvm::SmallVector> pcms; - - void addPCH(const PCHInfo& info) { - pch = info.path; - bounds = info.bounds(); - } + bool needUpdate(); }; struct CompliationParams { - llvm::StringRef path; llvm::StringRef content; - llvm::StringRef outpath; - llvm::StringRef mainpath; + llvm::SmallString<128> path; + llvm::SmallString<128> outpath; + llvm::SmallString<128> mainpath; llvm::ArrayRef args; /// Information about reuse PCH. @@ -138,6 +112,10 @@ struct CompliationParams { pch = info.path; bounds = info.bounds(); } + + void addPCM(const PCMInfo& info) { + pcms.emplace_back(info.name, info.path); + } }; /// Build AST from given file path and content. If pch or pcm provided, apply them to the compiler. @@ -145,9 +123,9 @@ struct CompliationParams { /// their reusability and update in time. llvm::Expected buildAST(CompliationParams& params); -llvm::Expected buildPCH(CompliationParams& params); +llvm::Expected buildPCH(CompliationParams& params, PCHInfo& out); -llvm::Expected buildPCM(CompliationParams& params); +llvm::Expected buildPCM(CompliationParams& params, PCMInfo& out); llvm::Expected codeCompleteAt(CompliationParams& params, uint32_t line, diff --git a/include/Server/Config.h b/include/Server/Config.h index 9e6405ba..5ab9c90d 100644 --- a/include/Server/Config.h +++ b/include/Server/Config.h @@ -29,6 +29,8 @@ struct FrontendOption { std::string resource_dictionary = "${binary}/../lib/clang/${llvm_version}"; }; +llvm::StringRef workplace(); + const ServerOption& server(); const FrontendOption& frontend(); diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index 4cde983c..ec81361a 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -10,45 +10,6 @@ namespace clice { class ASTInfo; -/// Information of building precompiled header. -struct PCH { - /// The path of this PCH. - std::string path; - /// The source file path. - std::string sourcePath; - /// The header part of source file used to build this PCH. - std::string preamble; - /// The arguments used to build this PCH. - std::string arguments; - /// All files involved in building this PCH(excluding the source file). - std::vector deps; - - uint32_t size() const { - return preamble.size() - preamble.ends_with('@'); - } - - /// FIXME: use asyncronous file system API. - bool needUpdate(llvm::StringRef sourceContent) { - /// Check whether the header part changed. - if(sourceContent.substr(0, size()) != preamble.substr(0, size())) { - return true; - } - - /// Check timestamp of all files involved in building this PCH. - // fs::file_status build; - // if(auto error = fs::status(path, build)) { - // llvm::errs() << "Error: " << error.message() << "\n"; - // std::terminate(); - // } - - /// TODO: check whether deps changed through comparing timestamps. - return false; - } -}; - -/// Information of building precompiled module. -struct PCM {}; - struct File; struct Task { @@ -117,7 +78,7 @@ public: } private: - llvm::StringMap pchs; + llvm::StringMap pchs; llvm::StringMap files; }; diff --git a/include/Support/Format.h b/include/Support/Format.h index 9c490690..ef10975a 100644 --- a/include/Support/Format.h +++ b/include/Support/Format.h @@ -39,6 +39,21 @@ struct std::formatter : std::formatter { } }; +template +struct std::formatter> : std::formatter { + using Base = std::formatter; + + template + constexpr auto parse(ParseContext& ctx) { + return Base::parse(ctx); + } + + template + auto format(const llvm::SmallString& s, FormatContext& ctx) const { + return Base::format(llvm::StringRef(s), ctx); + } +}; + template <> struct std::formatter : std::formatter { using Base = std::formatter; @@ -51,7 +66,7 @@ struct std::formatter : std::formatter { template auto format(const clice::json::Value& value, FormatContext& ctx) const { llvm::SmallString<128> buffer; - llvm::raw_svector_ostream os(buffer); + llvm::raw_svector_ostream os{buffer}; os << value; return Base::format(buffer, ctx); } diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp index cc308f36..ba283db4 100644 --- a/src/Compiler/Compiler.cpp +++ b/src/Compiler/Compiler.cpp @@ -4,7 +4,20 @@ namespace clice { -static void adjustInvocation(clang::CompilerInvocation& invocation) { +bool PCHInfo::needUpdate(llvm::StringRef content) { + auto size = this->bounds().Size; + if(content.substr(0, size) != preamble.substr(0, size)) { + return true; + } + + /// FIXME: check timestamp of all files involved in building this PCH. + + return false; +} + +namespace { + +void adjustInvocation(clang::CompilerInvocation& invocation) { auto& frontOpts = invocation.getFrontendOpts(); frontOpts.DisableFree = false; @@ -15,7 +28,7 @@ static void adjustInvocation(clang::CompilerInvocation& invocation) { // FIXME: add more. } -static auto createInstance(llvm::ArrayRef args) { +auto createInstance(llvm::ArrayRef args) { auto instance = std::make_unique(); /// TODO: Figure out `CreateInvocationOptions`. @@ -33,7 +46,7 @@ static auto createInstance(llvm::ArrayRef args) { return instance; } -static void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) { +void applyPreamble(clang::CompilerInstance& instance, CompliationParams& params) { auto& PPOpts = instance.getPreprocessorOpts(); auto& pch = params.pch; auto& bounds = params.bounds; @@ -52,9 +65,9 @@ static void applyPreamble(clang::CompilerInstance& instance, CompliationParams& } } -static llvm::Expected ExecuteAction(std::unique_ptr instance, - clang::frontend::ActionKind kind, - bool collectPP = true) { +llvm::Expected ExecuteAction(std::unique_ptr instance, + clang::frontend::ActionKind kind, + bool collectPP = true) { std::unique_ptr action; if(kind == clang::frontend::ActionKind::ParseSyntaxOnly) { action = std::make_unique(); @@ -100,6 +113,8 @@ static llvm::Expected ExecuteAction(std::unique_ptr buildAST(CompliationParams& params) { auto instance = createInstance(params.args); @@ -111,11 +126,11 @@ llvm::Expected buildAST(CompliationParams& params) { return ExecuteAction(std::move(instance), clang::frontend::ActionKind::ParseSyntaxOnly); } -llvm::Expected buildPCH(CompliationParams& params) { +llvm::Expected buildPCH(CompliationParams& params, PCHInfo& out) { auto instance = createInstance(params.args); clang::PreambleBounds bounds = {0, false}; - if(params.mainpath == params.path) { + if(params.mainpath.empty() || params.mainpath == params.path) { /// If mainpath is equal to path, just tokenize the content to get preamble bounds. bounds = clang::Lexer::ComputePreamble(params.content, {}, false); } else { @@ -125,7 +140,7 @@ llvm::Expected buildPCH(CompliationParams& params) { } /// Set options to generate PCH. - instance->getFrontendOpts().OutputFile = params.outpath; + instance->getFrontendOpts().OutputFile = params.outpath.str(); instance->getFrontendOpts().ProgramAction = clang::frontend::GeneratePCH; instance->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false}; instance->getPreprocessorOpts().GeneratePreamble = true; @@ -135,17 +150,25 @@ llvm::Expected buildPCH(CompliationParams& params) { instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); if(auto info = ExecuteAction(std::move(instance), clang::frontend::ActionKind::GeneratePCH)) { - return PCHInfo(std::move(*info), params.outpath, params.content, params.mainpath, bounds); + out.path = params.outpath.str(); + out.mainpath = params.mainpath.str(); + out.preamble = params.content.substr(0, bounds.Size).str(); + out.preamble = params.content.substr(0, bounds.Size).str(); + if(bounds.PreambleEndsAtStartOfLine) { + out.preamble.append("@"); + } + + return std::move(*info); } else { return info.takeError(); } } -llvm::Expected buildPCM(CompliationParams& params) { +llvm::Expected buildPCM(CompliationParams& params, PCMInfo& out) { auto instance = createInstance(params.args); /// Set options to generate PCM. - instance->getFrontendOpts().OutputFile = params.outpath; + instance->getFrontendOpts().OutputFile = params.outpath.str(); instance->getFrontendOpts().ProgramAction = clang::frontend::GenerateReducedModuleInterface; auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content); @@ -155,7 +178,9 @@ llvm::Expected buildPCM(CompliationParams& params) { if(auto info = ExecuteAction(std::move(instance), clang::frontend::ActionKind::GenerateReducedModuleInterface)) { - return PCMInfo(std::move(*info), params.outpath); + out.path = params.outpath.str(); + out.name = info->context().getCurrentNamedModule()->Name; + return std::move(*info); } else { return info.takeError(); } diff --git a/src/Server/Config.cpp b/src/Server/Config.cpp index 8dd338d3..b9b514ee 100644 --- a/src/Server/Config.cpp +++ b/src/Server/Config.cpp @@ -107,6 +107,10 @@ void init(std::string_view workplace) { return; } +llvm::StringRef workplace() { + return predefined["workplace"]; +} + const ServerOption& server() { return config.server; } diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index 2d9ed3ec..7d9225ad 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -1,3 +1,4 @@ +#include "Server/Config.h" #include "Server/Scheduler.h" #include "Server/Server.h" @@ -15,47 +16,50 @@ struct Tracer { async::promise Scheduler::updatePCH(llvm::StringRef filepath, llvm::StringRef content, llvm::ArrayRef args) { - - std::string outpath = "/home/ykiko/C++/clice2/build/cache/xxx.pch"; - auto [iter, success] = pchs.try_emplace(filepath); if(success || iter->second.needUpdate(content)) { - log::info("Start building PCH for {0}", filepath.str()); - Tracer tracer; - clang::PreambleBounds bounds = {0, 0}; + CompliationParams params; params.path = filepath; params.content = content; params.args = args; + params.outpath = filepath; + path::replace_path_prefix(params.outpath, + config::workplace(), + config::frontend().cache_directory); + path::replace_extension(params.outpath, ".pch"); + + log::info("Start building PCH for {0} at {1}", params.path, params.outpath); + + PCHInfo pch; co_await async::schedule_task([&] { - bounds = clang::Lexer::ComputePreamble(content, {}, false); - if(bounds.Size != 0) { - auto pch = buildPCH(params); - if(!pch) { - log::fatal("Failed to build PCH for {0}", filepath.str()); + auto dir = path::parent_path(params.outpath); + if(!fs::exists(dir)) { + if(auto error = fs::create_directories(dir)) { + log::fatal("Failed to create directory {0}, because {1}, build PCH stopped", + dir, + error.message()); return; } } + + if(auto info = buildPCH(params, pch); !info) { + log::fatal("Failed to build PCH for {0}, because {1}", + filepath.str(), + info.takeError()); + return; + } }); - auto preamble = content.substr(0, bounds.Size).str(); - if(bounds.PreambleEndsAtStartOfLine) { - preamble.append("@"); - } - - pchs[filepath] = PCH{ - .path = outpath, - .preamble = preamble, - .deps = {}, - }; - log::info("PCH for {0} is up-to-date, elapsed {1}ms", filepath.str(), tracer.duration().count()); + + pchs[filepath] = std::move(pch); } else { - log::info("Reuse PCH from {0}", filepath.str()); + log::info("Reuse PCH for {0} from {1}", filepath.str(), iter->second.path); } co_return; } @@ -96,7 +100,7 @@ async::promise Scheduler::buildAST(llvm::StringRef filepath, llvm::StringR params.path = path; params.content = content; params.args = args; - // params.addPCH(pchs.at(filepath)); + params.addPCH(pchs.at(filepath)); auto task = [&] { /// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be diff --git a/unittests/Compiler/Compiler.cpp b/unittests/Compiler/Compiler.cpp index f63a6ec0..4ab49d22 100644 --- a/unittests/Compiler/Compiler.cpp +++ b/unittests/Compiler/Compiler.cpp @@ -68,10 +68,10 @@ int main(){ params.outpath = outpath; params.args = compileArgs; - auto pch = clice::buildPCH(params); - ASSERT_TRUE(bool(pch)); + PCHInfo pch; + ASSERT_TRUE(bool(clice::buildPCH(params, pch))); - params.addPCH(*pch); + params.addPCH(pch); auto ast = buildAST(params); ASSERT_TRUE(bool(ast)); @@ -109,9 +109,9 @@ export int foo() { params.outpath = outpath; params.args = compileArgs; - auto pcm = clice::buildPCM(params); - ASSERT_TRUE(bool(pcm)); - ASSERT_EQ(pcm->name, "A"); + PCMInfo pcm; + ASSERT_TRUE(bool(clice::buildPCM(params, pcm))); + ASSERT_EQ(pcm.name, "A"); const char* code2 = R"cpp( import A; @@ -131,6 +131,8 @@ int main(){ params.path = "main.cpp"; params.content = code2; params.args = compileArgs; + params.addPCM(pcm); + auto info = buildAST(params); ASSERT_TRUE(bool(info)); }