From c13048b97692ad050382678ab0c8c2da9dfc5394 Mon Sep 17 00:00:00 2001 From: ykiko Date: Sat, 14 Dec 2024 22:32:32 +0800 Subject: [PATCH] Implement checking deps logic for building PCH and PCM. (#15) --- include/Compiler/Compiler.h | 6 -- include/Server/Async.h | 4 +- include/Server/Indexer.h | 10 ++ include/Server/Scheduler.h | 40 ++++++-- include/Support/JSON.h | 10 +- src/Compiler/Compiler.cpp | 11 -- src/Server/Feature.cpp | 14 ++- src/Server/Lifestyle.cpp | 4 +- src/Server/Scheduler.cpp | 193 ++++++++++++++++++++++++++++++------ 9 files changed, 225 insertions(+), 67 deletions(-) create mode 100644 include/Server/Indexer.h diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h index 60621fc5..480a0463 100644 --- a/include/Compiler/Compiler.h +++ b/include/Compiler/Compiler.h @@ -133,8 +133,6 @@ struct PCHInfo { unsigned int size = preamble.size() - endAtStart; return {size, endAtStart}; } - - bool needUpdate(llvm::StringRef content); }; /// Build PCH from given file path and content. @@ -165,10 +163,6 @@ inherited_struct(PCMInfo, ModuleInfo) { /// Files involved in building this PCM(not include module). std::vector deps; - - bool needUpdate() { - return true; - } }; /// Build PCM from given file path and content. diff --git a/include/Server/Async.h b/include/Server/Async.h index 555e3106..93f7d995 100644 --- a/include/Server/Async.h +++ b/include/Server/Async.h @@ -330,10 +330,12 @@ private: coroutine_handle h; }; + + /// Suspend current coroutine and invoke the callback with its handle. /// Note the callback invoked before the coroutine is suspended. So it is /// -template +template > Callback> auto suspend(Callback&& callback) { struct suspend_awaiter { Callback callback; diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h new file mode 100644 index 00000000..d5c2dea8 --- /dev/null +++ b/include/Server/Indexer.h @@ -0,0 +1,10 @@ +#pragma once + +#include "Support/Support.h" + +namespace clice { + +/// Responsible for index all files, distinguish active and inactive files. +class Indexer {}; + +} // namespace clice diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h index 59c406f1..da579a9b 100644 --- a/include/Server/Scheduler.h +++ b/include/Server/Scheduler.h @@ -44,31 +44,56 @@ struct llvm::DenseMapInfo { namespace clice { +struct File { + bool isIdle = true; + + /// The current ASTInfo. + std::unique_ptr info = std::make_unique(); + + /// Coroutine handles that are waiting for the file to be updated. + std::deque> waiters; +}; + /// Responsible for manage the files and schedule the tasks. class Scheduler { private: - async::promise<> updatePCH(CompilationParams& params, class Synchronizer& sync); + /// Update the PCH for the given source file. + async::promise<> updatePCH(llvm::StringRef srcPath, llvm::StringRef content); /// Clang requires all direct and indirect dependent modules to be added during module building. /// This function adds the dependencies of the given module to the compilation parameters. /// Note: It is assumed that all dependent modules have already been built. llvm::Error addModuleDeps(CompilationParams& params, const ModuleInfo& moduleInfo) const; - async::promise<> updatePCM(llvm::StringRef name, class Synchronizer& sync); + /// Update the PCM for the given module. + async::promise<> updatePCM(llvm::StringRef moduleName, class Synchronizer& sync); + + /// Update the AST for the given source file. + async::promise<> updateAST(llvm::StringRef filename, + llvm::StringRef content, + class Synchronizer& sync); + + /// Wait for until the file is idle. + async::promise<> waitForFile(llvm::StringRef filename); + + void scheduleNext(llvm::StringRef filename); public: + /// Update the given file. async::promise<> update(llvm::StringRef filename, llvm::StringRef content, class Synchronizer& sync); + /// Execute the given action on the given file. + async::promise<> execute(llvm::StringRef filename, + llvm::unique_function action); + /// Load all Information about PCHs and PCMs from disk. - void loadFromDisk(); + void loadCache(); /// Save all Information about PCHs and PCMs to disk. /// So that we can reuse them next time. - void saveToDisk() const; - - struct File {}; + void saveCache() const; private: /// [file name] -> [PCHInfo] @@ -76,6 +101,9 @@ private: /// [module name] -> [PCMInfo] llvm::StringMap pcms; + + /// [file name] -> [File] + llvm::StringMap files; }; } // namespace clice diff --git a/include/Support/JSON.h b/include/Support/JSON.h index 336d1f75..921acfba 100644 --- a/include/Support/JSON.h +++ b/include/Support/JSON.h @@ -312,11 +312,11 @@ struct Serde { if constexpr(!std::is_empty_v) { assert(value.kind() == json::Value::Object && "Expect an object"); refl::foreach(t, [&](std::string_view name, auto&& member) { - auto v = value.getAsObject()->get(llvm::StringRef(name)); - assert(v && "Member not found"); - member = json::deserialize>( - *v, - std::forward(serdes)...); + if(auto v = value.getAsObject()->get(llvm::StringRef(name))) { + member = json::deserialize>( + *v, + std::forward(serdes)...); + } }); } return t; diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp index 7afed45c..4a3a6235 100644 --- a/src/Compiler/Compiler.cpp +++ b/src/Compiler/Compiler.cpp @@ -6,17 +6,6 @@ namespace clice { -bool PCHInfo::needUpdate(llvm::StringRef content) { - auto size = this->bounds().Size; - if(content.substr(0, size) != preamble.substr(0, size)) { - return true; - } - - /// FIXME: check timestamp of all files involved in building this PCH. - - return false; -} - namespace { auto createInvocation(CompilationParams& params) { diff --git a/src/Server/Feature.cpp b/src/Server/Feature.cpp index d1a0efd6..06a2707a 100644 --- a/src/Server/Feature.cpp +++ b/src/Server/Feature.cpp @@ -90,10 +90,11 @@ async::promise Server::onDocumentSymbol(json::Value id, async::promise Server::onSemanticTokens(json::Value id, const proto::SemanticTokensParams& params) { auto path = URI::resolve(params.textDocument.uri); - // auto tokens = co_await scheduler.schedule(path, [&](ASTInfo& compiler) { - // return feature::semanticTokens(compiler, ""); - // }); - /// async::response(std::move(id), json::serialize(tokens)); + proto::SemanticTokens result; + co_await scheduler.execute(path, [&id, &path, &result](ASTInfo& info) { + result = feature::semanticTokens(info, path); + }); + async::response(std::move(id), json::serialize(result)); co_return; } @@ -103,10 +104,7 @@ async::promise Server::onInlayHint(json::Value id, const proto::InlayHintP async::promise Server::onCodeCompletion(json::Value id, const proto::CompletionParams& params) { - auto path = URI::resolve(params.textDocument.uri); - // auto result = co_await scheduler.codeComplete(path, - // params.position.line + 1, - // params.position.character + 1); + // auto path = URI::resolve(params.textDocument.uri); // async::response(std::move(id), json::serialize(result)); co_return; } diff --git a/src/Server/Lifestyle.cpp b/src/Server/Lifestyle.cpp index 98bb1adb..8eb7f8aa 100644 --- a/src/Server/Lifestyle.cpp +++ b/src/Server/Lifestyle.cpp @@ -34,7 +34,7 @@ async::promise Server::onInitialized(const proto::InitializedParams& param json::serialize(options)); /// Load all information about PCHs and PCMs from disk. - scheduler.loadFromDisk(); + scheduler.loadCache(); co_return; } @@ -44,7 +44,7 @@ async::promise Server::onExit(const proto::None&) { async::promise Server::onShutdown(json::Value id, const proto::None&) { /// Save all information about PCHs and PCMs to disk. - scheduler.saveToDisk(); + scheduler.saveCache(); co_return; } diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp index 18dcef12..d4b1c7e9 100644 --- a/src/Server/Scheduler.cpp +++ b/src/Server/Scheduler.cpp @@ -32,22 +32,65 @@ static std::string getPCMOutPath(llvm::StringRef srcPath) { return outPath.str().str(); } -async::promise<> Scheduler::updatePCH(CompilationParams& params, class Synchronizer& sync) { - llvm::StringRef srcPath = params.srcPath; +/// Check whether the file has been modified after the given status. +static bool hasModifiedAfter(fs::file_status& src, llvm::StringRef file) { + fs::file_status status; + if(auto error = fs::status(file, status)) { + log::warn("Failed to get status of {0}, because {1}", file, error.message()); + return true; + } - auto& pch = pchs[srcPath]; - /// FIXME: judge need update here ... - if(!pch.needUpdate(params.content)) { + return status.getLastModificationTime() > src.getLastModificationTime(); +} + +async::promise<> Scheduler::updatePCH(llvm::StringRef srcPath, llvm::StringRef content) { + bool needUpdate = false; + + PCHInfo info; + + /// Check whether the PCH needs to be updated. + if(auto iter = pchs.find(srcPath); iter == pchs.end()) { + needUpdate = true; + } else { + info = iter->second; + co_await async::schedule_task([&content, &needUpdate, &info] { + /// Check whether PCH file exists. + if(!fs::exists(info.path)) { + needUpdate = true; + } + + /// Check whether the content of the PCH is consistent with the source file. + auto size = info.bounds().Size; + if(content.substr(0, size) != info.preamble.substr(0, size)) { + needUpdate = true; + } + + /// Check whether the dependent files have been modified. + fs::file_status status; + if(auto error = fs::status(info.path, status)) { + log::warn("Failed to get status of {0}, because {1}", info.path, error.message()); + needUpdate = true; + } + + for(auto& dep: info.deps) { + if(hasModifiedAfter(status, dep)) { + needUpdate = true; + } + } + }); + } + + if(!needUpdate) { log::info("PCH for {0} is already up-to-date, reuse it", srcPath); co_return; } - /// Construct the output path. + CompilationParams params; + params.content = content; + params.srcPath = srcPath; params.outPath = getPCHOutPath(srcPath); /// Build PCH. - PCHInfo info; - Tracer tracer; log::info("Building PCH for {0}", srcPath); @@ -102,11 +145,7 @@ async::promise<> Scheduler::updatePCM(llvm::StringRef moduleName, class Synchron co_return; } - auto& pcm = pcms[moduleName]; - if(!pcm.needUpdate()) { - log::info("PCM for {0} is already up-to-date, reuse it", srcPath); - } - + /// At first, scan the module to get module name and dependent modules. CompilationParams params; params.srcPath = srcPath; params.command = sync.lookup(srcPath); @@ -118,14 +157,62 @@ async::promise<> Scheduler::updatePCM(llvm::StringRef moduleName, class Synchron co_return; } - /// Build prerequired PCM. + /// If the module is an interface unit, we need to update the module map. + if(moduleInfo->isInterfaceUnit) { + sync.sync(moduleName, srcPath); + } + + /// FIXME: If two pcms have same deps, we will check the same deps twice. + /// Try to skip this by using a set to store deps. + + /// Try to update dependent PCMs. for(auto& mod: moduleInfo->mods) { co_await updatePCM(mod, sync); } - /// Build PCM. + /// All dependent PCMs are up-to-date, check whether the PCM needs to be updated. + bool needUpdate = false; + PCMInfo info; + if(auto iter = pcms.find(moduleName); iter == pcms.end()) { + needUpdate = true; + } else { + info = iter->second; + needUpdate = co_await async::schedule_task([&info] { + /// Check whether PCM file exists. + if(!fs::exists(info.path)) { + return true; + } + + fs::file_status status; + if(auto error = fs::status(info.path, status)) { + log::warn("Failed to get status of {0}, because {1}", info.path, error.message()); + return true; + } + + /// Check whether the source file has been modified. + if(hasModifiedAfter(status, info.srcPath)) { + return true; + } + + /// Check whether the dependent files have been modified. + for(auto& dep: info.deps) { + if(hasModifiedAfter(status, dep)) { + return true; + } + } + + return false; + }); + } + + if(!needUpdate) { + log::info("PCM for {0} is already up-to-date, reuse it", srcPath); + co_return; + } + + /// Build PCM. Tracer tracer; log::info("Building PCM for {0}", srcPath); @@ -156,9 +243,9 @@ async::promise<> Scheduler::updatePCM(llvm::StringRef moduleName, class Synchron } } -async::promise<> Scheduler::update(llvm::StringRef filename, - llvm::StringRef content, - class Synchronizer& sync) { +async::promise<> Scheduler::updateAST(llvm::StringRef filename, + llvm::StringRef content, + class Synchronizer& sync) { CompilationParams params; params.content = content; params.srcPath = filename; @@ -171,10 +258,10 @@ async::promise<> Scheduler::update(llvm::StringRef filename, } if(moduleInfo->name.empty() && moduleInfo->mods.empty()) { - co_await updatePCH(params, sync); - params.bounds.reset(); + co_await updatePCH(filename, content); params.addPCH(pchs[params.srcPath]); } else { + /// FIXME: it is possible that building PCM parallelly, if they have same deps. for(auto& mod: moduleInfo->mods) { co_await updatePCM(mod, sync); } @@ -186,7 +273,7 @@ async::promise<> Scheduler::update(llvm::StringRef filename, } /// Build AST. - ASTInfo info; + ASTInfo& info = *files[filename].info; Tracer tracer; @@ -203,11 +290,10 @@ async::promise<> Scheduler::update(llvm::StringRef filename, }); /// Build AST successfully. - log::info("AST for {0} is up-to-date, elapsed {1}", filename, tracer.duration()); } -void Scheduler::loadFromDisk() { +void Scheduler::loadCache() { llvm::SmallString<128> fileName; path::append(fileName, config::frontend().cache_directory, "cache.json"); @@ -229,14 +315,14 @@ void Scheduler::loadFromDisk() { return; } - if(auto pchArray = object->getArray("pch")) { + if(auto pchArray = object->getArray("PCH")) { for(auto& value: *pchArray) { auto pch = json::deserialize(value); pchs[pch.srcPath] = std::move(pch); } } - if(auto pcmArray = object->getArray("pcm")) { + if(auto pcmArray = object->getArray("PCM")) { for(auto& value: *pcmArray) { auto pcm = json::deserialize(value); pcms[pcm.name] = std::move(pcm); @@ -246,20 +332,20 @@ void Scheduler::loadFromDisk() { log::info("Cache loaded from {0}", fileName); } -void Scheduler::saveToDisk() const { +void Scheduler::saveCache() const { json::Object result; json::Array pchArray; for(auto& [name, pch]: pchs) { pchArray.emplace_back(json::serialize(pch)); } - result.try_emplace("pch", std::move(pchArray)); + result.try_emplace("PCH", std::move(pchArray)); json::Array pcmArray; for(auto& [name, pcm]: pcms) { pcmArray.emplace_back(json::serialize(pcm)); } - result.try_emplace("pcm", std::move(pcmArray)); + result.try_emplace("PCM", std::move(pcmArray)); llvm::SmallString<128> fileName; path::append(fileName, config::frontend().cache_directory, "cache.json"); @@ -276,4 +362,55 @@ void Scheduler::saveToDisk() const { log::info("Cache saved to {0}", fileName); } +async::promise<> Scheduler::waitForFile(llvm::StringRef filename) { + File* file = &files[filename]; + + if(!file->isIdle) { + co_await async::suspend([file](auto handle) { file->waiters.push_back(handle); }); + } +} + +void Scheduler::scheduleNext(llvm::StringRef filename) { + auto file = &files[filename]; + if(file->waiters.empty()) { + file->isIdle = true; + } else { + /// If waiters exist, wake up the first waiter. + auto handle = file->waiters.front(); + file->waiters.pop_front(); + async::schedule(handle); + } +} + +async::promise<> Scheduler::update(llvm::StringRef filename, + llvm::StringRef content, + class Synchronizer& sync) { + co_await waitForFile(filename); + + /// files may be modified during the action. + auto file = &files[filename]; + file->isIdle = false; + + /// If the file is idle, execute the action directly. + assert(file->info && "ASTInfo is required"); + co_await updateAST(filename, content, sync); + + scheduleNext(filename); +} + +async::promise<> Scheduler::execute(llvm::StringRef filename, + llvm::unique_function action) { + co_await waitForFile(filename); + + /// files may be modified during the action. + auto file = &files[filename]; + assert(file->info && "ASTInfo is required"); + + /// If the file is idle, execute the action directly. + file->isIdle = false; + co_await async::schedule_task([&action, file] { action(*file->info); }); + + scheduleNext(filename); +} + } // namespace clice