diff --git a/include/Server/Async.h b/include/Server/Async.h index 29e7cef8..4c2179fa 100644 --- a/include/Server/Async.h +++ b/include/Server/Async.h @@ -282,7 +282,7 @@ struct promise_type : result { } }; -template +template class promise { public: using promise_type = async::promise_type; @@ -309,5 +309,27 @@ private: coroutine_handle h; }; +/// Suspend current coroutine and invoke the callback with its handle. +/// Note the callback invoked before the coroutine is suspended. So it is +/// +template +auto suspend(Callback&& callback) { + struct suspend_awaiter { + Callback callback; + + bool await_ready() noexcept { + return false; + } + + void await_suspend(std::coroutine_handle<> handle) noexcept { + callback(handle); + } + + void await_resume() noexcept {} + }; + + return suspend_awaiter{std::forward(callback)}; +} + } // namespace clice::async diff --git a/include/Server/Scheduler.h b/include/Server/Scheduler.h new file mode 100644 index 00000000..a5725253 --- /dev/null +++ b/include/Server/Scheduler.h @@ -0,0 +1,121 @@ +#pragma once + +#include + +#include "Async.h" +#include "llvm/ADT/StringMap.h" + +namespace clice { + +class Compiler; + +/// Information of building precompiled header. +struct PCH { + /// The path of this PCH. + std::string path; + /// The source file path. + std::string sourcePath; + /// The header part of source file used to build this PCH. + std::string preamble; + /// The arguments used to build this PCH. + std::string arguments; + /// All files involved in building this PCH(excluding the source file). + std::vector deps; + + /// FIXME: use asyncronous file system API. + bool needUpdate(llvm::StringRef sourceContent) { + /// Check whether the header part changed. + if(sourceContent.substr(0, preamble.size()) != preamble) { + return true; + } + + /// Check timestamp of all files involved in building this PCH. + fs::file_status build; + if(auto error = fs::status(path, build)) { + llvm::errs() << "Error: " << error.message() << "\n"; + std::terminate(); + } + + /// TODO: check whether deps changed through comparing timestamps. + return false; + } + + void apply(Compiler& compiler) const; +}; + +/// Information of building precompiled module. +struct PCM {}; + +struct File; + +struct Task { + /// Whether this task is a build task. + bool isBuild = false; + /// The coroutine handle of this task. + std::coroutine_handle<> waiting; +}; + +struct File { + bool isIdle = true; + + /// The compiler instance of this file. + std::unique_ptr compiler; + + std::deque waitings; +}; + +class Scheduler { +public: + async::promise updatePCH(llvm::StringRef path, + llvm::StringRef content, + llvm::ArrayRef args); + + async::promise updatePCM() { + co_return; + } + + async::promise buildAST(llvm::StringRef path, llvm::StringRef content); + + async::promise add(llvm::StringRef path, llvm::StringRef content); + + async::promise update(llvm::StringRef path, llvm::StringRef content); + + async::promise save(llvm::StringRef path); + + async::promise close(llvm::StringRef path); + + /// Schedule a task for a file. If the file is building, the task will be + /// appended to the task list of the file and wait for the building to finish. + /// Otherwise, the task will be executed immediately. + template + auto schedule(llvm::StringRef path, Task&& task) + -> async::promise()))> { + auto& file = files[path]; + if(!file.isIdle) { + co_await async::suspend([&](auto handle) { + file.waitings.push_back({.isBuild = false, .waiting = handle}); + }); + } + + file.isIdle = false; + auto& compiler = *file.compiler; + + auto result = co_await async::schedule_task([&task, &compiler] { return task(compiler); }); + + if(!file.waitings.empty()) { + auto task = std::move(file.waitings.front()); + async::schedule(task.waiting); + file.waitings.pop_front(); + } + + file.isIdle = true; + + co_return result; + } + +private: + llvm::StringMap pchs; + llvm::StringMap files; +}; + +} // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index a577ca0a..2a73c9f5 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -3,6 +3,8 @@ #include "Async.h" #include "Config.h" #include "Logger.h" +#include "Scheduler.h" +#include "Basic/URI.h" #include "Basic/Document.h" #include "Compiler/Compiler.h" #include "Support/JSON.h" @@ -151,9 +153,6 @@ private: }); } - llvm::StringMap requests; - llvm::StringMap notifications; - private: /// ============================================================================ /// Lifestyle Message @@ -245,81 +244,9 @@ private: const proto::DocumentRangeFormattingParams& params); private: - /// Information of building precompiled header. - struct PCH { - /// The path of this PCH. - std::string path; - /// The source file path. - std::string sourcePath; - /// The header part of source file used to build this PCH. - std::string preamble; - /// The arguments used to build this PCH. - std::string arguments; - /// All files involved in building this PCH(excluding the source file). - std::vector deps; - - /// FIXME: use asyncronous file system API. - bool needUpdate(llvm::StringRef sourceContent) { - /// Check whether the header part changed. - if(sourceContent.substr(0, preamble.size()) != preamble) { - return true; - } - - /// Check timestamp of all files involved in building this PCH. - fs::file_status build; - if(auto error = fs::status(path, build)) { - llvm::errs() << "Error: " << error.message() << "\n"; - std::terminate(); - } - - /// TODO: check whether deps changed through comparing timestamps. - return false; - } - }; - - /// Information of building precompiled module. - struct PCM {}; - - async::promise updatePCH(llvm::StringRef filepath, - llvm::StringRef content, - llvm::ArrayRef args); - - async::promise updatePCM() { - co_return; - } - - async::promise buildAST(llvm::StringRef filepath, llvm::StringRef content); - - struct TranslationUnit { - enum class State { - Building, - Ready, - }; - - enum class TaskKind { - Build, - Consume, - }; - - struct Task { - TaskKind kind; - llvm::unique_function(Compiler&)> request; - }; - - State state; - std::unique_ptr compiler; - std::vector tasks; - }; - - /// Schedule a task for a file. If the file is building, the task will be - /// appended to the task list of the file and wait for the building to finish. - /// Otherwise, the task will be executed immediately. - async::promise schedule(llvm::StringRef path, - llvm::unique_function(Compiler&)> callback); - -private: - llvm::StringMap pchs; - llvm::StringMap units; + Scheduler scheduler; + llvm::StringMap requests; + llvm::StringMap notifications; }; } // namespace clice diff --git a/src/Feature/SemanticTokens.cpp b/src/Feature/SemanticTokens.cpp index 4f668005..c2584913 100644 --- a/src/Feature/SemanticTokens.cpp +++ b/src/Feature/SemanticTokens.cpp @@ -618,9 +618,9 @@ public: auto spelledTokens = tokBuf.spelledTokens(mainFileID); for(auto& token: spelledTokens) { proto::SemanticTokenType type = proto::SemanticTokenType::Invalid; - llvm::outs() << clang::tok::getTokenName(token.kind()) << " " - << pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts()) - << "\n"; + // llvm::outs() << clang::tok::getTokenName(token.kind()) << " " + // << pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts()) + // << "\n"; auto kind = token.kind(); switch(kind) { diff --git a/src/Server/Scheduler.cpp b/src/Server/Scheduler.cpp new file mode 100644 index 00000000..a518e7b3 --- /dev/null +++ b/src/Server/Scheduler.cpp @@ -0,0 +1,116 @@ +#include "Server/Scheduler.h" +#include "Server/Server.h" + +namespace clice { + +void PCH::apply(Compiler& compiler) const { + bool endAtStart = preamble.ends_with('@'); + auto size = preamble.size() - endAtStart; + + if(size != 0) { + compiler.applyPCH(path, size, endAtStart); + } +} + +async::promise Scheduler::updatePCH(llvm::StringRef filepath, + llvm::StringRef content, + llvm::ArrayRef args) { + log::info("Start building PCH for {0}", filepath.str()); + std::string outpath = "/home/ykiko/C++/clice2/build/cache/xxx.pch"; + + clang::PreambleBounds bounds = {0, 0}; + co_await async::schedule_task([&] { + Compiler compiler(filepath, content, args); + bounds = clang::Lexer::ComputePreamble(content, {}, false); + if(bounds.Size != 0) { + compiler.generatePCH(outpath, bounds.Size, bounds.PreambleEndsAtStartOfLine); + } + }); + + log::info("Build PCH success"); + + auto preamble = content.substr(0, bounds.Size).str(); + if(bounds.PreambleEndsAtStartOfLine) { + preamble.append("@"); + } + + pchs.try_emplace(filepath, PCH{.path = outpath, .preamble = std::move(preamble)}); + co_return; +} + +async::promise Scheduler::buildAST(llvm::StringRef filepath, llvm::StringRef content) { + llvm::SmallString<128> path = filepath; + + auto [iter, success] = files.try_emplace(filepath); + if(!success && !iter->second.isIdle) { + /// If the file is already existed and is building, append the task to the waiting list. + co_await async::suspend([&](auto handle) { + iter->second.waitings.emplace_back(Task{ + .isBuild = true, + .waiting = handle, + }); + }); + } + + files[filepath].isIdle = false; + + /// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1 + llvm::SmallVector args = { + "clang++", + "-std=c++20", + path.c_str(), + "-resource-dir", + "/home/ykiko/C++/clice2/build/lib/clang/20", + }; + + /// through arguments to judge is it a module. + bool isModule = false; + co_await (isModule ? updatePCM() : updatePCH(filepath, content, args)); + + log::info("Start building AST for {0}", filepath.str()); + + auto task = [&path, &content, &args, pch = pchs.at(filepath)] { + /// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be + /// Invalid Because other changed the `pchs` map. We also cannot to retrieve the `pch` from + /// the `pchs` map in this task, beacuse it is called in thread pool which will result in + /// data race. So temporarily copy the `pch` here. There must be a better way to solve this + /// problem. + std::unique_ptr compiler = std::make_unique(path, content, args); + pch.apply(*compiler); + compiler->buildAST(); + return compiler; + }; + + auto compiler = co_await async::schedule_task(std::move(task)); + + auto& file = files[path]; + file.compiler = std::move(compiler); + + log::info("Build AST success"); + + if(!file.waitings.empty()) { + auto task = std::move(file.waitings.front()); + async::schedule(task.waiting); + file.waitings.pop_front(); + } +} + +async::promise Scheduler::add(llvm::StringRef path, llvm::StringRef content) { + co_await buildAST(path, content); + co_return; +} + +async::promise Scheduler::update(llvm::StringRef path, llvm::StringRef content) { + co_await buildAST(path, content); + co_return; +} + +async::promise Scheduler::save(llvm::StringRef path) { + co_return; +} + +async::promise Scheduler::close(llvm::StringRef path) { + co_return; +} + +} // namespace clice diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 884e68eb..98f4e854 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -1,5 +1,4 @@ #include "Server/Server.h" -#include "Basic/URI.h" namespace clice { @@ -94,24 +93,24 @@ async::promise Server::onDidOpen(const proto::DidOpenTextDocumentParams& p auto path = URI::resolve(params.textDocument.uri); llvm::StringRef content = params.textDocument.text; - co_await buildAST(path, content); - - co_return; + co_await scheduler.add(path, content); } async::promise Server::onDidChange(const proto::DidChangeTextDocumentParams& document) { auto path = URI::resolve(document.textDocument.uri); llvm::StringRef content = document.contentChanges[0].text; - - co_await buildAST(path, content); - co_return; + co_await scheduler.update(path, content); } async::promise Server::onDidSave(const proto::DidSaveTextDocumentParams& document) { + auto path = URI::resolve(document.textDocument.uri); + co_await scheduler.save(path); co_return; } async::promise Server::onDidClose(const proto::DidCloseTextDocumentParams& document) { + auto path = URI::resolve(document.textDocument.uri); + co_await scheduler.close(path); co_return; } @@ -203,12 +202,10 @@ async::promise Server::onDocumentSymbol(json::Value id, async::promise Server::onSemanticTokens(json::Value id, const proto::SemanticTokensParams& params) { auto path = URI::resolve(params.textDocument.uri); - auto task = [this, id = std::move(id)](Compiler& compiler) -> async::promise { - auto tokens = feature::semanticTokens(compiler, ""); - async::write(std::move(id), json::serialize(tokens)); - co_return; - }; - co_await schedule(path, std::move(task)); + auto tokens = co_await scheduler.schedule(path, [&](Compiler& compiler) { + return feature::semanticTokens(compiler, ""); + }); + async::write(std::move(id), json::serialize(tokens)); co_return; } @@ -240,87 +237,4 @@ async::promise Server::onRangeFormatting(json::Value id, co_return; } -async::promise Server::updatePCH(llvm::StringRef filepath, - llvm::StringRef content, - llvm::ArrayRef args) { - log::info("Start building PCH for {0}", filepath.str()); - clang::PreambleBounds bounds = {0, 0}; - std::string outpath = "/home/ykiko/C++/clice2/build/cache/xxx.pch"; - co_await async::schedule_task([&] { - Compiler compiler(filepath, content, args); - bounds = clang::Lexer::ComputePreamble(content, {}, false); - if(bounds.Size != 0) { - compiler.generatePCH(outpath, bounds.Size, bounds.PreambleEndsAtStartOfLine); - } - }); - log::info("Build PCH success"); - - auto preamble2 = content.substr(0, bounds.Size).str(); - if(bounds.PreambleEndsAtStartOfLine) { - preamble2.append("@"); - } - - pchs.try_emplace(filepath, PCH{.path = outpath, .preamble = std::move(preamble2)}); - co_return; -} - -async::promise Server::buildAST(llvm::StringRef filepath, llvm::StringRef content) { - llvm::SmallString<128> path = filepath; - - /// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1 - llvm::SmallVector args = { - "clang++", - "-std=c++20", - path.c_str(), - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; - - /// through arguments to judge is it a module. - bool isModule = false; - co_await (isModule ? updatePCM() : updatePCH(filepath, content, args)); - - auto& pch = pchs.at(filepath); - - log::info("Start building AST for {0}", filepath.str()); - auto compiler = co_await async::schedule_task([&] { - std::uint32_t boundSize = pch.preamble.size(); - bool endAtStart = false; - if(pch.preamble.back() == '@') { - boundSize -= 1; - endAtStart = true; - } - std::unique_ptr compiler = std::make_unique(path, content, args); - if(boundSize != 0) { - compiler->applyPCH(pch.path, boundSize, endAtStart); - } - compiler->buildAST(); - return compiler; - }); - log::info("Build AST success"); - - auto& unit = units[filepath]; - unit.state = TranslationUnit::State::Ready; - unit.compiler = std::move(compiler); - - for(auto& task: unit.tasks) { - co_await task.request(*unit.compiler); - if(task.kind == TranslationUnit::TaskKind::Build) { - break; - } - } -} - -async::promise - Server::schedule(llvm::StringRef path, - llvm::unique_function(Compiler&)> request) { - auto& unit = units[path]; - if(unit.state == TranslationUnit::State::Building) { - unit.tasks.emplace_back(TranslationUnit::TaskKind::Consume, std::move(request)); - } else { - co_await request(*units[path].compiler); - } - - co_return; -} } // namespace clice