caio/clice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Scheduler.

Browse Source
This commit is contained in:
ykiko
2024-11-28 16:06:15 +08:00
parent 939dbc3dc5
commit c240c7e7bc
6 changed files with 278 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
include/Server/Async.h
View File

@@ -282,7 +282,7 @@ struct promise_type : result<T> {
}
};
template <typename T>
template <typename T = void>
class promise {
public:
using promise_type = async::promise_type<T>;
@@ -309,5 +309,27 @@ private:
coroutine_handle h;
};
/// Suspend current coroutine and invoke the callback with its handle.
/// Note the callback invoked before the coroutine is suspended. So it is
///
template <typename Callback>
auto suspend(Callback&& callback) {
struct suspend_awaiter {
Callback callback;
bool await_ready() noexcept {
return false;
}
void await_suspend(std::coroutine_handle<> handle) noexcept {
callback(handle);
}
void await_resume() noexcept {}
};
return suspend_awaiter{std::forward<Callback>(callback)};
}
} // namespace clice::async

121
include/Server/Scheduler.h Normal file
View File

@@ -0,0 +1,121 @@
#pragma once
#include <deque>
#include "Async.h"
#include "llvm/ADT/StringMap.h"
namespace clice {
class Compiler;
/// Information of building precompiled header.
struct PCH {
/// The path of this PCH.
std::string path;
/// The source file path.
std::string sourcePath;
/// The header part of source file used to build this PCH.
std::string preamble;
/// The arguments used to build this PCH.
std::string arguments;
/// All files involved in building this PCH(excluding the source file).
std::vector<std::string> deps;
/// FIXME: use asyncronous file system API.
bool needUpdate(llvm::StringRef sourceContent) {
/// Check whether the header part changed.
if(sourceContent.substr(0, preamble.size()) != preamble) {
return true;
}
/// Check timestamp of all files involved in building this PCH.
fs::file_status build;
if(auto error = fs::status(path, build)) {
llvm::errs() << "Error: " << error.message() << "\n";
std::terminate();
}
/// TODO: check whether deps changed through comparing timestamps.
return false;
}
void apply(Compiler& compiler) const;
};
/// Information of building precompiled module.
struct PCM {};
struct File;
struct Task {
/// Whether this task is a build task.
bool isBuild = false;
/// The coroutine handle of this task.
std::coroutine_handle<> waiting;
};
struct File {
bool isIdle = true;
/// The compiler instance of this file.
std::unique_ptr<Compiler> compiler;
std::deque<Task> waitings;
};
class Scheduler {
public:
async::promise<void> updatePCH(llvm::StringRef path,
llvm::StringRef content,
llvm::ArrayRef<const char*> args);
async::promise<void> updatePCM() {
co_return;
}
async::promise<void> buildAST(llvm::StringRef path, llvm::StringRef content);
async::promise<void> add(llvm::StringRef path, llvm::StringRef content);
async::promise<void> update(llvm::StringRef path, llvm::StringRef content);
async::promise<void> save(llvm::StringRef path);
async::promise<void> close(llvm::StringRef path);
/// Schedule a task for a file. If the file is building, the task will be
/// appended to the task list of the file and wait for the building to finish.
/// Otherwise, the task will be executed immediately.
template <typename Task>
auto schedule(llvm::StringRef path, Task&& task)
-> async::promise<decltype(task(std::declval<Compiler&>()))> {
auto& file = files[path];
if(!file.isIdle) {
co_await async::suspend([&](auto handle) {
file.waitings.push_back({.isBuild = false, .waiting = handle});
});
}
file.isIdle = false;
auto& compiler = *file.compiler;
auto result = co_await async::schedule_task([&task, &compiler] { return task(compiler); });
if(!file.waitings.empty()) {
auto task = std::move(file.waitings.front());
async::schedule(task.waiting);
file.waitings.pop_front();
}
file.isIdle = true;
co_return result;
}
private:
llvm::StringMap<PCH> pchs;
llvm::StringMap<File> files;
};
} // namespace clice

83
include/Server/Server.h
View File

@@ -3,6 +3,8 @@
#include "Async.h"
#include "Config.h"
#include "Logger.h"
#include "Scheduler.h"
#include "Basic/URI.h"
#include "Basic/Document.h"
#include "Compiler/Compiler.h"
#include "Support/JSON.h"
@@ -151,9 +153,6 @@ private:
});
}
llvm::StringMap<onRequest> requests;
llvm::StringMap<onNotification> notifications;
private:
/// ============================================================================
/// Lifestyle Message
@@ -245,81 +244,9 @@ private:
const proto::DocumentRangeFormattingParams& params);
private:
/// Information of building precompiled header.
struct PCH {
/// The path of this PCH.
std::string path;
/// The source file path.
std::string sourcePath;
/// The header part of source file used to build this PCH.
std::string preamble;
/// The arguments used to build this PCH.
std::string arguments;
/// All files involved in building this PCH(excluding the source file).
std::vector<std::string> deps;
/// FIXME: use asyncronous file system API.
bool needUpdate(llvm::StringRef sourceContent) {
/// Check whether the header part changed.
if(sourceContent.substr(0, preamble.size()) != preamble) {
return true;
}
/// Check timestamp of all files involved in building this PCH.
fs::file_status build;
if(auto error = fs::status(path, build)) {
llvm::errs() << "Error: " << error.message() << "\n";
std::terminate();
}
/// TODO: check whether deps changed through comparing timestamps.
return false;
}
};
/// Information of building precompiled module.
struct PCM {};
async::promise<void> updatePCH(llvm::StringRef filepath,
llvm::StringRef content,
llvm::ArrayRef<const char*> args);
async::promise<void> updatePCM() {
co_return;
}
async::promise<void> buildAST(llvm::StringRef filepath, llvm::StringRef content);
struct TranslationUnit {
enum class State {
Building,
Ready,
};
enum class TaskKind {
Build,
Consume,
};
struct Task {
TaskKind kind;
llvm::unique_function<async::promise<void>(Compiler&)> request;
};
State state;
std::unique_ptr<Compiler> compiler;
std::vector<Task> tasks;
};
/// Schedule a task for a file. If the file is building, the task will be
/// appended to the task list of the file and wait for the building to finish.
/// Otherwise, the task will be executed immediately.
async::promise<void> schedule(llvm::StringRef path,
llvm::unique_function<async::promise<void>(Compiler&)> callback);
private:
llvm::StringMap<PCH> pchs;
llvm::StringMap<TranslationUnit> units;
Scheduler scheduler;
llvm::StringMap<onRequest> requests;
llvm::StringMap<onNotification> notifications;
};
} // namespace clice

6
src/Feature/SemanticTokens.cpp
View File

@@ -618,9 +618,9 @@ public:
auto spelledTokens = tokBuf.spelledTokens(mainFileID);
for(auto& token: spelledTokens) {
proto::SemanticTokenType type = proto::SemanticTokenType::Invalid;
llvm::outs() << clang::tok::getTokenName(token.kind()) << " "
<< pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts())
<< "\n";
// llvm::outs() << clang::tok::getTokenName(token.kind()) << " "
// << pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts())
// << "\n";
auto kind = token.kind();
switch(kind) {

116
src/Server/Scheduler.cpp Normal file
View File

@@ -0,0 +1,116 @@
#include "Server/Scheduler.h"
#include "Server/Server.h"
namespace clice {
void PCH::apply(Compiler& compiler) const {
bool endAtStart = preamble.ends_with('@');
auto size = preamble.size() - endAtStart;
if(size != 0) {
compiler.applyPCH(path, size, endAtStart);
}
}
async::promise<void> Scheduler::updatePCH(llvm::StringRef filepath,
llvm::StringRef content,
llvm::ArrayRef<const char*> args) {
log::info("Start building PCH for {0}", filepath.str());
std::string outpath = "/home/ykiko/C++/clice2/build/cache/xxx.pch";
clang::PreambleBounds bounds = {0, 0};
co_await async::schedule_task([&] {
Compiler compiler(filepath, content, args);
bounds = clang::Lexer::ComputePreamble(content, {}, false);
if(bounds.Size != 0) {
compiler.generatePCH(outpath, bounds.Size, bounds.PreambleEndsAtStartOfLine);
}
});
log::info("Build PCH success");
auto preamble = content.substr(0, bounds.Size).str();
if(bounds.PreambleEndsAtStartOfLine) {
preamble.append("@");
}
pchs.try_emplace(filepath, PCH{.path = outpath, .preamble = std::move(preamble)});
co_return;
}
async::promise<void> Scheduler::buildAST(llvm::StringRef filepath, llvm::StringRef content) {
llvm::SmallString<128> path = filepath;
auto [iter, success] = files.try_emplace(filepath);
if(!success && !iter->second.isIdle) {
/// If the file is already existed and is building, append the task to the waiting list.
co_await async::suspend([&](auto handle) {
iter->second.waitings.emplace_back(Task{
.isBuild = true,
.waiting = handle,
});
});
}
files[filepath].isIdle = false;
/// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1
llvm::SmallVector<const char*> args = {
"clang++",
"-std=c++20",
path.c_str(),
"-resource-dir",
"/home/ykiko/C++/clice2/build/lib/clang/20",
};
/// through arguments to judge is it a module.
bool isModule = false;
co_await (isModule ? updatePCM() : updatePCH(filepath, content, args));
log::info("Start building AST for {0}", filepath.str());
auto task = [&path, &content, &args, pch = pchs.at(filepath)] {
/// FIXME: We cannot use reference capture the `pch` here, beacuse the reference may be
/// Invalid Because other changed the `pchs` map. We also cannot to retrieve the `pch` from
/// the `pchs` map in this task, beacuse it is called in thread pool which will result in
/// data race. So temporarily copy the `pch` here. There must be a better way to solve this
/// problem.
std::unique_ptr<Compiler> compiler = std::make_unique<Compiler>(path, content, args);
pch.apply(*compiler);
compiler->buildAST();
return compiler;
};
auto compiler = co_await async::schedule_task(std::move(task));
auto& file = files[path];
file.compiler = std::move(compiler);
log::info("Build AST success");
if(!file.waitings.empty()) {
auto task = std::move(file.waitings.front());
async::schedule(task.waiting);
file.waitings.pop_front();
}
}
async::promise<void> Scheduler::add(llvm::StringRef path, llvm::StringRef content) {
co_await buildAST(path, content);
co_return;
}
async::promise<void> Scheduler::update(llvm::StringRef path, llvm::StringRef content) {
co_await buildAST(path, content);
co_return;
}
async::promise<void> Scheduler::save(llvm::StringRef path) {
co_return;
}
async::promise<void> Scheduler::close(llvm::StringRef path) {
co_return;
}
} // namespace clice

106
src/Server/Server.cpp
View File

@@ -1,5 +1,4 @@
#include "Server/Server.h"
#include "Basic/URI.h"
namespace clice {
@@ -94,24 +93,24 @@ async::promise<void> Server::onDidOpen(const proto::DidOpenTextDocumentParams& p
auto path = URI::resolve(params.textDocument.uri);
llvm::StringRef content = params.textDocument.text;
co_await buildAST(path, content);
co_return;
co_await scheduler.add(path, content);
}
async::promise<void> Server::onDidChange(const proto::DidChangeTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
llvm::StringRef content = document.contentChanges[0].text;
co_await buildAST(path, content);
co_return;
co_await scheduler.update(path, content);
}
async::promise<void> Server::onDidSave(const proto::DidSaveTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
co_await scheduler.save(path);
co_return;
}
async::promise<void> Server::onDidClose(const proto::DidCloseTextDocumentParams& document) {
auto path = URI::resolve(document.textDocument.uri);
co_await scheduler.close(path);
co_return;
}
@@ -203,12 +202,10 @@ async::promise<void> Server::onDocumentSymbol(json::Value id,
async::promise<void> Server::onSemanticTokens(json::Value id,
const proto::SemanticTokensParams& params) {
auto path = URI::resolve(params.textDocument.uri);
auto task = [this, id = std::move(id)](Compiler& compiler) -> async::promise<void> {
auto tokens = feature::semanticTokens(compiler, "");
async::write(std::move(id), json::serialize(tokens));
co_return;
};
co_await schedule(path, std::move(task));
auto tokens = co_await scheduler.schedule(path, [&](Compiler& compiler) {
return feature::semanticTokens(compiler, "");
});
async::write(std::move(id), json::serialize(tokens));
co_return;
}
@@ -240,87 +237,4 @@ async::promise<void> Server::onRangeFormatting(json::Value id,
co_return;
}
async::promise<void> Server::updatePCH(llvm::StringRef filepath,
llvm::StringRef content,
llvm::ArrayRef<const char*> args) {
log::info("Start building PCH for {0}", filepath.str());
clang::PreambleBounds bounds = {0, 0};
std::string outpath = "/home/ykiko/C++/clice2/build/cache/xxx.pch";
co_await async::schedule_task([&] {
Compiler compiler(filepath, content, args);
bounds = clang::Lexer::ComputePreamble(content, {}, false);
if(bounds.Size != 0) {
compiler.generatePCH(outpath, bounds.Size, bounds.PreambleEndsAtStartOfLine);
}
});
log::info("Build PCH success");
auto preamble2 = content.substr(0, bounds.Size).str();
if(bounds.PreambleEndsAtStartOfLine) {
preamble2.append("@");
}
pchs.try_emplace(filepath, PCH{.path = outpath, .preamble = std::move(preamble2)});
co_return;
}
async::promise<void> Server::buildAST(llvm::StringRef filepath, llvm::StringRef content) {
llvm::SmallString<128> path = filepath;
/// FIXME: lookup from CDB file and adjust and remove unnecessary arguments.1
llvm::SmallVector<const char*> args = {
"clang++",
"-std=c++20",
path.c_str(),
"-resource-dir",
"/home/ykiko/C++/clice2/build/lib/clang/20",
};
/// through arguments to judge is it a module.
bool isModule = false;
co_await (isModule ? updatePCM() : updatePCH(filepath, content, args));
auto& pch = pchs.at(filepath);
log::info("Start building AST for {0}", filepath.str());
auto compiler = co_await async::schedule_task([&] {
std::uint32_t boundSize = pch.preamble.size();
bool endAtStart = false;
if(pch.preamble.back() == '@') {
boundSize -= 1;
endAtStart = true;
}
std::unique_ptr<Compiler> compiler = std::make_unique<Compiler>(path, content, args);
if(boundSize != 0) {
compiler->applyPCH(pch.path, boundSize, endAtStart);
}
compiler->buildAST();
return compiler;
});
log::info("Build AST success");
auto& unit = units[filepath];
unit.state = TranslationUnit::State::Ready;
unit.compiler = std::move(compiler);
for(auto& task: unit.tasks) {
co_await task.request(*unit.compiler);
if(task.kind == TranslationUnit::TaskKind::Build) {
break;
}
}
}
async::promise<void>
Server::schedule(llvm::StringRef path,
llvm::unique_function<async::promise<void>(Compiler&)> request) {
auto& unit = units[path];
if(unit.state == TranslationUnit::State::Building) {
unit.tasks.emplace_back(TranslationUnit::TaskKind::Consume, std::move(request));
} else {
co_await request(*units[path].compiler);
}
co_return;
}
} // namespace clice