diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 70541266..afdd7207 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -1,11 +1,8 @@ #include "server/compiler.h" -#include #include #include #include -#include -#include #include "command/search_config.h" #include "eventide/ipc/lsp/position.h" @@ -18,326 +15,40 @@ #include "syntax/include_resolver.h" #include "syntax/scan.h" -#include "llvm/Support/Chrono.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/xxhash.h" namespace clice { -/// Hash a file's content using xxh3_64bits. Returns 0 on read failure. -static std::uint64_t hash_file(llvm::StringRef path) { - auto buf = llvm::MemoryBuffer::getFile(path); - if(!buf) - return 0; - return llvm::xxh3_64bits((*buf)->getBuffer()); -} - -/// Capture a two-layer staleness snapshot after a successful compilation. -/// Interns dependency paths into the PathPool and hashes each file's content. -static DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps) { - DepsSnapshot snap; - // Capture timestamp BEFORE hashing to avoid TOCTOU: if a file is modified - // during hashing, its mtime will be > build_at, triggering Layer 2 re-hash. - snap.build_at = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - snap.path_ids.reserve(deps.size()); - snap.hashes.reserve(deps.size()); - for(const auto& file: deps) { - snap.path_ids.push_back(pool.intern(file)); - snap.hashes.push_back(hash_file(file)); - } - return snap; -} - -/// Two-layer staleness check. -/// -/// Layer 1 (fast): stat each dep file, compare mtime against build_at. -/// If all mtimes <= build_at → nothing changed, return false immediately. -/// -/// Layer 2 (precise): for files with mtime > build_at, re-hash their content. -/// If the hash matches the stored hash → file was touched but not modified. -/// If any hash differs → truly changed, return true. -static bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { - for(std::size_t i = 0; i < snap.path_ids.size(); ++i) { - auto path = pool.resolve(snap.path_ids[i]); - llvm::sys::fs::file_status status; - if(auto ec = llvm::sys::fs::status(path, status)) { - // File disappeared — definitely changed. - if(snap.hashes[i] != 0) - return true; - continue; - } - - // Layer 1: mtime check (cheap, stat only). - auto current_mtime = llvm::sys::toTimeT(status.getLastModificationTime()); - if(current_mtime <= snap.build_at) - continue; - - // Layer 2: mtime is newer — re-hash content to confirm actual change. - auto current_hash = hash_file(path); - if(current_hash != snap.hashes[i]) - return true; - } - return false; -} - -/// Serializable cache structures for cache.json persistence. -/// Paths are stored in a shared table and referenced by index to avoid -/// redundant storage (a single file can depend on thousands of headers, -/// many of which are shared across entries). -namespace { - -struct CacheDepEntry { - std::uint32_t path; // index into CacheData::paths - std::uint64_t hash; -}; - -struct CachePCHEntry { - std::string filename; - std::uint32_t source_file; // index into CacheData::paths - std::uint64_t hash; - std::uint32_t bound; - std::int64_t build_at; - std::vector deps; -}; - -struct CachePCMEntry { - std::string filename; - std::uint32_t source_file; - std::string module_name; - std::int64_t build_at; - std::vector deps; -}; - -struct CacheData { - std::vector paths; - std::vector pch; - std::vector pcm; -}; - -} // namespace - namespace lsp = eventide::ipc::lsp; using serde_raw = et::serde::RawValue; +/// Detect whether the cursor is inside a preamble directive (include/import). + Compiler::Compiler(et::event_loop& loop, et::ipc::JsonPeer& peer, - PathPool& path_pool, + Workspace& workspace, WorkerPool& pool, - Indexer& indexer, - const CliceConfig& config, - CompilationDatabase& cdb, - DependencyGraph& dep_graph) : - loop(loop), peer(peer), path_pool(path_pool), pool(pool), indexer(indexer), config(config), - cdb(cdb), dep_graph(dep_graph) {} + llvm::DenseMap& sessions) : + loop(loop), peer(peer), workspace(workspace), pool(pool), sessions(sessions) {} Compiler::~Compiler() { - cancel_all(); -} - -void Compiler::fill_pcm_deps(std::unordered_map& pcms, - std::uint32_t exclude_path_id) const { - for(auto& [pid, pcm_path]: pcm_paths) { - if(pid == exclude_path_id) - continue; - auto mod_it = path_to_module.find(pid); - if(mod_it != path_to_module.end()) { - pcms[mod_it->second] = pcm_path; - } - } -} - -void Compiler::cancel_all() { - if(compile_graph) { - compile_graph->cancel_all(); - } -} - -void Compiler::load_cache() { - if(config.cache_dir.empty()) - return; - - auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); - auto content = fs::read(cache_path); - if(!content) { - LOG_DEBUG("No cache.json found at {}", cache_path); - return; - } - - CacheData data; - auto status = et::serde::json::from_json(*content, data); - if(!status) { - LOG_WARN("Failed to parse cache.json"); - return; - } - - auto resolve = [&](std::uint32_t idx) -> llvm::StringRef { - return idx < data.paths.size() ? llvm::StringRef(data.paths[idx]) : ""; - }; - - auto load_deps = [&](std::int64_t build_at, const auto& dep_entries) -> DepsSnapshot { - DepsSnapshot deps; - deps.build_at = build_at; - for(auto& dep: dep_entries) { - auto dep_path = resolve(dep.path); - if(dep_path.empty()) - continue; - deps.path_ids.push_back(path_pool.intern(dep_path)); - deps.hashes.push_back(dep.hash); - } - return deps; - }; - - for(auto& entry: data.pch) { - auto pch_path = path::join(config.cache_dir, "cache", "pch", entry.filename); - auto source = resolve(entry.source_file); - if(!llvm::sys::fs::exists(pch_path) || source.empty()) - continue; - - auto path_id = path_pool.intern(source); - auto& st = pch_states[path_id]; - st.path = pch_path; - st.hash = entry.hash; - st.bound = entry.bound; - st.deps = load_deps(entry.build_at, entry.deps); - - LOG_DEBUG("Loaded cached PCH: {} -> {}", source, pch_path); - } - - for(auto& entry: data.pcm) { - auto pcm_path = path::join(config.cache_dir, "cache", "pcm", entry.filename); - auto source = resolve(entry.source_file); - if(!llvm::sys::fs::exists(pcm_path) || source.empty()) - continue; - - auto path_id = path_pool.intern(source); - pcm_states[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; - pcm_paths[path_id] = pcm_path; - - LOG_DEBUG("Loaded cached PCM: {} (module {}) -> {}", source, entry.module_name, pcm_path); - } - - LOG_INFO("Loaded cache.json: {} PCH entries, {} PCM entries", - pch_states.size(), - pcm_states.size()); -} - -void Compiler::save_cache() { - if(config.cache_dir.empty()) - return; - - CacheData data; - std::unordered_map index_map; - - auto intern = [&](std::uint32_t runtime_path_id) -> std::uint32_t { - auto path = std::string(path_pool.resolve(runtime_path_id)); - auto [it, inserted] = - index_map.try_emplace(path, static_cast(data.paths.size())); - if(inserted) { - data.paths.push_back(path); - } - return it->second; - }; - - for(auto& [path_id, st]: pch_states) { - if(st.path.empty()) - continue; - - CachePCHEntry entry; - entry.filename = std::string(path::filename(st.path)); - entry.source_file = intern(path_id); - entry.hash = st.hash; - entry.bound = st.bound; - entry.build_at = st.deps.build_at; - for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { - entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); - } - data.pch.push_back(std::move(entry)); - } - - for(auto& [path_id, st]: pcm_states) { - if(st.path.empty()) - continue; - - CachePCMEntry entry; - entry.filename = std::string(path::filename(st.path)); - entry.source_file = intern(path_id); - auto mod_it = path_to_module.find(path_id); - entry.module_name = mod_it != path_to_module.end() ? mod_it->second : ""; - entry.build_at = st.deps.build_at; - for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { - entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); - } - data.pcm.push_back(std::move(entry)); - } - - auto json_str = et::serde::json::to_json(data); - if(!json_str) { - LOG_WARN("Failed to serialize cache.json"); - return; - } - - auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); - auto tmp_path = cache_path + ".tmp"; - auto write_result = fs::write(tmp_path, *json_str); - if(!write_result) { - LOG_WARN("Failed to write cache.json.tmp: {}", write_result.error().message()); - return; - } - auto rename_result = fs::rename(tmp_path, cache_path); - if(!rename_result) { - LOG_WARN("Failed to rename cache.json.tmp to cache.json: {}", - rename_result.error().message()); - } -} - -void Compiler::cleanup_cache(int max_age_days) { - if(config.cache_dir.empty()) - return; - - auto now = std::chrono::system_clock::now(); - auto max_age = std::chrono::hours(max_age_days * 24); - - for(auto* subdir: {"cache/pch", "cache/pcm"}) { - auto dir = path::join(config.cache_dir, subdir); - std::error_code ec; - for(auto it = llvm::sys::fs::directory_iterator(dir, ec); - !ec && it != llvm::sys::fs::directory_iterator(); - it.increment(ec)) { - llvm::sys::fs::file_status status; - if(auto stat_ec = llvm::sys::fs::status(it->path(), status)) - continue; - - auto mtime = status.getLastModificationTime(); - auto age = now - mtime; - if(age > max_age) { - llvm::sys::fs::remove(it->path()); - LOG_DEBUG("Cleaned up stale cache file: {}", it->path()); - } - } - } -} - -void Compiler::build_module_map() { - for(auto& [module_name, path_ids]: dep_graph.modules()) { - for(auto path_id: path_ids) { - path_to_module[path_id] = module_name.str(); - } - } + workspace.cancel_all(); } void Compiler::init_compile_graph() { - if(path_to_module.empty()) { + if(workspace.path_to_module.empty()) { LOG_INFO("No C++20 modules detected, skipping CompileGraph"); return; } // Lazy dependency resolver: scans a module file on demand to discover imports. auto resolve = [this](std::uint32_t path_id) -> llvm::SmallVector { - auto file_path = path_pool.resolve(path_id); - auto results = cdb.lookup(file_path, {.query_toolchain = true, .suppress_logging = true}); + auto file_path = workspace.path_pool.resolve(path_id); + auto results = + workspace.cdb.lookup(file_path, {.query_toolchain = true, .suppress_logging = true}); if(results.empty()) return {}; @@ -346,7 +57,7 @@ void Compiler::init_compile_graph() { llvm::SmallVector deps; for(auto& mod_name: scan_result.modules) { - auto mod_ids = dep_graph.lookup_module(mod_name); + auto mod_ids = workspace.dep_graph.lookup_module(mod_name); if(!mod_ids.empty()) { deps.push_back(mod_ids[0]); } @@ -354,7 +65,7 @@ void Compiler::init_compile_graph() { // Module implementation units implicitly depend on their interface unit. if(!scan_result.module_name.empty() && !scan_result.is_interface_unit) { - auto mod_ids = dep_graph.lookup_module(scan_result.module_name); + auto mod_ids = workspace.dep_graph.lookup_module(scan_result.module_name); if(!mod_ids.empty()) { deps.push_back(mod_ids[0]); } @@ -365,11 +76,11 @@ void Compiler::init_compile_graph() { // Dispatch: sends BuildPCM request to a stateless worker. auto dispatch = [this](std::uint32_t path_id) -> et::task { - auto mod_it = path_to_module.find(path_id); - if(mod_it == path_to_module.end()) + auto mod_it = workspace.path_to_module.find(path_id); + if(mod_it == workspace.path_to_module.end()) co_return false; - auto file_path = std::string(path_pool.resolve(path_id)); + auto file_path = std::string(workspace.path_pool.resolve(path_id)); worker::BuildParams bp; bp.kind = worker::BuildKind::BuildPCM; @@ -386,13 +97,13 @@ void Compiler::init_compile_graph() { } auto args_hash = llvm::xxh3_64bits(llvm::StringRef(hash_input)); auto pcm_filename = std::format("{}-{:016x}.pcm", safe_module_name, args_hash); - auto pcm_path = path::join(config.cache_dir, "cache", "pcm", pcm_filename); + auto pcm_path = path::join(workspace.config.cache_dir, "cache", "pcm", pcm_filename); // Check if cached PCM is still valid. - if(auto pcm_it = pcm_states.find(path_id); pcm_it != pcm_states.end()) { + if(auto pcm_it = workspace.pcm_cache.find(path_id); pcm_it != workspace.pcm_cache.end()) { if(!pcm_it->second.path.empty() && llvm::sys::fs::exists(pcm_it->second.path) && - !deps_changed(path_pool, pcm_it->second.deps)) { - pcm_paths[path_id] = pcm_it->second.path; + !deps_changed(workspace.path_pool, pcm_it->second.deps)) { + workspace.pcm_paths[path_id] = pcm_it->second.path; co_return true; } } @@ -401,7 +112,7 @@ void Compiler::init_compile_graph() { bp.output_path = pcm_path; // Clang needs ALL transitive PCM deps, not just direct imports. - fill_pcm_deps(bp.pcms); + workspace.fill_pcm_deps(bp.pcms); auto result = co_await pool.send_stateless(bp); if(!result.has_value() || !result.value().success) { @@ -411,39 +122,41 @@ void Compiler::init_compile_graph() { co_return false; } - pcm_paths[path_id] = result.value().output_path; - pcm_states[path_id] = {result.value().output_path, - capture_deps_snapshot(path_pool, result.value().deps)}; + workspace.pcm_paths[path_id] = result.value().output_path; + workspace.pcm_cache[path_id] = { + result.value().output_path, + capture_deps_snapshot(workspace.path_pool, result.value().deps)}; LOG_INFO("Built PCM for module {}: {}", mod_it->second, result.value().output_path); - // Merge module index into ProjectIndex/MergedIndex. - if(!result.value().tu_index_data.empty()) { - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - } - // Persist cache metadata after successful build. - save_cache(); + workspace.save_cache(); + + // Signal that new index data is available for background merge. + if(on_indexing_needed) + on_indexing_needed(); + co_return true; }; - compile_graph = std::make_unique(std::move(dispatch), std::move(resolve)); - LOG_INFO("CompileGraph initialized with {} module(s)", path_to_module.size()); + workspace.compile_graph = + std::make_unique(std::move(dispatch), std::move(resolve)); + LOG_INFO("CompileGraph initialized with {} module(s)", workspace.path_to_module.size()); } bool Compiler::fill_compile_args(llvm::StringRef path, std::string& directory, - std::vector& arguments) { - auto path_id = path_pool.intern(path); + std::vector& arguments, + Session* session) { + auto path_id = workspace.path_pool.intern(path); - // 1. If the user has set an active header context via switchContext, + // 1. If the session has an active header context via switchContext, // use the host source's CDB entry with file path replaced and preamble injected. - auto active_it = active_contexts.find(path_id); - if(active_it != active_contexts.end()) { - return fill_header_context_args(path, path_id, directory, arguments); + if(session && session->active_context.has_value()) { + return fill_header_context_args(path, path_id, directory, arguments, session); } // 2. Normal CDB lookup for the file itself. - auto results = cdb.lookup(path, {.query_toolchain = true}); + auto results = workspace.cdb.lookup(path, {.query_toolchain = true}); if(!results.empty()) { auto& ctx = results.front(); directory = ctx.directory.str(); @@ -455,38 +168,47 @@ bool Compiler::fill_compile_args(llvm::StringRef path, } // 3. No CDB entry — try automatic header context resolution. - return fill_header_context_args(path, path_id, directory, arguments); + return fill_header_context_args(path, path_id, directory, arguments, session); } bool Compiler::fill_header_context_args(llvm::StringRef path, std::uint32_t path_id, std::string& directory, - std::vector& arguments) { + std::vector& arguments, + Session* session) { // Use cached context if available; otherwise resolve. // If an active context override exists, invalidate cache if it points to // a different host so we re-resolve with the correct one. const HeaderFileContext* ctx_ptr = nullptr; - auto ctx_it = header_file_contexts.find(path_id); - auto active_it = active_contexts.find(path_id); - if(ctx_it != header_file_contexts.end()) { - if(active_it != active_contexts.end() && ctx_it->second.host_path_id != active_it->second) { - header_file_contexts.erase(ctx_it); + if(session && session->header_context.has_value()) { + if(session->active_context.has_value() && + session->header_context->host_path_id != *session->active_context) { + session->header_context.reset(); } else { - ctx_ptr = &ctx_it->second; + ctx_ptr = &*session->header_context; } } if(!ctx_ptr) { - auto resolved = resolve_header_context(path_id); + auto resolved = resolve_header_context(path_id, session); if(!resolved) { LOG_WARN("No CDB entry and no header context for {}", path); return false; } - header_file_contexts[path_id] = std::move(*resolved); - ctx_ptr = &header_file_contexts[path_id]; + if(session) { + session->header_context = std::move(*resolved); + ctx_ptr = &*session->header_context; + } else { + // Background indexing path — no session to store on. + // Use a temporary (caller will use it immediately). + // Store in a local and return. + static thread_local std::optional tl_ctx; + tl_ctx = std::move(*resolved); + ctx_ptr = &*tl_ctx; + } } - auto host_path = path_pool.resolve(ctx_ptr->host_path_id); - auto host_results = cdb.lookup(host_path, {.query_toolchain = true}); + auto host_path = workspace.path_pool.resolve(ctx_ptr->host_path_id); + auto host_results = workspace.cdb.lookup(host_path, {.query_toolchain = true}); if(host_results.empty()) { LOG_WARN("fill_header_context_args: host {} has no CDB entry", host_path); return false; @@ -527,9 +249,10 @@ bool Compiler::fill_header_context_args(llvm::StringRef path, return true; } -std::optional Compiler::resolve_header_context(std::uint32_t header_path_id) { +std::optional Compiler::resolve_header_context(std::uint32_t header_path_id, + Session* session) { // Find source files that transitively include this header. - auto hosts = dep_graph.find_host_sources(header_path_id); + auto hosts = workspace.dep_graph.find_host_sources(header_path_id); if(hosts.empty()) { LOG_DEBUG("resolve_header_context: no host sources for path_id={}", header_path_id); return std::nullopt; @@ -538,13 +261,12 @@ std::optional Compiler::resolve_header_context(std::uint32_t // If there's an active context override, prefer that host. std::uint32_t host_path_id = 0; std::vector chain; - auto active_it = active_contexts.find(header_path_id); - if(active_it != active_contexts.end()) { - auto preferred = active_it->second; - auto preferred_path = path_pool.resolve(preferred); - auto results = cdb.lookup(preferred_path, {.suppress_logging = true}); + if(session && session->active_context.has_value()) { + auto preferred = *session->active_context; + auto preferred_path = workspace.path_pool.resolve(preferred); + auto results = workspace.cdb.lookup(preferred_path, {.suppress_logging = true}); if(!results.empty()) { - auto c = dep_graph.find_include_chain(preferred, header_path_id); + auto c = workspace.dep_graph.find_include_chain(preferred, header_path_id); if(!c.empty()) { host_path_id = preferred; chain = std::move(c); @@ -555,11 +277,11 @@ std::optional Compiler::resolve_header_context(std::uint32_t // Fall back to the first available host that has a CDB entry. if(chain.empty()) { for(auto candidate: hosts) { - auto candidate_path = path_pool.resolve(candidate); - auto results = cdb.lookup(candidate_path, {.suppress_logging = true}); + auto candidate_path = workspace.path_pool.resolve(candidate); + auto results = workspace.cdb.lookup(candidate_path, {.suppress_logging = true}); if(results.empty()) continue; - auto c = dep_graph.find_include_chain(candidate, header_path_id); + auto c = workspace.dep_graph.find_include_chain(candidate, header_path_id); if(c.empty()) continue; host_path_id = candidate; @@ -582,22 +304,23 @@ std::optional Compiler::resolve_header_context(std::uint32_t auto cur_id = chain[i]; auto next_id = chain[i + 1]; - auto cur_path = path_pool.resolve(cur_id); - auto next_path = path_pool.resolve(next_id); + auto cur_path = workspace.path_pool.resolve(cur_id); + auto next_path = workspace.path_pool.resolve(next_id); auto next_filename = llvm::sys::path::filename(next_path); // Prefer in-memory document text over disk content. + // Use the session if this file matches the session's path, otherwise + // fall back to disk. std::string content; - if(auto doc_it = documents.find(cur_id); doc_it != documents.end()) { - content = doc_it->second.text; - } else { - auto buf = llvm::MemoryBuffer::getFile(cur_path); - if(!buf) { - LOG_WARN("resolve_header_context: cannot read {}", cur_path); - return std::nullopt; - } - content = (*buf)->getBuffer().str(); + // Note: we don't have the sessions map here, so we always read from disk + // for intermediate chain files. The session parameter only covers the + // header file itself (the target), not intermediate files in the chain. + auto buf = llvm::MemoryBuffer::getFile(cur_path); + if(!buf) { + LOG_WARN("resolve_header_context: cannot read {}", cur_path); + return std::nullopt; } + content = (*buf)->getBuffer().str(); // Scan line by line for the #include that brings in next_filename. llvm::StringRef content_ref(content); @@ -648,7 +371,7 @@ std::optional Compiler::resolve_header_context(std::uint32_t // Hash the preamble and write to cache directory. auto preamble_hash = llvm::xxh3_64bits(llvm::StringRef(preamble)); auto preamble_filename = std::format("{:016x}.h", preamble_hash); - auto preamble_dir = path::join(config.cache_dir, "header_context"); + auto preamble_dir = path::join(workspace.config.cache_dir, "header_context"); auto preamble_path = path::join(preamble_dir, preamble_filename); if(!llvm::sys::fs::exists(preamble_path)) { @@ -673,216 +396,7 @@ std::optional Compiler::resolve_header_context(std::uint32_t return HeaderFileContext{host_path_id, preamble_path, preamble_hash}; } -void Compiler::switch_context(std::uint32_t path_id, std::uint32_t context_path_id) { - active_contexts[path_id] = context_path_id; - header_file_contexts.erase(path_id); - pch_states.erase(path_id); - ast_deps.erase(path_id); - auto doc_it = documents.find(path_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; - } -} - -std::optional Compiler::get_active_context(std::uint32_t path_id) const { - auto it = active_contexts.find(path_id); - if(it != active_contexts.end()) - return it->second; - return std::nullopt; -} - -void Compiler::invalidate_host_contexts(std::uint32_t host_path_id, - llvm::SmallVectorImpl& stale_headers) { - for(auto& [hdr_id, hdr_ctx]: header_file_contexts) { - if(hdr_ctx.host_path_id == host_path_id) - stale_headers.push_back(hdr_id); - } - for(auto hdr_id: stale_headers) { - header_file_contexts.erase(hdr_id); - } -} - -et::task Compiler::ensure_pch(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, - const std::string& directory, - const std::vector& arguments) { - auto bound = compute_preamble_bound(text); - if(bound == 0) { - // No preamble directives — PCH would be empty. Clear any stale entry. - pch_states.erase(path_id); - co_return true; - } - - auto preamble_hash = llvm::xxh3_64bits(llvm::StringRef(text).substr(0, bound)); - - // Deterministic content-addressed PCH path. - auto pch_path = - path::join(config.cache_dir, "cache", "pch", std::format("{:016x}.pch", preamble_hash)); - - // Reuse existing PCH if preamble content and deps haven't changed. - if(auto it = pch_states.find(path_id); it != pch_states.end()) { - auto& st = it->second; - if(st.hash == preamble_hash && !st.path.empty() && !deps_changed(path_pool, st.deps)) { - st.bound = bound; - co_return true; - } - } - - // Preamble incomplete (user still typing) — defer rebuild, reuse old PCH if available. - if(!is_preamble_complete(text, bound)) { - LOG_DEBUG("Preamble incomplete for {}, deferring PCH rebuild", path); - co_return pch_states.count(path_id) && !pch_states[path_id].path.empty(); - } - - // If another coroutine is already building PCH for this file, wait for it. - if(auto it = pch_states.find(path_id); it != pch_states.end() && it->second.building) { - co_await it->second.building->wait(); - co_return !pch_states[path_id].path.empty(); - } - - // Register in-flight build so concurrent requests wait on us. - auto completion = std::make_shared(); - pch_states[path_id].building = completion; - - // Build a new PCH via stateless worker. - worker::BuildParams bp; - bp.kind = worker::BuildKind::BuildPCH; - bp.file = std::string(path); - bp.directory = directory; - bp.arguments = arguments; - bp.text = text; - bp.preamble_bound = bound; - bp.output_path = pch_path; - - LOG_DEBUG("Building PCH for {}, bound={}, output={}", path, bound, pch_path); - - auto result = co_await pool.send_stateless(bp); - - if(!result.has_value() || !result.value().success) { - LOG_WARN("PCH build failed for {}: {}", - path, - result.has_value() ? result.value().error : result.error().message); - pch_states[path_id].building.reset(); - completion->set(); - co_return false; - } - - auto& st = pch_states[path_id]; - st.path = result.value().output_path; - st.bound = bound; - st.hash = preamble_hash; - st.deps = capture_deps_snapshot(path_pool, result.value().deps); - st.building.reset(); - - LOG_INFO("PCH built for {}: {}", path, result.value().output_path); - - if(!result.value().tu_index_data.empty()) { - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - } - - // Persist cache metadata after successful build. - save_cache(); - - completion->set(); - co_return true; -} - -/// Compile module dependencies, build/reuse PCH, and fill PCM paths. -/// Shared preparation step used by both ensure_compiled() (stateful path) -/// and forward_stateless() (completion/signatureHelp path). -et::task Compiler::ensure_deps(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, - const std::string& directory, - const std::vector& arguments, - std::pair& pch, - std::unordered_map& pcms) { - // Compile C++20 module dependencies (PCMs). - if(compile_graph && !co_await compile_graph->compile_deps(path_id)) { - co_return false; - } - - // Scan buffer text for module imports that might not be in compile_graph yet. - // When a user adds `import std;` without saving, the compile_graph (disk-based) - // doesn't know about the new dependency. Scan the in-memory text to find them. - { - auto scan_result = scan(text); - for(auto& mod_name: scan_result.modules) { - if(mod_name.empty()) - continue; - bool found = false; - for(auto& [pid, name]: path_to_module) { - if(name == mod_name) { - // If PCM not already built, try to build it. - if(pcm_paths.find(pid) == pcm_paths.end()) { - if(compile_graph && compile_graph->has_unit(pid)) { - co_await compile_graph->compile_deps(pid); - } - } - found = true; - break; - } - } - if(!found) { - LOG_DEBUG("Buffer imports unknown module '{}', skipping", mod_name); - } - } - } - - // Build or reuse PCH. - auto pch_ok = co_await ensure_pch(path_id, path, text, directory, arguments); - if(pch_ok) { - if(auto pch_it = pch_states.find(path_id); pch_it != pch_states.end()) { - pch = {pch_it->second.path, pch_it->second.bound}; - } - } - - // Fill all available PCM paths, excluding the file's own PCM - // to avoid "multiple module declarations". - fill_pcm_deps(pcms, path_id); - - co_return true; -} - -bool Compiler::is_stale(std::uint32_t path_id) { - auto ast_deps_it = ast_deps.find(path_id); - if(ast_deps_it != ast_deps.end() && deps_changed(path_pool, ast_deps_it->second)) - return true; - - auto pch_it = pch_states.find(path_id); - if(pch_it != pch_states.end() && deps_changed(path_pool, pch_it->second.deps)) - return true; - - return false; -} - -void Compiler::record_deps(std::uint32_t path_id, llvm::ArrayRef deps) { - ast_deps[path_id] = capture_deps_snapshot(path_pool, deps); -} - -void Compiler::on_file_closed(std::uint32_t path_id) { - if(compile_graph && compile_graph->has_unit(path_id)) { - compile_graph->update(path_id); - } - pch_states.erase(path_id); - ast_deps.erase(path_id); -} - -llvm::SmallVector Compiler::on_file_saved(std::uint32_t path_id) { - llvm::SmallVector dirtied; - if(compile_graph) { - auto result = compile_graph->update(path_id); - for(auto id: result) { - dirtied.push_back(id); - pcm_paths.erase(id); - pcm_states.erase(id); - } - } - return dirtied; -} - -std::string Compiler::uri_to_path(const std::string& uri) { +std::string uri_to_path(const std::string& uri) { auto parsed = lsp::URI::parse(uri); if(parsed.has_value()) { auto path = parsed->file_path(); @@ -893,112 +407,6 @@ std::string Compiler::uri_to_path(const std::string& uri) { return uri; } -void Compiler::open_document(const std::string& uri, std::string text, int version) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - auto& doc = documents[path_id]; - doc.version = version; - doc.text = std::move(text); - doc.generation++; - - LOG_DEBUG("didOpen: {} (v{})", path, version); -} - -void Compiler::apply_changes(const protocol::DidChangeTextDocumentParams& params) { - auto path = uri_to_path(params.text_document.uri); - auto path_id = path_pool.intern(path); - - auto it = documents.find(path_id); - if(it == documents.end()) - return; - - auto& doc = it->second; - doc.version = params.text_document.version; - - for(auto& change: params.content_changes) { - std::visit( - [&](auto& c) { - using T = std::remove_cvref_t; - if constexpr(std::is_same_v) { - doc.text = c.text; - } else { - auto& range = c.range; - lsp::PositionMapper mapper(doc.text, lsp::PositionEncoding::UTF16); - auto start = mapper.to_offset(range.start); - auto end = mapper.to_offset(range.end); - if(start && end && *start <= *end) { - doc.text.replace(*start, *end - *start, c.text); - } - } - }, - change); - } - - doc.generation++; - doc.ast_dirty = true; - - LOG_DEBUG("didChange: path={} version={} gen={}", path, doc.version, doc.generation); - - worker::DocumentUpdateParams update; - update.path = path; - update.version = doc.version; - pool.notify_stateful(path_id, update); -} - -std::uint32_t Compiler::close_document(const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - on_file_closed(path_id); - indexer.remove_open_file(path_id, path); - pool.notify_stateful(path_id, worker::EvictParams{path}); - documents.erase(path_id); - clear_diagnostics(uri); - - LOG_DEBUG("didClose: {}", path); - return path_id; -} - -llvm::SmallVector Compiler::on_save(const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - llvm::SmallVector to_index; - - auto dirtied = on_file_saved(path_id); - for(auto dirty_id: dirtied) { - auto doc_it = documents.find(dirty_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; - } else { - to_index.push_back(dirty_id); - } - } - - llvm::SmallVector stale_headers; - invalidate_host_contexts(path_id, stale_headers); - for(auto hdr_id: stale_headers) { - auto doc_it = documents.find(hdr_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; - LOG_DEBUG("didSave: invalidated header context for path_id={}", hdr_id); - } - } - - LOG_DEBUG("didSave: {}", uri); - return to_index; -} - -bool Compiler::is_file_open(std::uint32_t path_id) const { - return documents.count(path_id); -} - -const DocumentState* Compiler::get_document(std::uint32_t path_id) const { - auto it = documents.find(path_id); - return it != documents.end() ? &it->second : nullptr; -} - void Compiler::publish_diagnostics(const std::string& uri, int version, const et::serde::RawValue& diagnostics_json) { @@ -1023,16 +431,192 @@ void Compiler::clear_diagnostics(const std::string& uri) { peer.send_notification(params); } +et::task Compiler::ensure_pch(Session& session, + const std::string& directory, + const std::vector& arguments) { + auto path_id = session.path_id; + auto path = workspace.path_pool.resolve(path_id); + auto& text = session.text; + auto bound = compute_preamble_bound(text); + if(bound == 0) { + // No preamble directives — PCH would be empty. Clear any stale entry. + workspace.pch_cache.erase(path_id); + session.pch_ref.reset(); + co_return true; + } + + // FIXME: hash should also include compile flags that affect preprocessing + // (e.g. -D, -I, -isystem, -std) so that files with the same preamble text + // but different flags produce separate PCHs. Currently only the preamble + // text is hashed — the source file path must be excluded from the hash + // to allow sharing across files with identical preambles. + auto preamble_text = llvm::StringRef(text).substr(0, bound); + auto preamble_hash = llvm::xxh3_64bits(preamble_text); + + // Deterministic content-addressed PCH path. + auto pch_path = path::join(workspace.config.cache_dir, + "cache", + "pch", + std::format("{:016x}.pch", preamble_hash)); + + // Reuse existing PCH if preamble content and deps haven't changed. + if(auto it = workspace.pch_cache.find(path_id); it != workspace.pch_cache.end()) { + auto& st = it->second; + if(st.hash == preamble_hash && !st.path.empty() && + !deps_changed(workspace.path_pool, st.deps)) { + st.bound = bound; + session.pch_ref = Session::PCHRef{path_id, preamble_hash, bound}; + co_return true; + } + } + + // Preamble incomplete (user still typing) — defer rebuild, reuse old PCH if available. + if(!is_preamble_complete(text, bound)) { + LOG_DEBUG("Preamble incomplete for {}, deferring PCH rebuild", path); + co_return workspace.pch_cache.count(path_id) && !workspace.pch_cache[path_id].path.empty(); + } + + // If another coroutine is already building PCH for this file, wait for it. + if(auto it = workspace.pch_cache.find(path_id); + it != workspace.pch_cache.end() && it->second.building) { + co_await it->second.building->wait(); + if(auto it2 = workspace.pch_cache.find(path_id); it2 != workspace.pch_cache.end()) { + session.pch_ref = Session::PCHRef{path_id, it2->second.hash, it2->second.bound}; + } + co_return workspace.pch_cache.count(path_id) && !workspace.pch_cache[path_id].path.empty(); + } + + // Register in-flight build so concurrent requests wait on us. + auto completion = std::make_shared(); + workspace.pch_cache[path_id].building = completion; + + // Build a new PCH via stateless worker. + worker::BuildParams bp; + bp.kind = worker::BuildKind::BuildPCH; + bp.file = std::string(path); + bp.directory = directory; + bp.arguments = arguments; + bp.text = text; + bp.preamble_bound = bound; + bp.output_path = pch_path; + + LOG_DEBUG("Building PCH for {}, bound={}, output={}", path, bound, pch_path); + + auto result = co_await pool.send_stateless(bp); + + if(!result.has_value() || !result.value().success) { + LOG_WARN("PCH build failed for {}: {}", + path, + result.has_value() ? result.value().error : result.error().message); + workspace.pch_cache[path_id].building.reset(); + completion->set(); + co_return false; + } + + auto& st = workspace.pch_cache[path_id]; + st.path = result.value().output_path; + st.bound = bound; + st.hash = preamble_hash; + st.deps = capture_deps_snapshot(workspace.path_pool, result.value().deps); + st.building.reset(); + + session.pch_ref = Session::PCHRef{path_id, preamble_hash, bound}; + + LOG_INFO("PCH built for {}: {}", path, result.value().output_path); + + // Persist cache metadata after successful build. + workspace.save_cache(); + + completion->set(); + co_return true; +} + +/// Compile module dependencies, build/reuse PCH, and fill PCM paths. +/// Shared preparation step used by both ensure_compiled() (stateful path) +/// and forward_stateless() (completion/signatureHelp path). +et::task Compiler::ensure_deps(Session& session, + const std::string& directory, + const std::vector& arguments, + std::pair& pch, + std::unordered_map& pcms) { + auto path_id = session.path_id; + + // Compile C++20 module dependencies (PCMs). + if(workspace.compile_graph && !co_await workspace.compile_graph->compile_deps(path_id)) { + co_return false; + } + + // Scan buffer text for module imports that might not be in compile_graph yet. + // When a user adds `import std;` without saving, the compile_graph (disk-based) + // doesn't know about the new dependency. Scan the in-memory text to find them. + { + auto scan_result = scan(session.text); + for(auto& mod_name: scan_result.modules) { + if(mod_name.empty()) + continue; + bool found = false; + for(auto& [pid, name]: workspace.path_to_module) { + if(name == mod_name) { + // If PCM not already built, try to build it. + if(workspace.pcm_paths.find(pid) == workspace.pcm_paths.end()) { + if(workspace.compile_graph && workspace.compile_graph->has_unit(pid)) { + co_await workspace.compile_graph->compile_deps(pid); + } + } + found = true; + break; + } + } + if(!found) { + LOG_DEBUG("Buffer imports unknown module '{}', skipping", mod_name); + } + } + } + + // Build or reuse PCH. + auto pch_ok = co_await ensure_pch(session, directory, arguments); + if(pch_ok) { + if(auto pch_it = workspace.pch_cache.find(path_id); pch_it != workspace.pch_cache.end()) { + pch = {pch_it->second.path, pch_it->second.bound}; + } + } + + // Fill all available PCM paths, excluding the file's own PCM + // to avoid "multiple module declarations". + workspace.fill_pcm_deps(pcms, path_id); + + co_return true; +} + +bool Compiler::is_stale(const Session& session) { + if(session.ast_deps.has_value() && deps_changed(workspace.path_pool, *session.ast_deps)) + return true; + + // Check PCH staleness via the session's pch_ref. + if(session.pch_ref.has_value()) { + auto pch_it = workspace.pch_cache.find(session.pch_ref->path_id); + if(pch_it != workspace.pch_cache.end() && + deps_changed(workspace.path_pool, pch_it->second.deps)) + return true; + } + + return false; +} + +void Compiler::record_deps(Session& session, llvm::ArrayRef deps) { + session.ast_deps = capture_deps_snapshot(workspace.path_pool, deps); +} + /// Pull-based compilation entry point for user-opened files. /// -/// Called lazily by forward_stateful() / forward_stateless() before every +/// Called lazily by forward_query() / forward_build() before every /// feature request (hover, semantic tokens, etc.). Guarantees that when it /// returns true the stateful worker assigned to `path_id` holds an up-to-date /// AST and diagnostics have been published to the client. /// /// Lifecycle overview (pull-based model): /// -/// didOpen / didChange – only update DocumentState, mark ast_dirty +/// didOpen / didChange – only update Session, mark ast_dirty /// didSave – mark dependents dirty, queue indexing /// feature request arrives – calls ensure_compiled() first /// 1. Fast-path exit if AST is already clean (!ast_dirty). @@ -1051,40 +635,30 @@ void Compiler::clear_diagnostics(const std::string& uri) { /// task via loop.schedule(); subsequent ones wait on the shared event. /// The detached task cannot be cancelled by LSP $/cancelRequest, preventing /// the race where cancellation wakes all waiters and they all start compiles. -et::task Compiler::ensure_compiled(std::uint32_t path_id) { - auto it = documents.find(path_id); - if(it == documents.end()) { - LOG_WARN("ensure_compiled: doc not found for path_id={} path={}", - path_id, - path_pool.resolve(path_id)); - co_return false; - } +et::task Compiler::ensure_compiled(Session& session) { + auto path_id = session.path_id; - auto& doc = it->second; LOG_DEBUG("ensure_compiled: path_id={} version={} gen={} ast_dirty={}", path_id, - doc.version, - doc.generation, - doc.ast_dirty); + session.version, + session.generation, + session.ast_dirty); - if(!doc.ast_dirty) { - if(!is_stale(path_id)) { + if(!session.ast_dirty) { + if(!is_stale(session)) { co_return true; } - doc.ast_dirty = true; + session.ast_dirty = true; } // If another compile is already in flight, wait for it. // This co_await may be cancelled by LSP $/cancelRequest — that's fine, // it just means this particular feature request is abandoned. The // detached compile task keeps running independently. - while(it->second.compiling) { - auto pending = it->second.compiling; + while(session.compiling) { + auto pending = session.compiling; co_await pending->done.wait(); - it = documents.find(path_id); - if(it == documents.end()) - co_return false; - if(!it->second.ast_dirty) + if(!session.ast_dirty) co_return true; } @@ -1093,80 +667,82 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { // to LSP $/cancelRequest cancellation. This eliminates the race where // cancellation fires the RAII guard, waking all waiters simultaneously // and causing them all to start new compiles. - auto pending_compile = std::make_shared(); - it->second.compiling = pending_compile; + auto pending_compile = std::make_shared(); + session.compiling = pending_compile; LOG_INFO("ensure_compiled: launching detached compile path_id={} gen={}", path_id, - doc.generation); + session.generation); + // Capture path_id by value so the detached lambda can re-lookup the session + // from the sessions map after co_await (DenseMap may invalidate pointers). loop.schedule([](Compiler* self, std::uint32_t pid, - std::shared_ptr pc) -> et::task<> { + std::shared_ptr pc) -> et::task<> { + // Re-lookup session from the sessions map (pointer may have been + // invalidated by DenseMap growth during co_await). + auto find_session = [&]() -> Session* { + auto it = self->sessions.find(pid); + return it != self->sessions.end() ? &it->second : nullptr; + }; + + auto* sess = find_session(); + if(!sess) { + pc->done.set(); + co_return; + } + auto finish_compile = [&]() { - if(auto it = self->documents.find(pid); it != self->documents.end()) { - if(it->second.compiling == pc) { - it->second.compiling.reset(); - } + auto* s = find_session(); + if(s && s->compiling == pc) { + s->compiling.reset(); } LOG_INFO("ensure_compiled: finish_compile (detached) path_id={}", pid); pc->done.set(); }; - auto it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); - co_return; - } - - auto gen = it->second.generation; + auto gen = sess->generation; LOG_INFO("ensure_compiled: starting compile (detached) path_id={} gen={}", pid, gen); - auto file_path = std::string(self->path_pool.resolve(pid)); + auto file_path = std::string(self->workspace.path_pool.resolve(pid)); auto uri = lsp::URI::from_file_path(file_path); std::string uri_str = uri.has_value() ? uri->str() : file_path; worker::CompileParams params; params.path = file_path; - params.version = it->second.version; - params.text = it->second.text; - if(!self->fill_compile_args(file_path, params.directory, params.arguments)) { + params.version = sess->version; + params.text = sess->text; + if(!self->fill_compile_args(file_path, params.directory, params.arguments, sess)) { finish_compile(); co_return; } - if(!co_await self->ensure_deps(pid, - params.path, - params.text, - params.directory, - params.arguments, - params.pch, - params.pcms)) { + if(!co_await self + ->ensure_deps(*sess, params.directory, params.arguments, params.pch, params.pcms)) { LOG_WARN("Dependency preparation failed for {}, skipping compile", uri_str); finish_compile(); co_return; } - it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); + // Re-lookup after co_await (DenseMap may have grown). + sess = find_session(); + if(!sess) { + pc->done.set(); co_return; } auto result = co_await self->pool.send_stateful(pid, params); - // Re-lookup: the document may have been closed while we were compiling. - it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); + // Re-lookup after co_await. + sess = find_session(); + if(!sess) { + pc->done.set(); co_return; } - auto& doc2 = it->second; - - if(doc2.generation != gen) { + if(sess->generation != gen) { LOG_INFO("ensure_compiled: generation mismatch ({} vs {}) for {}", - doc2.generation, + sess->generation, gen, uri_str); finish_compile(); @@ -1180,9 +756,9 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { co_return; } - doc2.ast_dirty = false; + sess->ast_dirty = false; pc->succeeded = true; - self->record_deps(pid, result.value().deps); + self->record_deps(*sess, result.value().deps); // Store open file index from the stateful worker's TUIndex. if(!result.value().tu_index_data.empty()) { @@ -1190,15 +766,17 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { OpenFileIndex ofi; ofi.file_index = std::move(tu_index.main_file_index); ofi.symbols = std::move(tu_index.symbols); - ofi.content = doc2.text; - self->indexer.set_open_file(pid, file_path, std::move(ofi)); + ofi.content = sess->text; + ofi.mapper.emplace(ofi.content, lsp::PositionEncoding::UTF16); + sess->file_index = std::move(ofi); } + auto version = sess->version; finish_compile(); // Publish diagnostics AFTER marking compile as done, so that concurrent - // forward_stateful() calls can proceed immediately. - self->publish_diagnostics(uri_str, doc2.version, result.value().diagnostics); + // forward_query() calls can proceed immediately. + self->publish_diagnostics(uri_str, version, result.value().diagnostics); if(self->on_indexing_needed) self->on_indexing_needed(); }(this, path_id, pending_compile)); @@ -1207,55 +785,49 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { // by LSP $/cancelRequest, the detached task continues unaffected. co_await pending_compile->done.wait(); - it = documents.find(path_id); - if(it == documents.end()) - co_return false; - - co_return !it->second.ast_dirty; -} - -Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - if(!co_await ensure_compiled(path_id)) { - co_return serde_raw{"null"}; - } - - auto dit = documents.find(path_id); - if(dit != documents.end() && dit->second.ast_dirty) { - co_return serde_raw{"null"}; - } - - worker::QueryParams wp{kind, path}; - auto result = co_await pool.send_stateful(path_id, wp); - if(!result.has_value()) { - co_return serde_raw{}; - } - co_return std::move(result.value()); + co_return !session.ast_dirty; } Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, - const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); + Session& session, + std::optional position, + std::optional range) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); + // Cache text before co_await — session reference may dangle if didClose + // erases the entry from the sessions map during suspension. + auto text = session.text; - if(!co_await ensure_compiled(path_id)) { + if(!co_await ensure_compiled(session)) { co_return serde_raw{"null"}; } - auto doc_it = documents.find(path_id); - if(doc_it == documents.end() || doc_it->second.ast_dirty) { + auto sit = sessions.find(path_id); + if(sit == sessions.end() || sit->second.ast_dirty) { co_return serde_raw{"null"}; } - lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); - auto offset = mapper.to_offset(position); - if(!offset) - co_return serde_raw{"null"}; + worker::QueryParams wp; + wp.kind = kind; + wp.path = path; + + lsp::PositionMapper mapper(text, lsp::PositionEncoding::UTF16); + + if(position) { + auto offset = mapper.to_offset(*position); + if(!offset) + co_return serde_raw{"null"}; + wp.offset = *offset; + } + + if(range) { + auto start = mapper.to_offset(range->start); + auto end = mapper.to_offset(range->end); + if(start && end) { + wp.range = {*start, *end}; + } + } - worker::QueryParams wp{kind, path, *offset}; auto result = co_await pool.send_stateful(path_id, wp); if(!result.has_value()) { co_return serde_raw{}; @@ -1264,28 +836,28 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, } Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, - const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - auto doc_it = documents.find(path_id); - if(doc_it == documents.end()) { - co_return serde_raw{}; - } - - auto& doc = doc_it->second; + const protocol::Position& position, + Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); worker::BuildParams wp; wp.kind = kind; wp.file = path; - wp.version = doc.version; - wp.text = doc.text; - if(!fill_compile_args(path, wp.directory, wp.arguments)) { + // Cache session fields before co_await — session reference may dangle + // if didClose erases the entry from the sessions map during suspension. + wp.version = session.version; + wp.text = session.text; + if(!fill_compile_args(path, wp.directory, wp.arguments, &session)) { co_return serde_raw{}; } - if(!co_await ensure_deps(path_id, path, wp.text, wp.directory, wp.arguments, wp.pch, wp.pcms)) { + if(!co_await ensure_deps(session, wp.directory, wp.arguments, wp.pch, wp.pcms)) { + co_return serde_raw{}; + } + + // After co_await, verify session still exists. + if(sessions.find(path_id) == sessions.end()) { co_return serde_raw{}; } @@ -1302,170 +874,62 @@ Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, co_return std::move(result.value().result_json); } -Compiler::RawResult Compiler::handle_completion(const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - auto doc_it = documents.find(path_id); - if(doc_it != documents.end()) { - lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); - auto offset = mapper.to_offset(position); - if(offset) { - auto pctx = detect_completion_context(doc_it->second.text, *offset); - if(pctx.kind == CompletionContext::IncludeQuoted || - pctx.kind == CompletionContext::IncludeAngled) { - co_return complete_include(pctx, path); +Compiler::RawResult Compiler::handle_completion(const protocol::Position& position, + Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); + + lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); + auto offset = mapper.to_offset(position); + if(offset) { + auto pctx = detect_completion_context(session.text, *offset); + if(pctx.kind == CompletionContext::IncludeQuoted || + pctx.kind == CompletionContext::IncludeAngled) { + std::string directory; + std::vector arguments; + if(!fill_compile_args(path, directory, arguments)) + co_return serde_raw{"[]"}; + + std::vector args_ptrs; + args_ptrs.reserve(arguments.size()); + for(auto& arg: arguments) + args_ptrs.push_back(arg.c_str()); + + auto search_config = extract_search_config(args_ptrs, directory); + DirListingCache dir_cache; + auto resolved = resolve_search_config(search_config, dir_cache); + bool angled = (pctx.kind == CompletionContext::IncludeAngled); + auto candidates = complete_include_path(resolved, pctx.prefix, angled, dir_cache); + + std::vector items; + items.reserve(candidates.size()); + for(auto& c: candidates) { + protocol::CompletionItem item; + item.label = c.is_directory ? c.name + "/" : c.name; + item.kind = protocol::CompletionItemKind::File; + items.push_back(std::move(item)); } - if(pctx.kind == CompletionContext::Import) { - co_return complete_import(pctx); + auto json = et::serde::json::to_json(items); + co_return serde_raw{json ? std::move(*json) : "[]"}; + } + if(pctx.kind == CompletionContext::Import) { + auto module_names = complete_module_import(workspace.path_to_module, pctx.prefix); + + std::vector items; + items.reserve(module_names.size()); + for(auto& name: module_names) { + protocol::CompletionItem item; + item.label = name; + item.kind = protocol::CompletionItemKind::Module; + item.insert_text = name + ";"; + items.push_back(std::move(item)); } + auto json = et::serde::json::to_json(items); + co_return serde_raw{json ? std::move(*json) : "[]"}; } } - co_return co_await forward_build(worker::BuildKind::Completion, uri, position); -} - -PreambleCompletionContext Compiler::detect_completion_context(const std::string& text, - uint32_t offset) { - auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); - line_start = (line_start == std::string::npos) ? 0 : line_start + 1; - - auto line_end = text.find('\n', offset); - if(line_end == std::string::npos) - line_end = text.size(); - - auto line = llvm::StringRef(text).slice(line_start, offset); - auto trimmed = line.ltrim(); - - if(trimmed.starts_with("#")) { - auto directive = trimmed.drop_front(1).ltrim(); - if(directive.consume_front("include")) { - directive = directive.ltrim(); - if(directive.consume_front("\"")) { - return {CompletionContext::IncludeQuoted, directive.str()}; - } - if(directive.consume_front("<")) { - return {CompletionContext::IncludeAngled, directive.str()}; - } - } - return {}; - } - - auto import_check = trimmed; - if(import_check.consume_front("export") && !import_check.empty() && - !std::isalnum(import_check[0])) { - import_check = import_check.ltrim(); - } - if(import_check.consume_front("import") && - (import_check.empty() || !std::isalnum(import_check[0]))) { - import_check = import_check.ltrim(); - auto rest_of_line = llvm::StringRef(text).slice(line_start, line_end); - if(!rest_of_line.contains(';')) { - return {CompletionContext::Import, import_check.str()}; - } - } - - return {}; -} - -et::serde::RawValue Compiler::complete_include(const PreambleCompletionContext& ctx, - llvm::StringRef path) { - std::string directory; - std::vector arguments; - if(!fill_compile_args(path, directory, arguments)) - return serde_raw{"[]"}; - - std::vector args_ptrs; - args_ptrs.reserve(arguments.size()); - for(auto& arg: arguments) { - args_ptrs.push_back(arg.c_str()); - } - - auto search_config = extract_search_config(args_ptrs, directory); - DirListingCache dir_cache; - auto resolved = resolve_search_config(search_config, dir_cache); - - unsigned start_idx = 0; - if(ctx.kind == CompletionContext::IncludeAngled) { - start_idx = resolved.angled_start_idx; - } - - llvm::StringRef prefix_ref(ctx.prefix); - llvm::StringRef dir_prefix; - llvm::StringRef file_prefix = prefix_ref; - auto slash_pos = prefix_ref.rfind('/'); - if(slash_pos != llvm::StringRef::npos) { - dir_prefix = prefix_ref.slice(0, slash_pos); - file_prefix = prefix_ref.slice(slash_pos + 1, llvm::StringRef::npos); - } - - std::vector items; - llvm::StringSet<> seen; - - for(unsigned i = start_idx; i < resolved.dirs.size(); ++i) { - auto& search_dir = resolved.dirs[i]; - - const llvm::StringSet<>* entries = nullptr; - if(!dir_prefix.empty()) { - llvm::SmallString<256> sub_path(search_dir.path); - llvm::sys::path::append(sub_path, dir_prefix); - entries = resolve_dir(sub_path, dir_cache); - } else { - entries = search_dir.entries; - } - - if(!entries) - continue; - - for(auto& entry: *entries) { - auto name = entry.getKey(); - if(!name.starts_with(file_prefix)) - continue; - if(!seen.insert(name).second) - continue; - - llvm::SmallString<256> full_path(search_dir.path); - if(!dir_prefix.empty()) { - llvm::sys::path::append(full_path, dir_prefix); - } - llvm::sys::path::append(full_path, name); - - bool is_dir = false; - llvm::sys::fs::is_directory(llvm::Twine(full_path), is_dir); - - protocol::CompletionItem item; - if(is_dir) { - item.label = (name + "/").str(); - } else { - item.label = name.str(); - } - item.kind = protocol::CompletionItemKind::File; - items.push_back(std::move(item)); - } - } - - auto json = et::serde::json::to_json(items); - return serde_raw{json ? std::move(*json) : "[]"}; -} - -et::serde::RawValue Compiler::complete_import(const PreambleCompletionContext& ctx) { - std::vector items; - llvm::StringRef prefix_ref(ctx.prefix); - - for(auto& [path_id, module_name]: path_to_module) { - llvm::StringRef name_ref(module_name); - if(!name_ref.starts_with(prefix_ref)) - continue; - - protocol::CompletionItem item; - item.label = module_name; - item.kind = protocol::CompletionItemKind::Module; - item.insert_text = module_name + ";"; - items.push_back(std::move(item)); - } - - auto json = et::serde::json::to_json(items); - return serde_raw{json ? std::move(*json) : "[]"}; + co_return co_await forward_build(worker::BuildKind::Completion, position, session); } } // namespace clice diff --git a/src/server/compiler.h b/src/server/compiler.h index f3012e4b..05ba039a 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -13,12 +12,10 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/peer.h" #include "eventide/serde/serde/raw_value.h" -#include "server/compile_graph.h" -#include "server/config.h" -#include "server/indexer.h" +#include "server/session.h" #include "server/worker_pool.h" -#include "support/path_pool.h" -#include "syntax/dependency_graph.h" +#include "server/workspace.h" +#include "syntax/completion.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -30,230 +27,106 @@ namespace clice { namespace et = eventide; namespace protocol = et::ipc::protocol; -/// State of a document opened by the client. -struct DocumentState { - int version = 0; - std::string text; - std::uint64_t generation = 0; - bool ast_dirty = true; +/// Convert a file:// URI to a local file path. +std::string uri_to_path(const std::string& uri); - /// Non-null while a compile is in flight. Callers wait on the event; - /// the compile task runs independently and cannot be cancelled by LSP - /// $/cancelRequest. - struct PendingCompile { - et::event done; - bool succeeded = false; - }; - - std::shared_ptr compiling; -}; - -/// Context for compiling a header file that lacks its own CDB entry. -struct HeaderFileContext { - std::uint32_t host_path_id; /// Source file acting as host - std::string preamble_path; /// Path to generated preamble file on disk - std::uint64_t preamble_hash; /// Hash of preamble content for staleness -}; - -/// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). +/// Compilation service — drives worker processes to build ASTs, PCHs, and PCMs. /// -/// Layer 1 (fast): compare each file's current mtime against build_at. -/// If all mtimes <= build_at, the artifact is fresh (zero I/O beyond stat). +/// Compiler holds no persistent state of its own. All project-wide data +/// lives in Workspace; per-file data lives in Session. Compiler reads from +/// both and writes compilation results back to Session (file_index, pch_ref, +/// ast_deps, diagnostics). /// -/// Layer 2 (precise): for files whose mtime changed, re-hash their content -/// and compare against the stored hash. If the hash matches, the file was -/// "touched" but not actually modified — skip the rebuild. -struct DepsSnapshot { - llvm::SmallVector path_ids; - llvm::SmallVector hashes; - std::int64_t build_at = 0; -}; - -/// Cached PCH state for a single source file. -struct PCHState { - std::string path; - std::uint32_t bound = 0; - std::uint64_t hash = 0; - DepsSnapshot deps; - std::shared_ptr building; -}; - -/// Cached PCM state for a single module file. -struct PCMState { - std::string path; - DepsSnapshot deps; -}; - -enum class CompletionContext { None, IncludeQuoted, IncludeAngled, Import }; - -struct PreambleCompletionContext { - CompletionContext kind = CompletionContext::None; - std::string prefix; -}; - -/// Manages the full compilation lifecycle: document state, compilation -/// artifacts (PCH/PCM cache), compile argument resolution, header context, -/// and feature request forwarding to workers. +/// Responsibilities: +/// - AST compilation lifecycle (ensure_compiled → ensure_pch → ensure_deps) +/// - Feature request forwarding to stateful/stateless workers +/// - Compile argument resolution (CDB lookup + header context fallback) +/// - Compile graph initialization (module DAG setup) /// -/// MasterServer delegates all document and compilation operations here, -/// keeping itself as a pure LSP handler registration layer. +/// NOT responsible for: +/// - Document lifecycle (didOpen/didChange/didClose) — handled by MasterServer +/// - Index queries — handled by Indexer +/// - Background indexing scheduling — handled by Indexer class Compiler { public: Compiler(et::event_loop& loop, et::ipc::JsonPeer& peer, - PathPool& path_pool, + Workspace& workspace, WorkerPool& pool, - Indexer& indexer, - const CliceConfig& config, - CompilationDatabase& cdb, - DependencyGraph& dep_graph); - + llvm::DenseMap& sessions); ~Compiler(); - /// Convert a file:// URI to a local file path. - static std::string uri_to_path(const std::string& uri); - - /// Document lifecycle — called from MasterServer handlers. - void open_document(const std::string& uri, std::string text, int version); - void apply_changes(const protocol::DidChangeTextDocumentParams& params); - std::uint32_t close_document(const std::string& uri); - llvm::SmallVector on_save(const std::string& uri); - - /// Document accessors. - bool is_file_open(std::uint32_t path_id) const; - const DocumentState* get_document(std::uint32_t path_id) const; - - /// Cache persistence. - void load_cache(); - void save_cache(); - void cleanup_cache(int max_age_days = 7); - - /// Build path_to_module reverse mapping from dependency graph. - void build_module_map(); - - /// Initialize the CompileGraph for C++20 module compilation ordering. void init_compile_graph(); /// Fill compile arguments for a file (CDB lookup + header context fallback). + /// @param session If non-null, used for header context resolution on open files. bool fill_compile_args(llvm::StringRef path, std::string& directory, - std::vector& arguments); + std::vector& arguments, + Session* session = nullptr); - /// Fill PCM paths for all built modules (for background indexing). - void fill_pcm_deps(std::unordered_map& pcms, - std::uint32_t exclude_path_id = UINT32_MAX) const; + /// Compile an open file's AST if dirty. On success, updates session's + /// file_index, pch_ref, ast_deps, and publishes diagnostics. + et::task ensure_compiled(Session& session); - /// Pull-based compilation entry point for user-opened files. - et::task ensure_compiled(std::uint32_t path_id); - - /// Feature request forwarding to workers. using RawResult = et::task; - /// Forward a stateful AST query to the worker owning this file. - RawResult forward_query(worker::QueryKind kind, const std::string& uri); + /// Forward a query to the stateful worker that holds this file's AST. + /// Ensures compilation first. For position-sensitive queries (hover, + /// goto-definition), pass a Position. For range-sensitive queries + /// (inlay hints), pass a Range. RawResult forward_query(worker::QueryKind kind, - const std::string& uri, - const protocol::Position& position); + Session& session, + std::optional position = {}, + std::optional range = {}); - /// Forward a stateless build request (completion/signatureHelp). + /// Forward a build request (signature help, etc.) to a stateless worker. + /// Sends the full buffer content and compile arguments. RawResult forward_build(worker::BuildKind kind, - const std::string& uri, - const protocol::Position& position); + const protocol::Position& position, + Session& session); - /// Completion with preamble-aware include/import handling. - RawResult handle_completion(const std::string& uri, const protocol::Position& position); + /// Handle completion requests. Detects preamble context (include/import) + /// and serves those locally; delegates code completion to a stateless worker. + RawResult handle_completion(const protocol::Position& position, Session& session); - /// Header context management. - void switch_context(std::uint32_t path_id, std::uint32_t context_path_id); - std::optional get_active_context(std::uint32_t path_id) const; - void invalidate_host_contexts(std::uint32_t host_path_id, - llvm::SmallVectorImpl& stale_headers); + /// Send an empty diagnostics notification to clear stale markers in the editor. + void clear_diagnostics(const std::string& uri); - CompileGraph* compile_graph_ptr() { - return compile_graph.get(); - } - - const llvm::DenseMap& module_map() const { - return path_to_module; - } - - void cancel_all(); - - /// Callback invoked when indexing should be scheduled (e.g. after compile success). + /// Callback invoked when indexing should be scheduled. std::function on_indexing_needed; private: - /// Compile module dependencies, build/reuse PCH, and fill PCM paths. - et::task ensure_deps(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, + et::task ensure_deps(Session& session, const std::string& directory, const std::vector& arguments, std::pair& pch, std::unordered_map& pcms); - /// Build or reuse PCH for a source file. - et::task ensure_pch(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, + et::task ensure_pch(Session& session, const std::string& directory, const std::vector& arguments); - /// Check if a file's AST or PCH deps have changed since last compile. - bool is_stale(std::uint32_t path_id); - - /// Record dependency snapshot after a successful compile. - void record_deps(std::uint32_t path_id, llvm::ArrayRef deps); + bool is_stale(const Session& session); + void record_deps(Session& session, llvm::ArrayRef deps); void publish_diagnostics(const std::string& uri, int version, const et::serde::RawValue& diags); - void clear_diagnostics(const std::string& uri); - /// Clean up compilation state for a closed file. - void on_file_closed(std::uint32_t path_id); + std::optional resolve_header_context(std::uint32_t header_path_id, + Session* session); - /// Invalidate artifacts after a file save. - /// Returns path_ids of all files dirtied (via compile_graph cascade). - llvm::SmallVector on_file_saved(std::uint32_t path_id); - - /// Header context resolution. - std::optional resolve_header_context(std::uint32_t header_path_id); bool fill_header_context_args(llvm::StringRef path, std::uint32_t path_id, std::string& directory, - std::vector& arguments); - - /// Include/import completion helpers. - PreambleCompletionContext detect_completion_context(const std::string& text, uint32_t offset); - et::serde::RawValue complete_include(const PreambleCompletionContext& ctx, - llvm::StringRef path); - et::serde::RawValue complete_import(const PreambleCompletionContext& ctx); + std::vector& arguments, + Session* session); private: et::event_loop& loop; et::ipc::JsonPeer& peer; - PathPool& path_pool; + Workspace& workspace; WorkerPool& pool; - Indexer& indexer; - const CliceConfig& config; - CompilationDatabase& cdb; - DependencyGraph& dep_graph; - - /// Open document state, keyed by server-level path_id. - llvm::DenseMap documents; - - /// PCH/PCM cache state. - llvm::DenseMap pch_states; - llvm::DenseMap pcm_states; - llvm::DenseMap pcm_paths; - - /// Module compilation ordering. - llvm::DenseMap path_to_module; - std::unique_ptr compile_graph; - - /// Per-file compilation state. - llvm::DenseMap ast_deps; - llvm::DenseMap header_file_contexts; - llvm::DenseMap active_contexts; + llvm::DenseMap& sessions; }; } // namespace clice diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index bae5fb20..13783125 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -1,6 +1,5 @@ #include "server/indexer.h" -#include #include #include #include @@ -9,6 +8,10 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/lsp/uri.h" #include "index/tu_index.h" +#include "server/compiler.h" +#include "server/protocol.h" +#include "server/session.h" +#include "server/worker_pool.h" #include "support/filesystem.h" #include "support/logging.h" @@ -21,77 +24,18 @@ namespace clice { namespace lsp = eventide::ipc::lsp; -/// Find the tightest (innermost) occurrence containing `offset` via binary search. -const static index::Occurrence* lookup_occurrence(const std::vector& occs, - std::uint32_t offset) { - auto it = std::ranges::lower_bound(occs, offset, {}, [](const index::Occurrence& o) { - return o.range.end; - }); - const index::Occurrence* best = nullptr; - while(it != occs.end() && it->range.contains(offset)) { - if(!best || (it->range.end - it->range.begin) < (best->range.end - best->range.begin)) { - best = &*it; - } - ++it; - } - return best; -} - -// ── OpenFileIndex ──────────────────────────────────────────────────────── - -std::optional> - OpenFileIndex::find_occurrence(std::uint32_t offset) const { - if(!mapper) - return std::nullopt; - auto* occ = lookup_occurrence(file_index.occurrences, offset); - if(!occ) - return std::nullopt; - auto start = mapper->to_position(occ->range.begin); - auto end = mapper->to_position(occ->range.end); - if(!start || !end) - return std::nullopt; - return std::pair{ - occ->target, - protocol::Range{*start, *end} - }; -} - -// ── MergedIndexShard ───────────────────────────────────────────────────── - -std::optional> - MergedIndexShard::find_occurrence(std::uint32_t offset) const { - auto* m = mapper(); - if(!m) - return std::nullopt; - std::optional> result; - index.lookup(offset, [&](const index::Occurrence& o) { - auto start = m->to_position(o.range.begin); - auto end = m->to_position(o.range.end); - if(start && end) { - result = { - o.target, - protocol::Range{*start, *end} - }; - } - return false; - }); - return result; -} - -// ── Indexer: data management ───────────────────────────────────────────── - void Indexer::merge(const void* tu_index_data, std::size_t size) { auto tu_index = index::TUIndex::from(tu_index_data); if(tu_index.graph.paths.empty()) { LOG_WARN("Ignoring TUIndex with empty path graph"); return; } - auto file_ids_map = project_index.merge(tu_index); + auto file_ids_map = workspace.project_index.merge(tu_index); auto main_tu_path_id = static_cast(tu_index.graph.paths.size() - 1); auto merge_file_index = [&](std::uint32_t tu_path_id, index::FileIndex& file_idx) { auto global_path_id = file_ids_map[tu_path_id]; - auto& shard = merged_indices[global_path_id]; + auto& shard = workspace.merged_indices[global_path_id]; if(tu_path_id == main_tu_path_id) { std::vector include_locs; @@ -100,7 +44,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { remapped.path_id = file_ids_map[loc.path_id]; include_locs.push_back(remapped); } - auto file_path = project_index.path_pool.path(global_path_id); + auto file_path = workspace.project_index.path_pool.path(global_path_id); llvm::StringRef file_content; std::string file_content_storage; auto buf = llvm::MemoryBuffer::getFile(file_path); @@ -125,7 +69,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_WARN("Skip merge for path {}: include location not found", global_path_id); return; } - auto header_path = project_index.path_pool.path(global_path_id); + auto header_path = workspace.project_index.path_pool.path(global_path_id); llvm::StringRef header_content; std::string header_content_storage; auto header_buf = llvm::MemoryBuffer::getFile(header_path); @@ -146,7 +90,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_INFO("Merged TUIndex: {} paths, {} symbols, {} merged_shards", tu_index.graph.paths.size(), tu_index.symbols.size(), - merged_indices.size()); + workspace.merged_indices.size()); } void Indexer::save(llvm::StringRef index_dir) { @@ -164,7 +108,7 @@ void Indexer::save(llvm::StringRef index_dir) { std::error_code write_ec; llvm::raw_fd_ostream os(project_path, write_ec); if(!write_ec) { - project_index.serialize(os); + workspace.project_index.serialize(os); LOG_INFO("Saved ProjectIndex to {}", project_path); } else { LOG_WARN("Failed to save ProjectIndex: {}", write_ec.message()); @@ -179,7 +123,7 @@ void Indexer::save(llvm::StringRef index_dir) { } std::size_t saved = 0; - for(auto& [path_id, shard]: merged_indices) { + for(auto& [path_id, shard]: workspace.merged_indices) { if(!shard.index.need_rewrite()) continue; auto shard_path = path::join(shards_dir, std::to_string(path_id) + ".idx"); @@ -190,7 +134,7 @@ void Indexer::save(llvm::StringRef index_dir) { ++saved; } } - LOG_INFO("Saved {} MergedIndex shards (of {} total)", saved, merged_indices.size()); + LOG_INFO("Saved {} MergedIndex shards (of {} total)", saved, workspace.merged_indices.size()); } void Indexer::load(llvm::StringRef index_dir) { @@ -200,8 +144,8 @@ void Indexer::load(llvm::StringRef index_dir) { auto project_path = path::join(index_dir, "project.idx"); auto buf = llvm::MemoryBuffer::getFile(project_path); if(buf) { - project_index = index::ProjectIndex::from((*buf)->getBufferStart()); - LOG_INFO("Loaded ProjectIndex: {} symbols", project_index.symbols.size()); + workspace.project_index = index::ProjectIndex::from((*buf)->getBufferStart()); + LOG_INFO("Loaded ProjectIndex: {} symbols", workspace.project_index.symbols.size()); } auto shards_dir = path::join(index_dir, "shards"); @@ -216,62 +160,43 @@ void Indexer::load(llvm::StringRef index_dir) { std::uint32_t path_id = 0; if(stem.getAsInteger(10, path_id)) continue; - merged_indices[path_id] = MergedIndexShard{index::MergedIndex::load(it->path())}; + workspace.merged_indices[path_id] = MergedIndexShard{index::MergedIndex::load(it->path())}; } - if(!merged_indices.empty()) { - LOG_INFO("Loaded {} MergedIndex shards", merged_indices.size()); + if(!workspace.merged_indices.empty()) { + LOG_INFO("Loaded {} MergedIndex shards", workspace.merged_indices.size()); } } bool Indexer::need_update(llvm::StringRef file_path) { - auto cache_it = project_index.path_pool.find(file_path); - if(cache_it == project_index.path_pool.cache.end()) + auto cache_it = workspace.project_index.path_pool.find(file_path); + if(cache_it == workspace.project_index.path_pool.cache.end()) return true; - auto merged_it = merged_indices.find(cache_it->second); - if(merged_it == merged_indices.end()) + auto merged_it = workspace.merged_indices.find(cache_it->second); + if(merged_it == workspace.merged_indices.end()) return true; llvm::SmallVector path_mapping; - for(auto& p: project_index.path_pool.paths) { + for(auto& p: workspace.project_index.path_pool.paths) { path_mapping.push_back(p); } return merged_it->second.index.need_update(path_mapping); } -void Indexer::set_open_file(std::uint32_t server_path_id, - llvm::StringRef file_path, - OpenFileIndex index) { - auto& stored = (open_file_indices[server_path_id] = std::move(index)); - stored.mapper.emplace(stored.content, lsp::PositionEncoding::UTF16); - auto proj_cache_it = project_index.path_pool.find(file_path); - if(proj_cache_it != project_index.path_pool.cache.end()) { - open_proj_path_ids.insert(proj_cache_it->second); - } -} - -void Indexer::remove_open_file(std::uint32_t server_path_id, llvm::StringRef file_path) { - open_file_indices.erase(server_path_id); - auto proj_cache_it = project_index.path_pool.find(file_path); - if(proj_cache_it != project_index.path_pool.cache.end()) { - open_proj_path_ids.erase(proj_cache_it->second); - } -} - -// ── Indexer: symbol queries ────────────────────────────────────────────── - bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const { - for(auto& [_, index]: open_file_indices) { - auto it = index.symbols.find(hash); - if(it != index.symbols.end()) { + for(auto& [_, session]: sessions) { + if(!session.file_index) + continue; + auto it = session.file_index->symbols.find(hash); + if(it != session.file_index->symbols.end()) { name = it->second.name; kind = it->second.kind; return true; } } - auto it = project_index.symbols.find(hash); - if(it != project_index.symbols.end()) { + auto it = workspace.project_index.symbols.find(hash); + if(it != workspace.project_index.symbols.end()) { name = it->second.name; kind = it->second.kind; return true; @@ -280,13 +205,11 @@ bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, Symbol } Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text) { - // Try open file index first. - auto it = open_file_indices.find(server_path_id); - if(it != open_file_indices.end()) { - auto& index = it->second; + Session* session) { + // Try the session's open file index first. + if(session && session->file_index) { + auto& index = *session->file_index; if(!index.mapper) return {}; auto offset = index.mapper->to_offset(position); @@ -297,7 +220,8 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, return {}; } - // Fallback to MergedIndex, using doc_text for position → offset. + // Fallback to MergedIndex, using session text (or reading from disk) for position -> offset. + const std::string* doc_text = session ? &session->text : nullptr; if(!doc_text) return {}; lsp::PositionMapper doc_mapper(*doc_text, lsp::PositionEncoding::UTF16); @@ -305,11 +229,11 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, if(!offset) return {}; - auto proj_it = project_index.path_pool.find(path); - if(proj_it == project_index.path_pool.cache.end()) + auto proj_it = workspace.project_index.path_pool.find(path); + if(proj_it == workspace.project_index.path_pool.cache.end()) return {}; - auto shard_it = merged_indices.find(proj_it->second); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(proj_it->second); + if(shard_it == workspace.merged_indices.end()) return {}; if(auto found = shard_it->second.find_occurrence(*offset)) @@ -318,25 +242,24 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, } std::vector Indexer::query_relations(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, RelationKind kind, - const std::string* doc_text) { - auto hit = resolve_cursor(path, server_path_id, position, doc_text); + Session* session) { + auto hit = resolve_cursor(path, position, session); if(hit.hash == 0) return {}; std::vector locations; - auto sym_it = project_index.symbols.find(hit.hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hit.hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - auto uri = lsp::URI::from_file_path(project_index.path_pool.path(file_id)); + auto uri = lsp::URI::from_file_path(workspace.project_index.path_pool.path(file_id)); if(!uri) continue; shard_it->second.find_relations(hit.hash, @@ -348,11 +271,13 @@ std::vector Indexer::query_relations(llvm::StringRef path, } } - for(auto& [id, index]: open_file_indices) { - auto uri = lsp::URI::from_file_path(std::string(path_pool.resolve(id))); + for(auto& [id, sess]: sessions) { + if(!sess.file_index) + continue; + auto uri = lsp::URI::from_file_path(std::string(workspace.path_pool.resolve(id))); if(!uri) continue; - index.find_relations(hit.hash, kind, [&](const auto&, protocol::Range range) { + sess.file_index->find_relations(hit.hash, kind, [&](const auto&, protocol::Range range) { locations.push_back({uri->str(), range}); return true; }); @@ -363,10 +288,9 @@ std::vector Indexer::query_relations(llvm::StringRef path, std::optional Indexer::lookup_symbol(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text) { - auto hit = resolve_cursor(path, server_path_id, position, doc_text); + Session* session) { + auto hit = resolve_cursor(path, position, session); if(hit.hash == 0) return std::nullopt; @@ -380,33 +304,35 @@ std::optional Indexer::lookup_symbol(const std::string& uri, std::optional Indexer::find_definition_location(index::SymbolHash hash) { // Open file indices first (fresher data for actively-edited files). - for(auto& [id, index]: open_file_indices) { - auto uri = lsp::URI::from_file_path(std::string(path_pool.resolve(id))); + for(auto& [id, sess]: sessions) { + if(!sess.file_index) + continue; + auto uri = lsp::URI::from_file_path(std::string(workspace.path_pool.resolve(id))); if(!uri) continue; std::optional result; - index.find_relations(hash, - RelationKind::Definition, - [&](const auto&, protocol::Range range) { - result = protocol::Location{uri->str(), range}; - return false; - }); + sess.file_index->find_relations(hash, + RelationKind::Definition, + [&](const auto&, protocol::Range range) { + result = protocol::Location{uri->str(), range}; + return false; + }); if(result) return result; } // Fall back to ProjectIndex reference files. - auto sym_it = project_index.symbols.find(hash); - if(sym_it == project_index.symbols.end()) + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it == workspace.project_index.symbols.end()) return std::nullopt; for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - auto uri = lsp::URI::from_file_path(project_index.path_pool.path(file_id)); + auto uri = lsp::URI::from_file_path(workspace.project_index.path_pool.path(file_id)); if(!uri) continue; std::optional result; @@ -426,10 +352,9 @@ std::optional Indexer::find_definition_location(index::Symbo std::optional Indexer::resolve_hierarchy_item(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Range& range, const std::optional& data, - const std::string* doc_text) { + Session* session) { if(data) { if(auto* int_val = std::get_if(&*data)) { auto hash = static_cast(*int_val); @@ -440,22 +365,20 @@ std::optional } } } - return lookup_symbol(uri, path, server_path_id, range.start, doc_text); + return lookup_symbol(uri, path, range.start, session); } -// ── Indexer: relation collection helpers ───────────────────────────────── - void Indexer::collect_grouped_relations( index::SymbolHash hash, RelationKind kind, llvm::DenseMap>& target_ranges) { - auto sym_it = project_index.symbols.find(hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; shard_it->second.find_relations(hash, kind, [&](const auto& r, protocol::Range range) { target_ranges[r.target_symbol].push_back(range); @@ -463,8 +386,10 @@ void Indexer::collect_grouped_relations( }); } } - for(auto& [_, index]: open_file_indices) { - index.find_relations(hash, kind, [&](const auto& r, protocol::Range range) { + for(auto& [_, sess]: sessions) { + if(!sess.file_index) + continue; + sess.file_index->find_relations(hash, kind, [&](const auto& r, protocol::Range range) { target_ranges[r.target_symbol].push_back(range); return true; }); @@ -475,15 +400,15 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, RelationKind kind, llvm::SmallVectorImpl& targets) { llvm::DenseSet seen; - auto sym_it = project_index.symbols.find(hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - /// No position conversion needed — just collect target symbol hashes. + /// No position conversion needed -- just collect target symbol hashes. shard_it->second.index.lookup(hash, kind, [&](const index::Relation& r) { if(seen.insert(r.target_symbol).second) { targets.push_back(r.target_symbol); @@ -492,9 +417,11 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, }); } } - for(auto& [_, index]: open_file_indices) { - auto rel_it = index.file_index.relations.find(hash); - if(rel_it == index.file_index.relations.end()) + for(auto& [_, sess]: sessions) { + if(!sess.file_index) + continue; + auto rel_it = sess.file_index->file_index.relations.find(hash); + if(rel_it == sess.file_index->file_index.relations.end()) continue; for(auto& r: rel_it->second) { if(r.kind & kind) { @@ -506,8 +433,6 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, } } -// ── Indexer: hierarchy queries ─────────────────────────────────────────── - /// Resolve a symbol hash into a SymbolInfo with definition location. /// Returns nullopt if the symbol or its definition cannot be found. std::optional Indexer::resolve_symbol(index::SymbolHash hash) { @@ -603,7 +528,7 @@ std::vector Indexer::search_symbols(llvm::StringRef std::vector results; llvm::DenseSet seen; - for(auto& [hash, symbol]: project_index.symbols) { + for(auto& [hash, symbol]: workspace.project_index.symbols) { if(results.size() >= max_results) break; if(!is_indexable_kind(symbol.kind) || symbol.name.empty()) @@ -622,10 +547,12 @@ std::vector Indexer::search_symbols(llvm::StringRef seen.insert(hash); } - for(auto& [_, index]: open_file_indices) { + for(auto& [_, sess]: sessions) { if(results.size() >= max_results) break; - for(auto& [hash, symbol]: index.symbols) { + if(!sess.file_index) + continue; + for(auto& [hash, symbol]: sess.file_index->symbols) { if(results.size() >= max_results) break; if(seen.contains(hash)) @@ -649,8 +576,6 @@ std::vector Indexer::search_symbols(llvm::StringRef return results; } -// ── Indexer: static utilities ──────────────────────────────────────────── - protocol::SymbolKind Indexer::to_lsp_symbol_kind(SymbolKind kind) { switch(kind) { case SymbolKind::Namespace: return protocol::SymbolKind::Namespace; @@ -695,4 +620,77 @@ protocol::TypeHierarchyItem Indexer::build_type_hierarchy_item(const SymbolInfo& return item; } +void Indexer::enqueue(std::uint32_t server_path_id) { + index_queue.push_back(server_path_id); +} + +void Indexer::schedule() { + if(!workspace.config.enable_indexing || indexing_active || indexing_scheduled) + return; + indexing_scheduled = true; + + if(!index_idle_timer) { + index_idle_timer = std::make_shared(et::timer::create(loop)); + } + index_idle_timer->start(std::chrono::milliseconds(workspace.config.idle_timeout_ms)); + loop.schedule(run_background_indexing()); +} + +et::task<> Indexer::run_background_indexing() { + if(index_idle_timer) { + co_await index_idle_timer->wait(); + } + indexing_scheduled = false; + + if(index_queue_pos >= index_queue.size()) { + LOG_DEBUG("Background indexing: queue exhausted"); + co_return; + } + + indexing_active = true; + std::size_t processed = 0; + + while(index_queue_pos < index_queue.size()) { + auto server_path_id = index_queue[index_queue_pos]; + index_queue_pos++; + + auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); + + if(sessions.contains(server_path_id)) + continue; + + if(!need_update(file_path)) + continue; + + worker::BuildParams params; + params.kind = worker::BuildKind::Index; + params.file = file_path; + if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) + continue; + + workspace.fill_pcm_deps(params.pcms); + + LOG_INFO("Background indexing: {}", file_path); + + auto result = co_await pool.send_stateless(params); + if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { + LOG_INFO("Background indexing got TUIndex for {}: {} bytes", + file_path, + result.value().tu_index_data.size()); + merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); + ++processed; + } else if(result.has_value() && !result.value().success) { + LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); + } else if(result.has_value() && result.value().tu_index_data.empty()) { + LOG_WARN("Background index returned empty TUIndex for {}", file_path); + } else { + LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); + } + } + + indexing_active = false; + LOG_INFO("Background indexing complete: {} files processed", processed); + save(workspace.config.index_dir); +} + } // namespace clice diff --git a/src/server/indexer.h b/src/server/indexer.h index 214e10d8..921bc73e 100644 --- a/src/server/indexer.h +++ b/src/server/indexer.h @@ -1,21 +1,20 @@ #pragma once #include +#include +#include #include #include -#include #include +#include "eventide/async/async.h" #include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" -#include "index/merged_index.h" -#include "index/project_index.h" #include "semantic/relation_kind.h" #include "semantic/symbol_kind.h" -#include "support/path_pool.h" +#include "server/workspace.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -25,88 +24,9 @@ namespace et = eventide; namespace protocol = et::ipc::protocol; namespace lsp = et::ipc::lsp; -/// In-memory index for an open file. Kept separate from MergedIndex because -/// open files change frequently, are based on unsaved buffer content, and only -/// need to track the main file (headers are covered by PCH/PCM indexing). -struct OpenFileIndex { - index::FileIndex file_index; - index::SymbolTable symbols; - std::string content; ///< Buffer text at index time (for position mapping). - - /// Cached PositionMapper built from `content`. Avoids re-scanning line - /// offsets on every query. Initialized by Indexer::set_open_file(). - std::optional mapper; - - /// Find the tightest occurrence containing `offset`. - /// Returns (symbol_hash, LSP range) with positions already converted. - std::optional> - find_occurrence(std::uint32_t offset) const; - - /// Iterate relations matching `kind`, calling back with pre-converted ranges. - /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). - template - void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { - if(!mapper) - return; - auto it = file_index.relations.find(hash); - if(it == file_index.relations.end()) - return; - for(auto& r: it->second) { - if(r.kind & kind) { - auto start = mapper->to_position(r.range.begin); - auto end = mapper->to_position(r.range.end); - if(start && end) { - if(!fn(r, protocol::Range{*start, *end})) - return; - } - } - } - } -}; - -/// Wraps index::MergedIndex with a lazily-cached PositionMapper. -struct MergedIndexShard { - index::MergedIndex index; - mutable std::optional cached_mapper; - - /// Get or lazily build a PositionMapper from the index's stored content. - const lsp::PositionMapper* mapper() const { - if(!cached_mapper) { - auto c = index.content(); - if(!c.empty()) { - cached_mapper.emplace(c, lsp::PositionEncoding::UTF16); - } - } - return cached_mapper ? &*cached_mapper : nullptr; - } - - /// Invalidate the cached mapper (call after merge changes content). - void invalidate_mapper() { - cached_mapper.reset(); - } - - /// Find occurrence at byte offset. - /// Returns (symbol_hash, LSP range) with positions already converted. - std::optional> - find_occurrence(std::uint32_t offset) const; - - /// Iterate relations matching `kind`, calling back with pre-converted ranges. - /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). - template - void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { - auto* m = mapper(); - if(!m) - return; - index.lookup(hash, kind, [&](const index::Relation& r) { - auto start = m->to_position(r.range.begin); - auto end = m->to_position(r.range.end); - if(start && end) { - return fn(r, protocol::Range{*start, *end}); - } - return true; - }); - } -}; +struct Session; +class Compiler; +class WorkerPool; /// Information about a symbol at a given position. struct SymbolInfo { @@ -117,49 +37,63 @@ struct SymbolInfo { protocol::Range range; }; -/// Owns all index state (ProjectIndex, MergedIndex shards, open file indices) -/// and provides query methods for cross-file navigation. +/// Index query layer and background indexing scheduler. /// -/// Background indexing scheduling is driven by MasterServer; Indexer is the -/// pure data + query layer. +/// Indexer holds no index data of its own. All persistent data lives in +/// Workspace (disk-derived ProjectIndex + MergedIndex shards) and per-file +/// data lives in Session (OpenFileIndex from unsaved buffers). +/// +/// Responsibilities: +/// - Cross-file navigation queries (definition, references, hierarchy) +/// - Symbol search (workspace/symbol) +/// - Background indexing scheduling (enqueue → idle timer → worker dispatch) +/// - Merging TUIndex results into Workspace's ProjectIndex +/// +/// NOT responsible for: +/// - Compilation — handled by Compiler +/// - Document lifecycle — handled by MasterServer class Indexer { public: - explicit Indexer(PathPool& path_pool) : path_pool(path_pool) {} + Indexer(et::event_loop& loop, + Workspace& workspace, + llvm::DenseMap& sessions, + WorkerPool& pool, + Compiler& compiler, + std::function is_file_open = {}) : + loop(loop), workspace(workspace), sessions(sessions), pool(pool), compiler(compiler), + is_file_open(std::move(is_file_open)) {} - /// Merge a TUIndex result into ProjectIndex and MergedIndex shards. + /// Add a file to the background indexing queue. + void enqueue(std::uint32_t server_path_id); + + /// Schedule background indexing (respects idle timeout and dedup). + void schedule(); + + /// Merge a TUIndex result into Workspace's ProjectIndex and MergedIndex shards. void merge(const void* tu_index_data, std::size_t size); - /// Save ProjectIndex and MergedIndex shards to disk. + /// Save Workspace's ProjectIndex and MergedIndex shards to disk. void save(llvm::StringRef index_dir); - /// Load ProjectIndex and MergedIndex shards from disk. + /// Load Workspace's ProjectIndex and MergedIndex shards from disk. void load(llvm::StringRef index_dir); /// Check whether a file needs re-indexing (stale or missing shard). bool need_update(llvm::StringRef file_path); - /// Store or replace the open file index for a server-level path_id. - void set_open_file(std::uint32_t server_path_id, - llvm::StringRef file_path, - OpenFileIndex index); - - /// Remove the open file index and untrack project-level path_id. - void remove_open_file(std::uint32_t server_path_id, llvm::StringRef file_path); - /// Query relations (Definition, Reference, etc.) for a symbol at cursor. - /// @param doc_text Fallback text when the file has no open file index yet. + /// @param session Active Session for this file, or nullptr to use MergedIndex only. std::vector query_relations(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, RelationKind kind, - const std::string* doc_text); + Session* session); /// Look up symbol info (hash, name, kind, range) at a cursor position. + /// @param session Active Session for this file, or nullptr. std::optional lookup_symbol(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text); + Session* session); /// Find the definition location of a symbol by hash. std::optional find_definition_location(index::SymbolHash hash); @@ -168,12 +102,12 @@ public: bool find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const; /// Resolve a hierarchy item (from stored data or by position lookup). + /// @param session Active Session for this file, or nullptr. std::optional resolve_hierarchy_item(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Range& range, const std::optional& data, - const std::string* doc_text); + Session* session); /// Find incoming calls to a function. std::vector find_incoming_calls(index::SymbolHash hash); @@ -198,11 +132,6 @@ public: static protocol::CallHierarchyItem build_call_hierarchy_item(const SymbolInfo& info); static protocol::TypeHierarchyItem build_type_hierarchy_item(const SymbolInfo& info); - /// Direct access to ProjectIndex for background indexing. - index::ProjectIndex& project_index_ref() { - return project_index; - } - private: /// Result of resolving a symbol at a cursor position. struct CursorHit { @@ -210,12 +139,11 @@ private: protocol::Range range{}; }; - /// Resolve the symbol at (position), checking open file index first then - /// falling back to MergedIndex. + /// Resolve the symbol at (position), checking Session's file_index first + /// then falling back to Workspace's MergedIndex. CursorHit resolve_cursor(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text); + Session* session); /// Collect relations grouped by target symbol, across all index sources. void collect_grouped_relations( @@ -231,21 +159,31 @@ private: /// Resolve a symbol hash into a SymbolInfo with definition location. std::optional resolve_symbol(index::SymbolHash hash); + /// Check whether a project-level path_id has an active Session. + bool is_proj_path_open(std::uint32_t proj_path_id) const { + return is_file_open && is_file_open(proj_path_id); + } + private: - PathPool& path_pool; + et::event_loop& loop; + Workspace& workspace; + llvm::DenseMap& sessions; + WorkerPool& pool; + Compiler& compiler; - /// Global symbol table and path pool shared across all TUs. - index::ProjectIndex project_index; + /// Callback that checks if a *project-level* path_id has an active + /// Session. Set by the owner (e.g. MasterServer) to bridge the + /// server-path-id-keyed sessions map to project-level path_ids. + std::function is_file_open; - /// Per-file MergedIndex shards (keyed by project-level path_id). - llvm::DenseMap merged_indices; + /// Background indexing queue and scheduling state. + std::vector index_queue; + std::size_t index_queue_pos = 0; + bool indexing_active = false; + bool indexing_scheduled = false; + std::shared_ptr index_idle_timer; - /// In-memory indices for currently open files (keyed by server-level path_id). - llvm::DenseMap open_file_indices; - - /// Project-level path_ids of open files, used to skip stale MergedIndex - /// shards when fresher OpenFileIndex data is available. - llvm::DenseSet open_proj_path_ids; + et::task<> run_background_indexing(); }; } // namespace clice diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index efe85988..6a9302e0 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -3,7 +3,10 @@ #include #include #include +#include +#include +#include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/lsp/uri.h" #include "eventide/reflection/enum.h" @@ -34,8 +37,19 @@ static serde_raw to_raw(const T& value) { } MasterServer::MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path) : - loop(loop), peer(peer), pool(loop), indexer(path_pool), - compiler(loop, peer, path_pool, pool, indexer, config, cdb, dependency_graph), + loop(loop), peer(peer), pool(loop), compiler(loop, peer, workspace, pool, sessions), + indexer(loop, + workspace, + sessions, + pool, + compiler, + [this](uint32_t proj_path_id) { + // Bridge project-level path_id to server-level path_id. + // The two PathPools may assign different IDs to the same path. + auto path = workspace.project_index.path_pool.path(proj_path_id); + auto server_id = workspace.path_pool.intern(path); + return sessions.contains(server_id); + }), self_path(std::move(self_path)) {} MasterServer::~MasterServer() = default; @@ -44,16 +58,18 @@ et::task<> MasterServer::load_workspace() { if(workspace_root.empty()) co_return; - if(!config.cache_dir.empty()) { - auto ec = llvm::sys::fs::create_directories(config.cache_dir); + if(!workspace.config.cache_dir.empty()) { + auto ec = llvm::sys::fs::create_directories(workspace.config.cache_dir); if(ec) { - LOG_WARN("Failed to create cache directory {}: {}", config.cache_dir, ec.message()); + LOG_WARN("Failed to create cache directory {}: {}", + workspace.config.cache_dir, + ec.message()); } else { - LOG_INFO("Cache directory: {}", config.cache_dir); + LOG_INFO("Cache directory: {}", workspace.config.cache_dir); } for(auto* subdir: {"cache/pch", "cache/pcm"}) { - auto dir = path::join(config.cache_dir, subdir); + auto dir = path::join(workspace.config.cache_dir, subdir); auto ec2 = llvm::sys::fs::create_directories(dir); if(ec2) { LOG_WARN("Failed to create {}: {}", dir, ec2.message()); @@ -62,17 +78,17 @@ et::task<> MasterServer::load_workspace() { // Clean up stale files first, then load — load_cache() only restores // entries still listed in cache.json, so cleanup won't delete live files. - compiler.cleanup_cache(); - compiler.load_cache(); + workspace.cleanup_cache(); + workspace.load_cache(); } std::string cdb_path; - if(!config.compile_commands_path.empty()) { - if(llvm::sys::fs::exists(config.compile_commands_path)) { - cdb_path = config.compile_commands_path; + if(!workspace.config.compile_commands_path.empty()) { + if(llvm::sys::fs::exists(workspace.config.compile_commands_path)) { + cdb_path = workspace.config.compile_commands_path; } else { LOG_WARN("Configured compile_commands_path not found: {}", - config.compile_commands_path); + workspace.config.compile_commands_path); } } @@ -91,11 +107,11 @@ et::task<> MasterServer::load_workspace() { co_return; } - auto count = cdb.load(cdb_path); + auto count = workspace.cdb.load(cdb_path); LOG_INFO("Loaded CDB from {} with {} entries", cdb_path, count); - auto report = scan_dependency_graph(cdb, path_pool, dependency_graph); - dependency_graph.build_reverse_map(); + auto report = scan_dependency_graph(workspace.cdb, workspace.path_pool, workspace.dep_graph); + workspace.dep_graph.build_reverse_map(); auto unresolved = report.includes_found - report.includes_resolved; double accuracy = @@ -117,95 +133,21 @@ et::task<> MasterServer::load_workspace() { LOG_WARN("{} unresolved includes", unresolved); } - compiler.build_module_map(); - indexer.load(config.index_dir); + workspace.build_module_map(); + indexer.load(workspace.config.index_dir); - if(config.enable_indexing) { - for(auto& entry: cdb.get_entries()) { - auto file = cdb.resolve_path(entry.file); - auto server_id = path_pool.intern(file); - index_queue.push_back(server_id); - } - if(!index_queue.empty()) { - LOG_INFO("Queued {} files for background indexing", index_queue.size()); - schedule_indexing(); + if(workspace.config.enable_indexing) { + for(auto& entry: workspace.cdb.get_entries()) { + auto file = workspace.cdb.resolve_path(entry.file); + auto server_id = workspace.path_pool.intern(file); + indexer.enqueue(server_id); } + indexer.schedule(); } compiler.init_compile_graph(); } -void MasterServer::schedule_indexing() { - if(!config.enable_indexing || indexing_active || indexing_scheduled) - return; - indexing_scheduled = true; - - if(!index_idle_timer) { - index_idle_timer = std::make_shared(et::timer::create(loop)); - } - index_idle_timer->start(std::chrono::milliseconds(config.idle_timeout_ms)); - loop.schedule(run_background_indexing()); -} - -et::task<> MasterServer::run_background_indexing() { - if(index_idle_timer) { - co_await index_idle_timer->wait(); - } - indexing_scheduled = false; - - if(index_queue_pos >= index_queue.size()) { - LOG_DEBUG("Background indexing: queue exhausted"); - co_return; - } - - indexing_active = true; - std::size_t processed = 0; - - while(index_queue_pos < index_queue.size()) { - auto server_path_id = index_queue[index_queue_pos]; - index_queue_pos++; - - auto file_path = std::string(path_pool.resolve(server_path_id)); - - /// Skip open files — their index comes from the stateful worker. - if(compiler.is_file_open(server_path_id)) { - continue; - } - - if(!indexer.need_update(file_path)) - continue; - - worker::BuildParams params; - params.kind = worker::BuildKind::Index; - params.file = file_path; - if(!compiler.fill_compile_args(file_path, params.directory, params.arguments)) - continue; - - compiler.fill_pcm_deps(params.pcms); - - LOG_INFO("Background indexing: {}", file_path); - - auto result = co_await pool.send_stateless(params); - if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { - LOG_INFO("Background indexing got TUIndex for {}: {} bytes", - file_path, - result.value().tu_index_data.size()); - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - ++processed; - } else if(result.has_value() && !result.value().success) { - LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); - } else if(result.has_value() && result.value().tu_index_data.empty()) { - LOG_WARN("Background index returned empty TUIndex for {}", file_path); - } else { - LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); - } - } - - indexing_active = false; - LOG_INFO("Background indexing complete: {} files processed", processed); - indexer.save(config.index_dir); -} - void MasterServer::register_handlers() { using StringVec = std::vector; @@ -217,7 +159,7 @@ void MasterServer::register_handlers() { auto& init = params.lsp__initialize_params; if(init.root_uri.has_value()) { - workspace_root = Compiler::uri_to_path(*init.root_uri); + workspace_root = uri_to_path(*init.root_uri); } lifecycle = ServerLifecycle::Initialized; @@ -300,27 +242,27 @@ void MasterServer::register_handlers() { }); peer.on_notification([this](const protocol::InitializedParams& params) { - config = CliceConfig::load_from_workspace(workspace_root); + workspace.config = CliceConfig::load_from_workspace(workspace_root); - if(!config.logging_dir.empty()) { + if(!workspace.config.logging_dir.empty()) { auto now = std::chrono::system_clock::now(); auto pid = llvm::sys::Process::getProcessId(); - auto session_dir = - path::join(config.logging_dir, std::format("{:%Y-%m-%d_%H-%M-%S}_{}", now, pid)); + auto session_dir = path::join(workspace.config.logging_dir, + std::format("{:%Y-%m-%d_%H-%M-%S}_{}", now, pid)); logging::file_logger("master", session_dir, logging::options); session_log_dir = session_dir; } LOG_INFO("Server ready (stateful={}, stateless={}, idle={}ms)", - config.stateful_worker_count, - config.stateless_worker_count, - config.idle_timeout_ms); + workspace.config.stateful_worker_count, + workspace.config.stateless_worker_count, + workspace.config.idle_timeout_ms); WorkerPoolOptions pool_opts; pool_opts.self_path = self_path; - pool_opts.stateful_count = config.stateful_worker_count; - pool_opts.stateless_count = config.stateless_worker_count; - pool_opts.worker_memory_limit = config.worker_memory_limit; + pool_opts.stateful_count = workspace.config.stateful_worker_count; + pool_opts.stateless_count = workspace.config.stateless_worker_count; + pool_opts.worker_memory_limit = workspace.config.worker_memory_limit; pool_opts.log_dir = session_log_dir; if(!pool.start(pool_opts)) { LOG_ERROR("Failed to start worker pool"); @@ -330,7 +272,7 @@ void MasterServer::register_handlers() { lifecycle = ServerLifecycle::Ready; compiler.on_indexing_needed = [this]() { - schedule_indexing(); + indexer.schedule(); }; loop.schedule(load_workspace()); @@ -348,8 +290,8 @@ void MasterServer::register_handlers() { lifecycle = ServerLifecycle::Exited; LOG_INFO("Exit notification received"); - indexer.save(config.index_dir); - compiler.save_cache(); + indexer.save(workspace.config.index_dir); + workspace.save_cache(); loop.schedule([this]() -> et::task<> { co_await pool.stop(); @@ -357,105 +299,225 @@ void MasterServer::register_handlers() { }()); }); - /// Document lifecycle — delegate to Compiler. + /// Document lifecycle — handled directly by MasterServer. + peer.on_notification([this](const protocol::DidOpenTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - compiler.open_document(params.text_document.uri, - params.text_document.text, - params.text_document.version); + + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto [it, _] = sessions.try_emplace(path_id); + auto& session = it->second; + session.path_id = path_id; + session.version = params.text_document.version; + session.text = params.text_document.text; + session.generation++; + + LOG_DEBUG("didOpen: {} (v{})", path, params.text_document.version); }); peer.on_notification([this](const protocol::DidChangeTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - compiler.apply_changes(params); + + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto it = sessions.find(path_id); + if(it == sessions.end()) + return; + + auto& session = it->second; + session.version = params.text_document.version; + + for(auto& change: params.content_changes) { + std::visit( + [&](auto& c) { + using T = std::remove_cvref_t; + if constexpr(std::is_same_v) { + session.text = c.text; + } else { + auto& range = c.range; + lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); + auto start = mapper.to_offset(range.start); + auto end = mapper.to_offset(range.end); + if(start && end && *start <= *end) { + session.text.replace(*start, *end - *start, c.text); + } + } + }, + change); + } + + session.generation++; + session.ast_dirty = true; + + LOG_DEBUG("didChange: path={} version={} gen={}", + path, + session.version, + session.generation); + + worker::DocumentUpdateParams update; + update.path = path; + update.version = session.version; + pool.notify_stateful(path_id, update); }); peer.on_notification([this](const protocol::DidCloseTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - auto path_id = compiler.close_document(params.text_document.uri); - index_queue.push_back(path_id); - schedule_indexing(); + + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + workspace.on_file_closed(path_id); + pool.notify_stateful(path_id, worker::EvictParams{path}); + + // Clear diagnostics for the closed file. + protocol::PublishDiagnosticsParams diag_params; + diag_params.uri = params.text_document.uri; + peer.send_notification(diag_params); + + sessions.erase(path_id); + + indexer.enqueue(path_id); + indexer.schedule(); + + LOG_DEBUG("didClose: {}", path); }); peer.on_notification([this](const protocol::DidSaveTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - auto to_index = compiler.on_save(params.text_document.uri); - for(auto id: to_index) - index_queue.push_back(id); - schedule_indexing(); + + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto dirtied = workspace.on_file_saved(path_id); + for(auto dirty_id: dirtied) { + if(auto sit = sessions.find(dirty_id); sit != sessions.end()) { + sit->second.ast_dirty = true; + } else { + indexer.enqueue(dirty_id); + } + } + + // Invalidate header contexts for sessions whose host is this file. + for(auto& [hdr_id, session]: sessions) { + if(session.header_context && session.header_context->host_path_id == path_id) { + session.header_context.reset(); + session.ast_dirty = true; + } + } + + indexer.schedule(); + + LOG_DEBUG("didSave: {}", path); }); /// Feature requests — stateful forwarding. + peer.on_request([this](RequestContext& ctx, const protocol::HoverParams& params) -> RawResult { - co_return co_await compiler.forward_query( - worker::QueryKind::Hover, - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); + auto path = uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::Hover, + sit->second, + params.text_document_position_params.position); + }); + + peer.on_request([this](RequestContext& ctx, + const protocol::SemanticTokensParams& params) -> RawResult { + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::SemanticTokens, sit->second); }); - peer.on_request( - [this](RequestContext& ctx, const protocol::SemanticTokensParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::SemanticTokens, - params.text_document.uri); - }); - peer.on_request( [this](RequestContext& ctx, const protocol::InlayHintParams& params) -> RawResult { + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; co_return co_await compiler.forward_query(worker::QueryKind::InlayHints, - params.text_document.uri); + sit->second, + {}, + params.range); }); peer.on_request( [this](RequestContext& ctx, const protocol::FoldingRangeParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::FoldingRange, - params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::FoldingRange, sit->second); }); - peer.on_request( - [this](RequestContext& ctx, const protocol::DocumentSymbolParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::DocumentSymbol, - params.text_document.uri); - }); + peer.on_request([this](RequestContext& ctx, + const protocol::DocumentSymbolParams& params) -> RawResult { + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::DocumentSymbol, sit->second); + }); peer.on_request( [this](RequestContext& ctx, const protocol::DocumentLinkParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::DocumentLink, - params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::DocumentLink, sit->second); }); peer.on_request( [this](RequestContext& ctx, const protocol::CodeActionParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::CodeAction, - params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::CodeAction, sit->second); }); - /// Resolve URI to the context needed for index queries. + /// Helper: resolve URI to path, path_id, and Session pointer. auto resolve_uri = [this](const std::string& uri) { struct Result { std::string path; std::uint32_t path_id; - const std::string* doc_text; + Session* session; }; - auto path = Compiler::uri_to_path(uri); - auto path_id = path_pool.intern(path); - auto* doc = compiler.get_document(path_id); - return Result{std::move(path), path_id, doc ? &doc->text : nullptr}; + auto path = uri_to_path(uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + Session* session = (sit != sessions.end()) ? &sit->second : nullptr; + return Result{std::move(path), path_id, session}; }; auto lookup_at = [this, resolve_uri](const std::string& uri, const protocol::Position& pos) { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.lookup_symbol(uri, path, path_id, pos, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.lookup_symbol(uri, path, pos, session); }; auto query_at = [this, resolve_uri](const std::string& uri, const protocol::Position& pos, RelationKind kind) -> std::vector { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.query_relations(path, path_id, pos, kind, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.query_relations(path, pos, kind, session); }; auto resolve_item = @@ -463,11 +525,12 @@ void MasterServer::register_handlers() { resolve_uri](const std::string& uri, const protocol::Range& range, const std::optional& data) -> std::optional { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.resolve_hierarchy_item(uri, path, path_id, range, data, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.resolve_hierarchy_item(uri, path, range, data, session); }; /// Feature requests — index-based with AST fallback. + peer.on_request([this, query_at](RequestContext& ctx, const protocol::DefinitionParams& params) -> RawResult { auto& uri = params.text_document_position_params.text_document.uri; @@ -478,7 +541,14 @@ void MasterServer::register_handlers() { co_return to_raw(result); } - co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition, uri, pos); + auto path = uri_to_path(uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition, + sit->second, + pos); }); peer.on_request([this, query_at](RequestContext& ctx, @@ -516,22 +586,32 @@ void MasterServer::register_handlers() { }); /// Feature requests — stateless forwarding. - peer.on_request( - [this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult { - co_return co_await compiler.handle_completion( - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); - }); + + peer.on_request([this](RequestContext& ctx, + const protocol::CompletionParams& params) -> RawResult { + auto path = uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.handle_completion(params.text_document_position_params.position, + sit->second); + }); peer.on_request( [this](RequestContext& ctx, const protocol::SignatureHelpParams& params) -> RawResult { - co_return co_await compiler.forward_build( - worker::BuildKind::SignatureHelp, - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); + auto path = uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp, + params.text_document_position_params.position, + sit->second); }); /// Hierarchy queries — index-based. + peer.on_request( [this, lookup_at](RequestContext& ctx, const protocol::CallHierarchyPrepareParams& params) -> RawResult { @@ -624,21 +704,22 @@ void MasterServer::register_handlers() { }); /// clice/ extension commands. + peer.on_request( "clice/queryContext", [this](RequestContext& ctx, const ext::QueryContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); + auto path = uri_to_path(params.uri); + auto path_id = workspace.path_pool.intern(path); int offset_val = std::max(0, params.offset.value_or(0)); constexpr int page_size = 10; ext::QueryContextResult result; std::vector all_items; - auto hosts = dependency_graph.find_host_sources(path_id); + auto hosts = workspace.dep_graph.find_host_sources(path_id); for(auto host_id: hosts) { - auto host_path = path_pool.resolve(host_id); - auto host_cdb = cdb.lookup(host_path, {.suppress_logging = true}); + auto host_path = workspace.path_pool.resolve(host_id); + auto host_cdb = workspace.cdb.lookup(host_path, {.suppress_logging = true}); if(host_cdb.empty()) continue; auto host_uri_opt = lsp::URI::from_file_path(std::string(host_path)); @@ -652,7 +733,7 @@ void MasterServer::register_handlers() { } if(hosts.empty()) { - auto entries = cdb.lookup(path, {.suppress_logging = true}); + auto entries = workspace.cdb.lookup(path, {.suppress_logging = true}); for(std::size_t i = 0; i < entries.size(); ++i) { auto& entry = entries[i]; std::string desc; @@ -693,13 +774,13 @@ void MasterServer::register_handlers() { peer.on_request( "clice/currentContext", [this](RequestContext& ctx, const ext::CurrentContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); + auto path = uri_to_path(params.uri); + auto path_id = workspace.path_pool.intern(path); ext::CurrentContextResult result; - auto active_ctx = compiler.get_active_context(path_id); - if(active_ctx) { - auto ctx_path = path_pool.resolve(*active_ctx); + auto sit = sessions.find(path_id); + if(sit != sessions.end() && sit->second.active_context) { + auto ctx_path = workspace.path_pool.resolve(*sit->second.active_context); auto ctx_uri_opt = lsp::URI::from_file_path(std::string(ctx_path)); if(ctx_uri_opt) { ext::ContextItem item; @@ -715,20 +796,31 @@ void MasterServer::register_handlers() { peer.on_request( "clice/switchContext", [this](RequestContext& ctx, const ext::SwitchContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); - auto context_path = Compiler::uri_to_path(params.context_uri); - auto context_path_id = path_pool.intern(context_path); + auto path = uri_to_path(params.uri); + auto path_id = workspace.path_pool.intern(path); + auto context_path = uri_to_path(params.context_uri); + auto context_path_id = workspace.path_pool.intern(context_path); ext::SwitchContextResult result; - auto context_cdb = cdb.lookup(context_path, {.suppress_logging = true}); + auto context_cdb = workspace.cdb.lookup(context_path, {.suppress_logging = true}); if(context_cdb.empty()) { result.success = false; co_return to_raw(result); } - compiler.switch_context(path_id, context_path_id); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) { + result.success = false; + co_return to_raw(result); + } + + sit->second.active_context = context_path_id; + sit->second.header_context.reset(); + sit->second.pch_ref.reset(); + sit->second.ast_deps.reset(); + sit->second.ast_dirty = true; + result.success = true; co_return to_raw(result); }); diff --git a/src/server/master_server.h b/src/server/master_server.h index 99a2a717..2a208801 100644 --- a/src/server/master_server.h +++ b/src/server/master_server.h @@ -9,11 +9,12 @@ #include "eventide/ipc/peer.h" #include "eventide/serde/serde/raw_value.h" #include "server/compiler.h" -#include "server/config.h" #include "server/indexer.h" +#include "server/session.h" #include "server/worker_pool.h" -#include "support/path_pool.h" -#include "syntax/dependency_graph.h" +#include "server/workspace.h" + +#include "llvm/ADT/DenseMap.h" namespace clice { @@ -27,11 +28,20 @@ enum class ServerLifecycle : std::uint8_t { Exited, }; -/// Top-level LSP server. +/// Top-level LSP server — the single orchestration point for the language +/// server process. /// -/// Registers LSP handlers and delegates all compilation, document management, -/// and index queries to Compiler and Indexer respectively. MasterServer -/// itself only owns workspace initialization and background indexing scheduling. +/// Responsibilities: +/// - Owns the two-layer state model: Workspace (disk truth) and Sessions +/// (per-open-file volatile state). +/// - Manages Session lifecycle directly: didOpen creates, didChange mutates, +/// didSave syncs to Workspace, didClose destroys. +/// - Dispatches compilation and feature queries to Compiler. +/// - Dispatches index lookups and background indexing to Indexer. +/// +/// Design principle: +/// Open files are never depended upon by other files. Dependencies always +/// point to disk files. The only path from Session to Workspace is didSave. class MasterServer { public: MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path); @@ -42,31 +52,29 @@ public: private: et::event_loop& loop; et::ipc::JsonPeer& peer; + + /// Persistent project-wide state (config, CDB, path pool, dependency + /// graphs, compilation caches, symbol index). + Workspace workspace; + + /// Per-file editing sessions, keyed by server-level path_id. + llvm::DenseMap sessions; + + /// Worker process pool for offloading compilation and queries. WorkerPool pool; - PathPool path_pool; + + /// Compilation lifecycle manager (reads/writes workspace and sessions). + Compiler compiler; + + /// Index query and background scheduling (reads from workspace and sessions). Indexer indexer; ServerLifecycle lifecycle = ServerLifecycle::Uninitialized; std::string self_path; std::string workspace_root; - CliceConfig config; std::string session_log_dir; - CompilationDatabase cdb; - DependencyGraph dependency_graph; - - Compiler compiler; - - /// Background indexing state. - std::vector index_queue; - std::size_t index_queue_pos = 0; - bool indexing_active = false; - bool indexing_scheduled = false; - std::shared_ptr index_idle_timer; - et::task<> load_workspace(); - void schedule_indexing(); - et::task<> run_background_indexing(); using RawResult = et::task; }; diff --git a/src/server/protocol.h b/src/server/protocol.h index 5ea5c565..dba9dd1d 100644 --- a/src/server/protocol.h +++ b/src/server/protocol.h @@ -10,6 +10,7 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/protocol.h" #include "eventide/serde/serde/raw_value.h" +#include "syntax/token.h" namespace clice::worker { @@ -32,7 +33,8 @@ enum class QueryKind : uint8_t { struct QueryParams { QueryKind kind; std::string path; - uint32_t offset = 0; ///< Used by Hover and GoToDefinition. + uint32_t offset = 0; ///< Byte offset for position-sensitive queries (Hover, GoToDefinition). + LocalSourceRange range; ///< Byte range for range-sensitive queries (InlayHints). }; /// Parameters for stateful compilation (builds AST, publishes diagnostics). @@ -103,8 +105,6 @@ struct BuildResult { eventide::serde::RawValue result_json; ///< Completion/SignatureHelp result }; -// === Notifications === - struct DocumentUpdateParams { std::string path; int version; @@ -120,8 +120,6 @@ struct EvictedParams { } // namespace clice::worker -// === clice/ LSP Extension Types === - namespace clice::ext { struct ContextItem { @@ -179,8 +177,6 @@ struct RequestTraits { constexpr inline static std::string_view method = "clice/worker/build"; }; -// === Notifications === - template <> struct NotificationTraits { constexpr inline static std::string_view method = "clice/worker/documentUpdate"; diff --git a/src/server/session.h b/src/server/session.h new file mode 100644 index 00000000..be58aea9 --- /dev/null +++ b/src/server/session.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include + +#include "eventide/async/async.h" +#include "server/workspace.h" + +#include "llvm/ADT/SmallVector.h" + +namespace clice { + +namespace et = eventide; + +/// An editing session for a single file opened in the editor. +/// +/// Design principle: open files are never depended upon by other files. +/// Dependencies always point to disk files. The only path from Session +/// to Workspace is didSave, which tells Workspace to rescan the disk file. +/// +/// Created on didOpen, destroyed on didClose. All fields are local to this +/// file's translation unit and NEVER leak to Workspace or other Sessions. +/// Sessions may READ from Workspace (e.g. to obtain PCH/PCM paths, module +/// mappings, include graph) but all compilation results stay here. +struct Session { + /// Path ID of this file in PathPool. Set on creation, never changes. + std::uint32_t path_id = 0; + + /// LSP document version, incremented by the client on each edit. + int version = 0; + + /// Current buffer content (may differ from disk until saved). + std::string text; + + /// Monotonic generation counter, incremented on every didChange. + /// Used to detect stale compilation results (ABA prevention). + std::uint64_t generation = 0; + + /// Whether the AST needs to be rebuilt before serving queries. + bool ast_dirty = true; + + /// Non-null while a compilation is in flight for this file. + /// Other queries wait on the event; the compilation task itself + /// runs independently and cannot be cancelled by LSP $/cancelRequest. + struct PendingCompile { + et::event done; + bool succeeded = false; + }; + + std::shared_ptr compiling; + + /// Reference to the PCH entry in Workspace.pch_cache, if any. + /// The PCH itself is owned by Workspace (shared, content-addressed); + /// Session only stores enough to locate and validate it. + struct PCHRef { + std::uint32_t path_id = 0; ///< Key into Workspace.pch_cache. + std::uint64_t hash = 0; ///< Preamble hash at build time. + std::uint32_t bound = 0; ///< Preamble byte boundary. + }; + + std::optional pch_ref; + + /// Dependency snapshot from the last successful AST compilation. + /// Used for two-layer staleness detection (mtime + content hash). + std::optional ast_deps; + + /// Compilation context for header files that lack their own CDB entry. + /// Stores the host source file and synthesized preamble for this header. + std::optional header_context; + + /// User-selected compilation context override (via clice/switchContext). + /// When set, overrides automatic header context resolution. + std::optional active_context; + + /// Symbol index built from the latest compilation of this file's buffer. + /// Used for queries (hover, goto, references) on this file. + /// NOT merged into Workspace.project_index — that only gets disk-derived + /// data from background indexing. + std::optional file_index; +}; + +} // namespace clice diff --git a/src/server/stateful_worker.cpp b/src/server/stateful_worker.cpp index 9e93afa4..8d31838d 100644 --- a/src/server/stateful_worker.cpp +++ b/src/server/stateful_worker.cpp @@ -249,7 +249,9 @@ void StatefulWorker::register_handlers() { }); case K::InlayHints: co_return co_await with_ast(params.path, [&](DocumentEntry& doc) { - LocalSourceRange range{0, static_cast(doc.text.size())}; + auto range = params.range; + if(range.begin == static_cast(-1)) + range = LocalSourceRange{0, static_cast(doc.text.size())}; return to_raw(feature::inlay_hints(doc.unit, range)); }); case K::FoldingRange: diff --git a/src/server/worker_pool.h b/src/server/worker_pool.h index 162bfcd6..6e83860d 100644 --- a/src/server/worker_pool.h +++ b/src/server/worker_pool.h @@ -83,8 +83,6 @@ private: bool spawn_worker(const std::string& self_path, bool stateful, std::uint64_t memory_limit); }; -// --- Template implementations --------------------------------------------------- - template RequestResult WorkerPool::send_stateful(std::uint32_t path_id, const Params& params, diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp new file mode 100644 index 00000000..66f18315 --- /dev/null +++ b/src/server/workspace.cpp @@ -0,0 +1,376 @@ +#include "server/workspace.h" + +#include +#include + +#include "eventide/ipc/lsp/position.h" +#include "eventide/ipc/lsp/protocol.h" +#include "eventide/serde/json/json.h" +#include "support/filesystem.h" +#include "support/logging.h" +#include "syntax/scan.h" + +#include "llvm/Support/Chrono.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/xxhash.h" + +namespace clice { + +namespace lsp = eventide::ipc::lsp; + +/// Find the tightest (innermost) occurrence containing `offset` via binary search. +const static index::Occurrence* lookup_occurrence(const std::vector& occs, + std::uint32_t offset) { + auto it = std::ranges::lower_bound(occs, offset, {}, [](const index::Occurrence& o) { + return o.range.end; + }); + const index::Occurrence* best = nullptr; + while(it != occs.end() && it->range.contains(offset)) { + if(!best || (it->range.end - it->range.begin) < (best->range.end - best->range.begin)) { + best = &*it; + } + ++it; + } + return best; +} + +std::optional> + OpenFileIndex::find_occurrence(std::uint32_t offset) const { + if(!mapper) + return std::nullopt; + auto* occ = lookup_occurrence(file_index.occurrences, offset); + if(!occ) + return std::nullopt; + auto start = mapper->to_position(occ->range.begin); + auto end = mapper->to_position(occ->range.end); + if(!start || !end) + return std::nullopt; + return std::pair{ + occ->target, + protocol::Range{*start, *end} + }; +} + +std::optional> + MergedIndexShard::find_occurrence(std::uint32_t offset) const { + auto* m = mapper(); + if(!m) + return std::nullopt; + std::optional> result; + index.lookup(offset, [&](const index::Occurrence& o) { + auto start = m->to_position(o.range.begin); + auto end = m->to_position(o.range.end); + if(start && end) { + result = { + o.target, + protocol::Range{*start, *end} + }; + } + return false; + }); + return result; +} + +llvm::SmallVector Workspace::on_file_saved(std::uint32_t path_id) { + llvm::SmallVector dirtied; + + // Re-scan the saved file for module declarations and update path_to_module. + auto file_path = path_pool.resolve(path_id); + if(auto buf = llvm::MemoryBuffer::getFile(file_path)) { + auto result = scan((*buf)->getBuffer()); + if(!result.module_name.empty()) { + path_to_module[path_id] = std::move(result.module_name); + } else { + path_to_module.erase(path_id); + } + } + + if(compile_graph) { + auto result = compile_graph->update(path_id); + for(auto id: result) { + dirtied.push_back(id); + pcm_paths.erase(id); + pcm_cache.erase(id); + } + } + return dirtied; +} + +void Workspace::on_file_closed(std::uint32_t path_id) { + if(compile_graph && compile_graph->has_unit(path_id)) { + compile_graph->update(path_id); + } + pch_cache.erase(path_id); +} + +std::uint64_t hash_file(llvm::StringRef path) { + auto buf = llvm::MemoryBuffer::getFile(path); + if(!buf) + return 0; + return llvm::xxh3_64bits((*buf)->getBuffer()); +} + +DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps) { + DepsSnapshot snap; + // Capture timestamp BEFORE hashing to avoid TOCTOU: if a file is modified + // during hashing, its mtime will be > build_at, triggering Layer 2 re-hash. + snap.build_at = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + snap.path_ids.reserve(deps.size()); + snap.hashes.reserve(deps.size()); + for(const auto& file: deps) { + snap.path_ids.push_back(pool.intern(file)); + snap.hashes.push_back(hash_file(file)); + } + return snap; +} + +bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { + for(std::size_t i = 0; i < snap.path_ids.size(); ++i) { + auto path = pool.resolve(snap.path_ids[i]); + llvm::sys::fs::file_status status; + if(auto ec = llvm::sys::fs::status(path, status)) { + // File disappeared — definitely changed. + if(snap.hashes[i] != 0) + return true; + continue; + } + + // Layer 1: mtime check (cheap, stat only). + auto current_mtime = llvm::sys::toTimeT(status.getLastModificationTime()); + if(current_mtime <= snap.build_at) + continue; + + // Layer 2: mtime is newer — re-hash content to confirm actual change. + auto current_hash = hash_file(path); + if(current_hash != snap.hashes[i]) + return true; + } + return false; +} + +namespace { + +struct CacheDepEntry { + std::uint32_t path; // index into CacheData::paths + std::uint64_t hash; +}; + +struct CachePCHEntry { + std::string filename; + std::uint32_t source_file; // index into CacheData::paths + std::uint64_t hash; + std::uint32_t bound; + std::int64_t build_at; + std::vector deps; +}; + +struct CachePCMEntry { + std::string filename; + std::uint32_t source_file; + std::string module_name; + std::int64_t build_at; + std::vector deps; +}; + +struct CacheData { + std::vector paths; + std::vector pch; + std::vector pcm; +}; + +} // namespace + +void Workspace::load_cache() { + if(config.cache_dir.empty()) + return; + + auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto content = fs::read(cache_path); + if(!content) { + LOG_DEBUG("No cache.json found at {}", cache_path); + return; + } + + CacheData data; + auto status = eventide::serde::json::from_json(*content, data); + if(!status) { + LOG_WARN("Failed to parse cache.json"); + return; + } + + auto resolve = [&](std::uint32_t idx) -> llvm::StringRef { + return idx < data.paths.size() ? llvm::StringRef(data.paths[idx]) : ""; + }; + + auto load_deps = [&](std::int64_t build_at, const auto& dep_entries) -> DepsSnapshot { + DepsSnapshot deps; + deps.build_at = build_at; + for(auto& dep: dep_entries) { + auto dep_path = resolve(dep.path); + if(dep_path.empty()) + continue; + deps.path_ids.push_back(path_pool.intern(dep_path)); + deps.hashes.push_back(dep.hash); + } + return deps; + }; + + for(auto& entry: data.pch) { + auto pch_path = path::join(config.cache_dir, "cache", "pch", entry.filename); + auto source = resolve(entry.source_file); + if(!llvm::sys::fs::exists(pch_path) || source.empty()) + continue; + + auto path_id = path_pool.intern(source); + auto& st = pch_cache[path_id]; + st.path = pch_path; + st.hash = entry.hash; + st.bound = entry.bound; + st.deps = load_deps(entry.build_at, entry.deps); + + LOG_DEBUG("Loaded cached PCH: {} -> {}", source, pch_path); + } + + for(auto& entry: data.pcm) { + auto pcm_path = path::join(config.cache_dir, "cache", "pcm", entry.filename); + auto source = resolve(entry.source_file); + if(!llvm::sys::fs::exists(pcm_path) || source.empty()) + continue; + + auto path_id = path_pool.intern(source); + pcm_cache[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; + pcm_paths[path_id] = pcm_path; + + LOG_DEBUG("Loaded cached PCM: {} (module {}) -> {}", source, entry.module_name, pcm_path); + } + + LOG_INFO("Loaded cache.json: {} PCH entries, {} PCM entries", + pch_cache.size(), + pcm_cache.size()); +} + +void Workspace::save_cache() { + if(config.cache_dir.empty()) + return; + + CacheData data; + std::unordered_map index_map; + + auto intern = [&](std::uint32_t runtime_path_id) -> std::uint32_t { + auto path = std::string(path_pool.resolve(runtime_path_id)); + auto [it, inserted] = + index_map.try_emplace(path, static_cast(data.paths.size())); + if(inserted) { + data.paths.push_back(path); + } + return it->second; + }; + + for(auto& [path_id, st]: pch_cache) { + if(st.path.empty()) + continue; + + CachePCHEntry entry; + entry.filename = std::string(path::filename(st.path)); + entry.source_file = intern(path_id); + entry.hash = st.hash; + entry.bound = st.bound; + entry.build_at = st.deps.build_at; + for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { + entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); + } + data.pch.push_back(std::move(entry)); + } + + for(auto& [path_id, st]: pcm_cache) { + if(st.path.empty()) + continue; + + CachePCMEntry entry; + entry.filename = std::string(path::filename(st.path)); + entry.source_file = intern(path_id); + auto mod_it = path_to_module.find(path_id); + entry.module_name = mod_it != path_to_module.end() ? mod_it->second : ""; + entry.build_at = st.deps.build_at; + for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { + entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); + } + data.pcm.push_back(std::move(entry)); + } + + auto json_str = eventide::serde::json::to_json(data); + if(!json_str) { + LOG_WARN("Failed to serialize cache.json"); + return; + } + + auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto tmp_path = cache_path + ".tmp"; + auto write_result = fs::write(tmp_path, *json_str); + if(!write_result) { + LOG_WARN("Failed to write cache.json.tmp: {}", write_result.error().message()); + return; + } + auto rename_result = fs::rename(tmp_path, cache_path); + if(!rename_result) { + LOG_WARN("Failed to rename cache.json.tmp to cache.json: {}", + rename_result.error().message()); + } +} + +void Workspace::cleanup_cache(int max_age_days) { + if(config.cache_dir.empty()) + return; + + auto now = std::chrono::system_clock::now(); + auto max_age = std::chrono::hours(max_age_days * 24); + + for(auto* subdir: {"cache/pch", "cache/pcm"}) { + auto dir = path::join(config.cache_dir, subdir); + std::error_code ec; + for(auto it = llvm::sys::fs::directory_iterator(dir, ec); + !ec && it != llvm::sys::fs::directory_iterator(); + it.increment(ec)) { + llvm::sys::fs::file_status status; + if(auto stat_ec = llvm::sys::fs::status(it->path(), status)) + continue; + + auto mtime = status.getLastModificationTime(); + auto age = now - mtime; + if(age > max_age) { + llvm::sys::fs::remove(it->path()); + LOG_DEBUG("Cleaned up stale cache file: {}", it->path()); + } + } + } +} + +void Workspace::build_module_map() { + for(auto& [module_name, path_ids]: dep_graph.modules()) { + for(auto path_id: path_ids) { + path_to_module[path_id] = module_name.str(); + } + } +} + +void Workspace::fill_pcm_deps(std::unordered_map& pcms, + std::uint32_t exclude_path_id) const { + for(auto& [pid, pcm_path]: pcm_paths) { + if(pid == exclude_path_id) + continue; + auto mod_it = path_to_module.find(pid); + if(mod_it != path_to_module.end()) { + pcms[mod_it->second] = pcm_path; + } + } +} + +void Workspace::cancel_all() { + if(compile_graph) { + compile_graph->cancel_all(); + } +} + +} // namespace clice diff --git a/src/server/workspace.h b/src/server/workspace.h new file mode 100644 index 00000000..fd8b7e6d --- /dev/null +++ b/src/server/workspace.h @@ -0,0 +1,247 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "command/command.h" +#include "eventide/ipc/lsp/position.h" +#include "eventide/ipc/lsp/protocol.h" +#include "index/merged_index.h" +#include "index/project_index.h" +#include "semantic/relation_kind.h" +#include "server/compile_graph.h" +#include "server/config.h" +#include "support/path_pool.h" +#include "syntax/dependency_graph.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +namespace et = eventide; +namespace protocol = et::ipc::protocol; +namespace lsp = et::ipc::lsp; + +/// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). +/// +/// Layer 1 (fast): compare each file's current mtime against build_at. +/// If all mtimes <= build_at, the artifact is fresh (zero I/O beyond stat). +/// +/// Layer 2 (precise): for files whose mtime changed, re-hash their content +/// and compare against the stored hash. If the hash matches, the file was +/// "touched" but not actually modified — skip the rebuild. +struct DepsSnapshot { + llvm::SmallVector path_ids; + llvm::SmallVector hashes; + std::int64_t build_at = 0; +}; + +/// Context for compiling a header file that lacks its own CDB entry. +struct HeaderFileContext { + std::uint32_t host_path_id; ///< Source file acting as host. + std::string preamble_path; ///< Path to generated preamble file on disk. + std::uint64_t preamble_hash; ///< Hash of preamble content for staleness. +}; + +/// In-memory index for an open file. Kept separate from MergedIndex because +/// open files change frequently, are based on unsaved buffer content, and only +/// need to track the main file (headers are covered by PCH/PCM indexing). +struct OpenFileIndex { + index::FileIndex file_index; + index::SymbolTable symbols; + std::string content; ///< Buffer text at index time (for position mapping). + + /// Cached PositionMapper built from `content`. Avoids re-scanning line + /// offsets on every query. Initialized by Indexer::set_open_file(). + std::optional mapper; + + /// Find the tightest occurrence containing `offset`. + /// Returns (symbol_hash, LSP range) with positions already converted. + std::optional> + find_occurrence(std::uint32_t offset) const; + + /// Iterate relations matching `kind`, calling back with pre-converted ranges. + /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). + template + void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { + if(!mapper) + return; + auto it = file_index.relations.find(hash); + if(it == file_index.relations.end()) + return; + for(auto& r: it->second) { + if(r.kind & kind) { + auto start = mapper->to_position(r.range.begin); + auto end = mapper->to_position(r.range.end); + if(start && end) { + if(!fn(r, protocol::Range{*start, *end})) + return; + } + } + } + } +}; + +/// Wraps index::MergedIndex with a lazily-cached PositionMapper. +struct MergedIndexShard { + index::MergedIndex index; + mutable std::optional cached_mapper; + + /// Get or lazily build a PositionMapper from the index's stored content. + const lsp::PositionMapper* mapper() const { + if(!cached_mapper) { + auto c = index.content(); + if(!c.empty()) { + cached_mapper.emplace(c, lsp::PositionEncoding::UTF16); + } + } + return cached_mapper ? &*cached_mapper : nullptr; + } + + /// Invalidate the cached mapper (call after merge changes content). + void invalidate_mapper() { + cached_mapper.reset(); + } + + /// Find occurrence at byte offset. + /// Returns (symbol_hash, LSP range) with positions already converted. + std::optional> + find_occurrence(std::uint32_t offset) const; + + /// Iterate relations matching `kind`, calling back with pre-converted ranges. + /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). + template + void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { + auto* m = mapper(); + if(!m) + return; + index.lookup(hash, kind, [&](const index::Relation& r) { + auto start = m->to_position(r.range.begin); + auto end = m->to_position(r.range.end); + if(start && end) { + return fn(r, protocol::Range{*start, *end}); + } + return true; + }); + } +}; + +/// Cached PCH state. Content-addressed by preamble hash — shared across all +/// files (open or on-disk) that have the same preamble content. +struct PCHState { + std::string path; + std::uint32_t bound = 0; + std::uint64_t hash = 0; + DepsSnapshot deps; + std::shared_ptr building; +}; + +/// Cached PCM state for a single C++20 module. Shared across all files that +/// import the same module. +struct PCMState { + std::string path; + DepsSnapshot deps; +}; + +/// All persistent, project-wide state derived from files on disk. +/// +/// Design principle: open files are never depended upon by other files. +/// Dependencies always point to disk files. This enforces a clean two-layer +/// architecture: +/// - Global layer (Workspace): tracks disk truth, shared by all files +/// - Per-file layer (Session): tracks buffer truth, isolated per TU +/// +/// Workspace is the single source of truth for: +/// - dependency relationships (include graph, module DAG) +/// - compilation artifacts shared across files (PCH/PCM caches) +/// - symbol index (ProjectIndex + per-file MergedIndex shards) +/// - compilation database and configuration +/// +/// Workspace is NEVER modified by unsaved buffer content. The only mutation +/// paths are: +/// - Initialization (load_workspace at startup) +/// - didSave (on_file_saved: rescan disk, cascade invalidation) +/// - Background index (merge TUIndex results from stateless workers) +struct Workspace { + CliceConfig config; + CompilationDatabase cdb; + + PathPool path_pool; + + /// Include relationships between files on disk (#include edges). + /// Built once at startup from CDB scan; updated incrementally on didSave. + DependencyGraph dep_graph; + + /// C++20 module compilation ordering DAG. + /// Lazily resolves module dependencies; updated on didSave via cascade. + std::unique_ptr compile_graph; + + /// Reverse mapping: file path_id → module name (e.g. "std", "foo.bar"). + /// Built from dep_graph at startup; updated on didSave when module + /// declarations change. + llvm::DenseMap path_to_module; + + /// PCH cache, keyed by file path_id. + /// TODO: re-key by preamble content hash to enable cross-file sharing and + /// add LRU eviction. Compile flags should also be part of the key. + llvm::DenseMap pch_cache; + + /// PCM cache, keyed by module source path_id. + llvm::DenseMap pcm_cache; + + /// PCM output paths, keyed by module source path_id. + /// Maps to the .pcm file on disk used as -fmodule-file argument. + llvm::DenseMap pcm_paths; + + /// Global symbol table across all indexed translation units. + index::ProjectIndex project_index; + + /// Per-file index shards from background indexing, keyed by project-level + /// path_id. Contains symbol occurrences, relations, and stored content + /// for position mapping. + llvm::DenseMap merged_indices; + + /// Called when a file is saved to disk. Cascades invalidation through + /// compile_graph and clears affected PCM caches. + /// Returns path_ids of all files dirtied by the cascade. + llvm::SmallVector on_file_saved(std::uint32_t path_id); + + /// Called when a file is closed. Notifies compile_graph if this file + /// is a module unit so dependents can be re-evaluated on next compile. + void on_file_closed(std::uint32_t path_id); + + /// Load PCH/PCM cache from cache.json on disk. + void load_cache(); + /// Save PCH/PCM cache to cache.json on disk. + void save_cache(); + /// Remove stale PCH/PCM files older than max_age_days. + void cleanup_cache(int max_age_days = 7); + /// Build path_to_module reverse mapping from dep_graph. + void build_module_map(); + /// Fill PCM paths for all built modules, excluding exclude_path_id. + void fill_pcm_deps(std::unordered_map& pcms, + std::uint32_t exclude_path_id = UINT32_MAX) const; + /// Cancel all in-flight compilations. + void cancel_all(); +}; + +/// Hash a file's content using xxh3_64bits. Returns 0 on read failure. +std::uint64_t hash_file(llvm::StringRef path); + +/// Capture a two-layer staleness snapshot after a successful compilation. +/// Interns dependency paths into the PathPool and hashes each file's content. +DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps); + +/// Two-layer staleness check. +/// Layer 1 (fast): stat each dep file, compare mtime against build_at. +/// Layer 2 (precise): for files with mtime > build_at, re-hash content. +bool deps_changed(const PathPool& pool, const DepsSnapshot& snap); + +} // namespace clice diff --git a/src/syntax/completion.cpp b/src/syntax/completion.cpp new file mode 100644 index 00000000..5d68f69f --- /dev/null +++ b/src/syntax/completion.cpp @@ -0,0 +1,121 @@ +#include "syntax/completion.h" + +#include "syntax/include_resolver.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +namespace clice { + +PreambleCompletionContext detect_completion_context(llvm::StringRef text, std::uint32_t offset) { + auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); + line_start = (line_start == llvm::StringRef::npos) ? 0 : line_start + 1; + + auto line = text.slice(line_start, offset); + auto trimmed = line.ltrim(); + + if(trimmed.starts_with("#")) { + auto directive = trimmed.drop_front(1).ltrim(); + if(directive.consume_front("include")) { + directive = directive.ltrim(); + if(directive.consume_front("\"")) { + return {CompletionContext::IncludeQuoted, directive.str()}; + } + if(directive.consume_front("<")) { + return {CompletionContext::IncludeAngled, directive.str()}; + } + } + return {}; + } + + auto import_check = trimmed; + if(import_check.consume_front("export") && !import_check.empty() && + !std::isalnum(import_check[0])) { + import_check = import_check.ltrim(); + } + if(import_check.consume_front("import") && + (import_check.empty() || !std::isalnum(import_check[0]))) { + import_check = import_check.ltrim(); + + auto line_end = text.find('\n', offset); + if(line_end == llvm::StringRef::npos) + line_end = text.size(); + auto rest_of_line = text.slice(line_start, line_end); + if(!rest_of_line.contains(';')) { + return {CompletionContext::Import, import_check.str()}; + } + } + + return {}; +} + +std::vector + complete_module_import(const llvm::DenseMap& modules, + llvm::StringRef prefix) { + std::vector results; + for(auto& [path_id, module_name]: modules) { + if(llvm::StringRef(module_name).starts_with(prefix)) { + results.push_back(module_name); + } + } + return results; +} + +std::vector complete_include_path(const ResolvedSearchConfig& resolved, + llvm::StringRef prefix, + bool angled, + DirListingCache& dir_cache) { + llvm::StringRef dir_prefix; + llvm::StringRef file_prefix = prefix; + auto slash_pos = prefix.rfind('/'); + if(slash_pos != llvm::StringRef::npos) { + dir_prefix = prefix.slice(0, slash_pos); + file_prefix = prefix.slice(slash_pos + 1, llvm::StringRef::npos); + } + + unsigned start_idx = angled ? resolved.angled_start_idx : 0; + + std::vector results; + llvm::StringSet<> seen; + + for(unsigned i = start_idx; i < resolved.dirs.size(); ++i) { + auto& search_dir = resolved.dirs[i]; + + const llvm::StringSet<>* entries = nullptr; + if(!dir_prefix.empty()) { + llvm::SmallString<256> sub_path(search_dir.path); + llvm::sys::path::append(sub_path, dir_prefix); + entries = resolve_dir(sub_path, dir_cache); + } else { + entries = search_dir.entries; + } + + if(!entries) + continue; + + for(auto& entry: *entries) { + auto name = entry.getKey(); + if(!name.starts_with(file_prefix)) + continue; + if(!seen.insert(name).second) + continue; + + llvm::SmallString<256> full_path(search_dir.path); + if(!dir_prefix.empty()) { + llvm::sys::path::append(full_path, dir_prefix); + } + llvm::sys::path::append(full_path, name); + + bool is_dir = false; + llvm::sys::fs::is_directory(llvm::Twine(full_path), is_dir); + + results.push_back({name.str(), is_dir}); + } + } + + return results; +} + +} // namespace clice diff --git a/src/syntax/completion.h b/src/syntax/completion.h new file mode 100644 index 00000000..80193e6a --- /dev/null +++ b/src/syntax/completion.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +struct ResolvedSearchConfig; +struct DirListingCache; + +/// What kind of preamble-level completion is being requested. +enum class CompletionContext : std::uint8_t { + None, + IncludeQuoted, + IncludeAngled, + Import, +}; + +/// Result of detecting the completion context from source text. +struct PreambleCompletionContext { + CompletionContext kind = CompletionContext::None; + std::string prefix; +}; + +/// Detect whether the cursor is inside a #include or import directive. +/// Pure text parsing — no compiler state needed. +PreambleCompletionContext detect_completion_context(llvm::StringRef text, std::uint32_t offset); + +/// Return module names matching a prefix, suitable for `import` completion. +/// @param modules Module name map (path_id → module name). +/// @param prefix Partially-typed module name to match against. +std::vector + complete_module_import(const llvm::DenseMap& modules, + llvm::StringRef prefix); + +/// Entry in the include path completion result. +struct IncludeCandidate { + std::string name; + bool is_directory = false; +}; + +/// Return file/directory names matching a prefix in the given search paths. +/// @param resolved Pre-resolved search directories with cached directory listings. +/// @param angled_start Index where angled (<>) search dirs begin. +/// @param prefix Partially-typed include path (e.g. "vec" or "sys/"). +/// @param angled True for <> includes, false for "" includes. +/// @param dir_cache Shared directory listing cache (for subdirectory lookups). +std::vector complete_include_path(const ResolvedSearchConfig& resolved, + llvm::StringRef prefix, + bool angled, + DirListingCache& dir_cache); + +} // namespace clice diff --git a/src/syntax/dependency_graph.cpp b/src/syntax/dependency_graph.cpp index 9443dc13..47fbee66 100644 --- a/src/syntax/dependency_graph.cpp +++ b/src/syntax/dependency_graph.cpp @@ -20,9 +20,7 @@ namespace clice { namespace et = eventide; -// ============================================================================ // DependencyGraph implementation -// ============================================================================ void DependencyGraph::add_module(llvm::StringRef module_name, std::uint32_t path_id) { auto& ids = module_to_path[module_name]; @@ -205,9 +203,7 @@ std::vector DependencyGraph::find_include_chain(std::uint32_t hos return chain; } -// ============================================================================ // Wavefront BFS scanner — async implementation -// ============================================================================ namespace { @@ -819,9 +815,7 @@ et::task<> scan_impl(CompilationDatabase& cdb, } // namespace -// ============================================================================ // Public sync entry point -// ============================================================================ ScanReport scan_dependency_graph(CompilationDatabase& cdb, PathPool& path_pool, diff --git a/src/syntax/token.h b/src/syntax/token.h index 31ade4a6..ad844fad 100644 --- a/src/syntax/token.h +++ b/src/syntax/token.h @@ -44,10 +44,6 @@ struct LocalSourceRange { std::uint32_t begin = static_cast(-1); std::uint32_t end = static_cast(-1); - constexpr LocalSourceRange() = default; - - constexpr LocalSourceRange(std::uint32_t begin, std::uint32_t end) : begin(begin), end(end) {} - constexpr bool operator==(const LocalSourceRange& other) const = default; constexpr std::uint32_t length() const { diff --git a/tests/unit/syntax/completion_tests.cpp b/tests/unit/syntax/completion_tests.cpp new file mode 100644 index 00000000..aed64c2e --- /dev/null +++ b/tests/unit/syntax/completion_tests.cpp @@ -0,0 +1,125 @@ +#include "test/test.h" +#include "syntax/completion.h" + +#include "llvm/ADT/DenseMap.h" + +namespace clice::testing { +namespace { + +TEST_SUITE(DetectCompletionContext) { + +TEST_CASE(IncludeAngled) { + auto ctx = detect_completion_context("#include modules; + modules[1] = "std"; + modules[2] = "std.io"; + modules[3] = "std.net"; + modules[4] = "my_lib"; + + auto results = complete_module_import(modules, "std"); + EXPECT_EQ(results.size(), 3u); + for(auto& name: results) { + EXPECT_TRUE(name.starts_with("std")); + } +} + +TEST_CASE(EmptyPrefix) { + llvm::DenseMap modules; + modules[1] = "std"; + modules[2] = "my_lib"; + + auto results = complete_module_import(modules, ""); + EXPECT_EQ(results.size(), 2u); +} + +TEST_CASE(NoMatch) { + llvm::DenseMap modules; + modules[1] = "std"; + modules[2] = "my_lib"; + + auto results = complete_module_import(modules, "xyz"); + EXPECT_TRUE(results.empty()); +} + +TEST_CASE(EmptyModules) { + llvm::DenseMap modules; + auto results = complete_module_import(modules, "std"); + EXPECT_TRUE(results.empty()); +} + +}; // TEST_SUITE(CompleteModuleImport) + +} // namespace +} // namespace clice::testing