1 Commits

Author SHA1 Message Date
ykiko
d241ea8492 refactor(index): migrate FlatBuffers from flatc IDL to kotatsu reflection
Replace the flatc-generated serialization layer with kotatsu's arena codec
driven directly by the in-memory index types. No hand-written DTOs: the
on-wire layout is derived from reflection over the existing structs, with
type-level customization where needed.

- Drop `schema.fbs`, `serialization.h`, and the flatc build step
- Delete `wire_types.h` — no more parallel wire representation
- Add `kotatsu_adapters.h` with `kota::codec::type_adapter<T>` specializations
  for RelationKind, SymbolKind, Bitmap, and std::chrono::milliseconds
- Mark runtime-only FileID-keyed maps with `kota::meta::skip<>` so they
  are excluded from reflection slots; serialize via `main_file_index` and
  `path_file_indices` (keyed by path id)
- Restore MergedIndex's dual dispatch: in-memory path when `impl` is live,
  lazy flatbuffers path via `kfb::table_view<Impl>::from_bytes()` and
  `root[&Impl::field]` proxy access when only the buffer is held
- Add default member initializers to LocalSourceRange, padding field to
  Relation, and a path_id lookup struct to IncludeLocation so reflection
  picks up all stored state
- Propagate buffer size through `TUIndex::from` / `ProjectIndex::from`
  (kota codec requires an explicit size for bounds verification)

All 551 unit tests pass; 9 environment-gated integration tests skipped.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-20 09:51:28 +08:00
53 changed files with 628 additions and 1512 deletions

View File

@@ -7,10 +7,6 @@ on:
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
changes:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}

View File

@@ -96,20 +96,9 @@ jobs:
if-no-files-found: error
retention-days: 1
- name: Unit tests
- name: Run tests
if: ${{ !matrix.build_only }}
timeout-minutes: 5
run: pixi run unit-test ${{ matrix.build_type }}
- name: Integration tests
if: ${{ !matrix.build_only }}
timeout-minutes: 20
run: pixi run integration-test ${{ matrix.build_type }}
- name: Smoke tests
if: ${{ !matrix.build_only }}
timeout-minutes: 15
run: pixi run smoke-test ${{ matrix.build_type }}
run: pixi run test ${{ matrix.build_type }}
- name: Print cache stats and stop server
if: always()
@@ -157,14 +146,5 @@ jobs:
if: runner.os != 'Windows'
run: chmod +x build/${{ matrix.build_type }}/bin/*
- name: Unit tests
timeout-minutes: 5
run: pixi run -e test-run unit-test ${{ matrix.build_type }}
- name: Integration tests
timeout-minutes: 20
run: pixi run -e test-run integration-test ${{ matrix.build_type }}
- name: Smoke tests
timeout-minutes: 10
run: pixi run -e test-run smoke-test ${{ matrix.build_type }}
- name: Run tests
run: pixi run -e test-run test ${{ matrix.build_type }}

View File

@@ -124,42 +124,21 @@ if(CLICE_CI_ENVIRONMENT)
target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT=1)
endif()
set(FBS_SCHEMA_FILE "${PROJECT_SOURCE_DIR}/src/index/schema.fbs")
set(GENERATED_HEADER "${PROJECT_BINARY_DIR}/generated/schema_generated.h")
if(CMAKE_CROSSCOMPILING)
find_program(FLATC_EXECUTABLE flatc REQUIRED)
set(FLATC_CMD "${FLATC_EXECUTABLE}")
else()
set(FLATC_CMD "$<TARGET_FILE:flatc>")
endif()
add_custom_command(
OUTPUT "${GENERATED_HEADER}"
COMMAND ${FLATC_CMD} --cpp -o "${PROJECT_BINARY_DIR}/generated" "${FBS_SCHEMA_FILE}"
DEPENDS "${FBS_SCHEMA_FILE}"
COMMENT "Generating C++ header from ${FBS_SCHEMA_FILE}"
)
add_custom_target(generate_flatbuffers_schema DEPENDS "${GENERATED_HEADER}")
file(GLOB_RECURSE CLICE_CORE_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/*.cpp")
add_library(clice-core STATIC ${CLICE_CORE_SOURCES})
add_library(clice::core ALIAS clice-core)
add_dependencies(clice-core generate_flatbuffers_schema)
target_include_directories(clice-core PUBLIC
"${PROJECT_SOURCE_DIR}/src"
"${PROJECT_BINARY_DIR}/generated"
)
target_link_libraries(clice-core PUBLIC
clice_options
llvm-libs
spdlog::spdlog
roaring::roaring
flatbuffers
kota::ipc::lsp
kota::codec::toml
kota::codec::flatbuffers
simdjson::simdjson
)

View File

@@ -26,7 +26,7 @@
#include "support/path_pool.h"
#include "syntax/dependency_graph.h"
#include "kota/codec/json/json.h"
#include "kota/codec/json/serializer.h"
#include "kota/deco/deco.h"
#include "llvm/Support/FileSystem.h"

View File

@@ -27,21 +27,10 @@ FetchContent_Declare(
set(ENABLE_ROARING_TESTS OFF CACHE INTERNAL "" FORCE)
set(ENABLE_ROARING_MICROBENCHMARKS OFF CACHE INTERNAL "" FORCE)
# flatbuffers
FetchContent_Declare(
flatbuffers
GIT_REPOSITORY https://github.com/google/flatbuffers.git
GIT_TAG v25.9.23
GIT_SHALLOW TRUE
)
set(FLATBUFFERS_BUILD_GRPC OFF CACHE BOOL "" FORCE)
set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
kotatsu
GIT_REPOSITORY https://github.com/clice-io/kotatsu
GIT_TAG main
GIT_TAG refactor/flatbuffers-schema-driven
GIT_SHALLOW TRUE
)
@@ -50,7 +39,8 @@ set(KOTA_ENABLE_TEST OFF)
set(KOTA_CODEC_ENABLE_SIMDJSON ON)
set(KOTA_CODEC_ENABLE_YYJSON ON)
set(KOTA_CODEC_ENABLE_TOML ON)
set(KOTA_CODEC_ENABLE_FLATBUFFERS ON)
set(KOTA_ENABLE_EXCEPTIONS OFF)
set(KOTA_ENABLE_RTTI OFF)
FetchContent_MakeAvailable(kotatsu spdlog croaring flatbuffers)
FetchContent_MakeAvailable(kotatsu spdlog croaring)

23
pixi.lock generated
View File

@@ -1078,7 +1078,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
linux-aarch64:
- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda
@@ -1153,7 +1152,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-64:
- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda
@@ -1226,7 +1224,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-arm64:
- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda
@@ -1292,7 +1289,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
win-64:
- conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda
@@ -1347,7 +1343,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
format:
channels:
@@ -1709,7 +1704,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
linux-aarch64:
- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda
@@ -1788,7 +1782,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-64:
- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda
@@ -1865,7 +1858,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-arm64:
- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda
@@ -1934,7 +1926,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
win-64:
- conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda
@@ -1991,7 +1982,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
test-run:
channels:
@@ -2035,7 +2025,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
linux-aarch64:
- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda
@@ -2069,7 +2058,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-64:
- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda
@@ -2125,7 +2113,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
osx-arm64:
- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_9.conda
@@ -2181,7 +2168,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
win-64:
- conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_9.conda
@@ -2213,7 +2199,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
win-arm64:
- conda: https://conda.anaconda.org/conda-forge/win-arm64/bzip2-1.0.8-h50b96f5_9.conda
@@ -2244,7 +2229,6 @@ environments:
- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl
packages:
- conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
@@ -7811,13 +7795,6 @@ packages:
- coverage>=6.2 ; extra == 'testing'
- hypothesis>=5.7.1 ; extra == 'testing'
requires_python: '>=3.10'
- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl
name: pytest-timeout
version: 2.4.0
sha256: c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2
requires_dist:
- pytest>=7.0.0
requires_python: '>=3.7'
- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.2-h32b2ec7_100_cp314.conda
build_number: 100
sha256: a120fb2da4e4d51dd32918c149b04a08815fd2bd52099dad1334647984bb07f1

View File

@@ -102,7 +102,6 @@ lld = "==20.1.8"
[feature.test.pypi-dependencies]
pytest = "*"
pytest-asyncio = ">=1.1.0"
pytest-timeout = "*"
pygls = ">=2.0.0"
lsprotocol = ">=2024.0.0"
@@ -166,8 +165,8 @@ cmd = './build/{{ type }}/bin/unit_tests --test-dir="./tests/data"'
[feature.test.tasks.integration-test]
args = [{ arg = "type", default = "RelWithDebInfo" }]
cmd = """
pytest -s --log-cli-level=INFO --timeout=300 --timeout-method=thread \
tests/integration --executable=./build/{{ type }}/bin/clice
pytest -s --log-cli-level=INFO tests/integration \
--executable=./build/{{ type }}/bin/clice
"""
[feature.test.tasks.smoke-test]

View File

@@ -219,10 +219,9 @@ public:
auto CreateASTConsumer(clang::CompilerInstance& instance, llvm::StringRef file)
-> std::unique_ptr<clang::ASTConsumer> final {
auto consumer = WrapperFrontendAction::CreateASTConsumer(instance, file);
if(!consumer)
return nullptr;
return std::make_unique<ProxyASTConsumer>(std::move(consumer), unit);
return std::make_unique<ProxyASTConsumer>(
WrapperFrontendAction::CreateASTConsumer(instance, file),
unit);
}
/// Make this public.
@@ -242,7 +241,6 @@ CompilationStatus CompilationUnitRef::Self::run_clang(
std::unique_ptr diagnostic_consumer = self.create_diagnostic();
std::unique_ptr invocation = self.create_invocation(params, diagnostic_consumer.get());
if(!invocation) {
LOG_WARN("run_clang: invocation creation failed");
return CompilationStatus::SetupFail;
}
@@ -257,7 +255,6 @@ CompilationStatus CompilationUnitRef::Self::run_clang(
}
if(!instance.createTarget()) {
LOG_WARN("run_clang: target creation failed");
return CompilationStatus::SetupFail;
}
@@ -272,7 +269,6 @@ CompilationStatus CompilationUnitRef::Self::run_clang(
/// But if we fail to `BeginSourceFile` we don't need to call `EndSourceFile`. So just
/// reset it.
self.action.reset();
LOG_WARN("run_clang: BeginSourceFile failed");
return CompilationStatus::SetupFail;
}
@@ -306,8 +302,6 @@ CompilationStatus CompilationUnitRef::Self::run_clang(
/// in crash frequently. So forbidden it here and return as error.
if(!instance.getFrontendOpts().OutputFile.empty() &&
instance.getDiagnostics().hasErrorOccurred()) {
LOG_WARN("run_clang: errors during PCH/PCM generation, output={}",
instance.getFrontendOpts().OutputFile);
return CompilationStatus::FatalError;
}

View File

@@ -81,8 +81,7 @@ auto CompilationUnitRef::file_offset(clang::SourceLocation location) -> std::uin
}
auto CompilationUnitRef::file_path(clang::FileID fid) -> llvm::StringRef {
if(!fid.isValid())
return {};
assert(fid.isValid() && "Invalid fid");
if(auto it = self->path_cache.find(fid); it != self->path_cache.end()) {
return it->second;
}

View File

@@ -7,6 +7,7 @@
#include "syntax/token.h"
#include "kota/meta/annotation.h"
#include "llvm/ADT/DenseMap.h"
namespace clice {
@@ -42,7 +43,10 @@ struct IncludeGraph {
/// Each `FileID` represents a new header context and is introduced
/// by a new include directive. So a include directive is a new header
/// context. A map between FileID and its include location.
llvm::DenseMap<clang::FileID, std::uint32_t> file_table;
///
/// Runtime-only: `clang::FileID` is an AST-scoped handle; on-disk the
/// include graph is fully described by `paths` + `locations`.
kota::meta::skip<llvm::DenseMap<clang::FileID, std::uint32_t>> file_table;
static IncludeGraph from(CompilationUnitRef unit);

View File

@@ -0,0 +1,121 @@
#pragma once
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <utility>
#include <vector>
#include "semantic/relation_kind.h"
#include "semantic/symbol_kind.h"
#include "support/bitmap.h"
#include "kota/codec/arena/traits.h"
#include "kota/codec/detail/fwd.h"
/// Type-level wire traits for clice index types.
///
/// These partially specialize the primary
/// `kota::codec::serialize_traits<S, T>` / `deserialize_traits<D, T>`
/// templates, constrained so only arena backends pick them up. They
/// declare the wire representation for `T` and propagate through map
/// values, sequence elements, and nested containers — no per-field
/// `annotation<T, with<...>>` required.
namespace kota::codec {
/// `std::chrono::milliseconds` ⇄ `int64` tick count.
template <typename S>
requires arena::arena_serializer_like<S>
struct serialize_traits<S, std::chrono::milliseconds> {
using wire_type = std::int64_t;
static std::int64_t serialize(S&, std::chrono::milliseconds value) noexcept {
return value.count();
}
};
template <typename D>
requires arena::arena_deserializer_like<D>
struct deserialize_traits<D, std::chrono::milliseconds> {
using wire_type = std::int64_t;
static std::chrono::milliseconds deserialize(const D&, std::int64_t value) noexcept {
return std::chrono::milliseconds(value);
}
};
/// `RelationKind` ⇄ underlying `uint32` bitflags.
template <typename S>
requires arena::arena_serializer_like<S>
struct serialize_traits<S, clice::RelationKind> {
using wire_type = std::uint32_t;
static std::uint32_t serialize(S&, const clice::RelationKind& k) noexcept {
return k.value();
}
};
template <typename D>
requires arena::arena_deserializer_like<D>
struct deserialize_traits<D, clice::RelationKind> {
using wire_type = std::uint32_t;
static clice::RelationKind deserialize(const D&, std::uint32_t v) noexcept {
return clice::RelationKind(static_cast<clice::RelationKind::Kind>(v));
}
};
/// `SymbolKind` ⇄ underlying `uint8`.
template <typename S>
requires arena::arena_serializer_like<S>
struct serialize_traits<S, clice::SymbolKind> {
using wire_type = std::uint8_t;
static std::uint8_t serialize(S&, const clice::SymbolKind& k) noexcept {
return k.value();
}
};
template <typename D>
requires arena::arena_deserializer_like<D>
struct deserialize_traits<D, clice::SymbolKind> {
using wire_type = std::uint8_t;
static clice::SymbolKind deserialize(const D&, std::uint8_t v) noexcept {
return clice::SymbolKind(v);
}
};
/// `clice::Bitmap` (= `roaring::Roaring`) ⇄ opaque byte blob produced by
/// Roaring's non-portable serialization (matches the legacy wire format).
template <typename S>
requires arena::arena_serializer_like<S>
struct serialize_traits<S, clice::Bitmap> {
using wire_type = std::vector<std::byte>;
static std::vector<std::byte> serialize(S&, const clice::Bitmap& bitmap) {
std::vector<std::byte> buffer;
if(bitmap.isEmpty()) {
return buffer;
}
buffer.resize(bitmap.getSizeInBytes(false));
bitmap.write(reinterpret_cast<char*>(buffer.data()), false);
return buffer;
}
};
template <typename D>
requires arena::arena_deserializer_like<D>
struct deserialize_traits<D, clice::Bitmap> {
using wire_type = std::vector<std::byte>;
static clice::Bitmap deserialize(const D&, std::vector<std::byte> bytes) {
if(bytes.empty()) {
return clice::Bitmap();
}
return clice::Bitmap::read(reinterpret_cast<const char*>(bytes.data()), false);
}
};
} // namespace kota::codec

View File

@@ -1,11 +1,18 @@
#include "index/merged_index.h"
#include <cassert>
#include <cstdint>
#include <ranges>
#include <span>
#include <tuple>
#include "index/serialization.h"
#include "index/kotatsu_adapters.h" // type_adapter specializations
#include "support/filesystem.h"
#include "kota/codec/flatbuffers/deserializer.h"
#include "kota/codec/flatbuffers/proxy.h"
#include "kota/codec/flatbuffers/serializer.h"
#include "kota/meta/annotation.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/Support/raw_os_ostream.h"
@@ -97,7 +104,7 @@ struct CompilationContext {
std::uint32_t canonical_id = 0;
std::uint64_t build_at;
std::uint64_t build_at = 0;
std::vector<IncludeLocation> include_locations;
@@ -125,8 +132,9 @@ struct MergedIndex::Impl {
/// The max canonical id we have allocated.
std::uint32_t max_canonical_id = 0;
/// The reference count of each canonical id.
std::vector<std::uint32_t> canonical_ref_counts;
/// Reference counts per canonical id — derivable from header/compilation
/// contexts at load time, so it doesn't need to live on the wire.
kota::meta::skip<std::vector<std::uint32_t>> canonical_ref_counts;
/// The canonical id set of removed index.
roaring::Roaring removed;
@@ -137,8 +145,8 @@ struct MergedIndex::Impl {
/// All merged symbol relations.
llvm::DenseMap<SymbolHash, llvm::DenseMap<Relation, roaring::Roaring>> relations;
/// Sorted occurrences cache for fast lookup.
std::vector<Occurrence> occurrences_cache;
/// Sorted occurrences cache for fast lookup — rebuilt on demand.
kota::meta::skip<std::vector<Occurrence>> occurrences_cache;
void merge(this Impl& self, std::uint32_t path_id, FileIndex& index, auto&& add_context) {
auto hash = index.hash();
@@ -172,6 +180,18 @@ struct MergedIndex::Impl {
friend bool operator==(const Impl&, const Impl&) = default;
};
namespace {
namespace kfb = kota::codec::flatbuffers;
std::span<const std::uint8_t> buffer_bytes(const llvm::MemoryBuffer& buffer) {
return std::span<const std::uint8_t>(
reinterpret_cast<const std::uint8_t*>(buffer.getBufferStart()),
buffer.getBufferSize());
}
} // namespace
MergedIndex::MergedIndex(std::unique_ptr<llvm::MemoryBuffer> buffer, std::unique_ptr<Impl> impl) :
buffer(std::move(buffer)), impl(std::move(impl)) {}
@@ -196,65 +216,24 @@ void MergedIndex::load_in_memory(this Self& self) {
return;
}
auto bytes = buffer_bytes(*self.buffer);
auto result = kfb::from_flatbuffer(bytes, *self.impl);
if(!result) {
self.buffer.reset();
return;
}
// Rebuild the ref count table from the already-loaded contexts.
auto& index = *self.impl;
auto root = fbs::GetRoot<binary::MergedIndex>(self.buffer->getBufferStart());
index.max_canonical_id = root->max_canonical_id();
for(auto entry: *root->canonical_cache()) {
index.canonical_cache.try_emplace(entry->sha256()->string_view(), entry->canonical_id());
}
index.canonical_ref_counts.clear();
index.canonical_ref_counts.resize(index.max_canonical_id, 0);
for(auto entry: *root->header_contexts()) {
HeaderContext context;
auto path = entry->path_id();
context.version = entry->version();
for(auto include: *entry->includes()) {
index.canonical_ref_counts[include->canonical_id()] += 1;
context.includes.emplace_back(*safe_cast<IncludeContext>(include));
}
index.header_contexts.try_emplace(path, std::move(context));
}
for(auto entry: *root->compilation_contexts()) {
CompilationContext context;
auto path = entry->path_id();
context.version = entry->version();
context.canonical_id = entry->canonical_id();
context.build_at = entry->build_at();
for(auto include: *entry->include_locations()) {
context.include_locations.emplace_back(*safe_cast<IncludeLocation>(include));
}
index.compilation_contexts.try_emplace(path, std::move(context));
}
// Count ref counts from compilation contexts.
for(auto entry: *root->compilation_contexts()) {
index.canonical_ref_counts[entry->canonical_id()] += 1;
}
// Deserialize removed bitmap.
if(root->removed() && root->removed()->size() > 0) {
index.removed = read_bitmap(root->removed());
}
for(auto entry: *root->occurrences()) {
index.occurrences.try_emplace(*safe_cast<Occurrence>(entry->occurrence()),
read_bitmap(entry->context()));
}
for(auto entry: *root->relations()) {
auto& relations = index.relations[entry->symbol()];
for(auto relation_entry: *entry->relations()) {
relations.try_emplace(*safe_cast<Relation>(relation_entry->relation()),
read_bitmap(relation_entry->context()));
for(auto& [_, ctx]: index.header_contexts) {
for(auto& inc: ctx.includes) {
index.canonical_ref_counts[inc.canonical_id] += 1;
}
}
if(root->content()) {
index.content = root->content()->str();
for(auto& [_, ctx]: index.compilation_contexts) {
index.canonical_ref_counts[ctx.canonical_id] += 1;
}
self.buffer.reset();
@@ -279,100 +258,9 @@ void MergedIndex::serialize(this const Self& self, llvm::raw_ostream& out) {
return;
}
auto& index = self.impl;
fbs::FlatBufferBuilder builder(1024);
llvm::SmallVector<char, 1024> buffer;
auto canonical_cache = transform(index->canonical_cache, [&](auto&& value) {
auto&& [hash, canonical_id] = value;
return binary::CreateCacheEntry(builder, CreateString(builder, hash), canonical_id);
});
auto header_contexts = transform(index->header_contexts, [&](auto&& value) {
auto& [path_id, context] = value;
return binary::CreateHeaderContextEntry(
builder,
path_id,
context.version,
CreateStructVector<binary::IncludeContext>(builder, context.includes));
});
auto compilation_contexts = transform(index->compilation_contexts, [&](auto&& value) {
auto& [path_id, context] = value;
return binary::CreateCompilationContextEntry(
builder,
path_id,
context.version,
context.canonical_id,
context.build_at,
CreateStructVector<binary::IncludeLocation>(builder, context.include_locations));
});
llvm::SmallVector<const Occurrence*> occurrence_keys;
occurrence_keys.reserve(index->occurrences.size());
auto occurrences = transform(index->occurrences, [&](auto&& value) {
auto&& [occurrence, bitmap] = value;
buffer.clear();
buffer.resize_for_overwrite(bitmap.getSizeInBytes(false));
bitmap.write(buffer.data(), false);
occurrence_keys.emplace_back(&occurrence);
return binary::CreateOccurrenceEntry(builder,
safe_cast<binary::Occurrence>(&occurrence),
CreateVector(builder, buffer));
});
std::ranges::sort(std::views::zip(occurrence_keys, occurrences), [](auto lhs, auto rhs) {
const auto& lo = *std::get<0>(lhs);
const auto& ro = *std::get<0>(rhs);
return std::tuple(lo.range.begin, lo.range.end, lo.target) <
std::tuple(ro.range.begin, ro.range.end, ro.target);
});
llvm::SmallVector<std::uint64_t> relation_keys;
relation_keys.reserve(index->relations.size());
auto relations = transform(index->relations, [&](auto&& value) {
auto&& [symbol_id, symbol_relations] = value;
auto relations = transform(symbol_relations, [&](auto&& value) {
auto&& [relation, bitmap] = value;
buffer.clear();
buffer.resize_for_overwrite(bitmap.getSizeInBytes(false));
bitmap.write(buffer.data(), false);
return binary::CreateRelationEntry(builder,
safe_cast<binary::Relation>(&relation),
CreateVector(builder, buffer));
});
relation_keys.emplace_back(symbol_id);
return binary::CreateSymbolRelationsEntry(builder,
symbol_id,
CreateVector(builder, relations));
});
std::ranges::sort(std::views::zip(relation_keys, relations), {}, [](auto e) {
return std::get<0>(e);
});
// Serialize removed bitmap.
buffer.clear();
if(!index->removed.isEmpty()) {
buffer.resize_for_overwrite(index->removed.getSizeInBytes(false));
index->removed.write(buffer.data(), false);
}
auto removed = CreateVector(builder, buffer);
auto content_offset = CreateString(builder, index->content);
auto merged_index = binary::CreateMergedIndex(builder,
index->max_canonical_id,
CreateVector(builder, canonical_cache),
CreateVector(builder, header_contexts),
CreateVector(builder, compilation_contexts),
CreateVector(builder, occurrences),
CreateVector(builder, relations),
removed,
content_offset);
builder.Finish(merged_index);
out.write(safe_cast<char>(builder.GetBufferPointer()), builder.GetSize());
auto bytes = kfb::to_flatbuffer(*self.impl);
assert(bytes && "MergedIndex flatbuffer serialization failed");
out.write(reinterpret_cast<const char*>(bytes->data()), bytes->size());
}
void MergedIndex::lookup(this const Self& self,
@@ -420,25 +308,43 @@ void MergedIndex::lookup(this const Self& self,
break;
}
} else if(self.buffer) {
auto index = fbs::GetRoot<binary::MergedIndex>(self.buffer->getBufferStart());
auto& occurrences = *index->occurrences();
// Lazy path: binary-search the sorted occurrences array directly in
// the flatbuffer without materializing the in-memory Impl.
auto root = kfb::table_view<Impl>::from_bytes(buffer_bytes(*self.buffer));
auto entries = root[&Impl::occurrences];
auto it = std::ranges::lower_bound(occurrences, offset, {}, [](auto o) {
return o->occurrence()->range().end();
});
auto read_occurrence = [](auto occ_view) -> Occurrence {
auto range_view = occ_view[&Occurrence::range];
return Occurrence{
LocalSourceRange{range_view[&LocalSourceRange::begin],
range_view[&LocalSourceRange::end]},
occ_view[&Occurrence::target],
};
};
while(it != occurrences.end()) {
auto o = safe_cast<Occurrence>(it->occurrence());
if(o->range.contains(offset)) {
if(!callback(*o)) {
break;
}
it++;
continue;
const std::size_t count = entries.size();
std::size_t lo = 0;
std::size_t hi = count;
while(lo < hi) {
auto mid = lo + (hi - lo) / 2;
auto entry = entries.at(mid);
auto range_view = entry.template get<0>()[&Occurrence::range];
if(range_view[&LocalSourceRange::end] < offset) {
lo = mid + 1;
} else {
hi = mid;
}
}
break;
for(; lo < count; ++lo) {
auto entry = entries.at(lo);
auto occurrence = read_occurrence(entry.template get<0>());
if(!occurrence.range.contains(offset)) {
break;
}
if(!callback(occurrence)) {
break;
}
}
}
}
@@ -470,18 +376,31 @@ void MergedIndex::lookup(this const Self& self,
}
}
} else if(self.buffer) {
auto index = fbs::GetRoot<binary::MergedIndex>(self.buffer->getBufferStart());
auto& entries = *index->relations();
auto it = std::ranges::lower_bound(entries, symbol, {}, [](auto e) { return e->symbol(); });
if(it == entries.end() || it->symbol() != symbol) [[unlikely]] {
// Lazy path: binary-search the outer relations map and iterate the
// inner map without materializing Impl.
auto root = kfb::table_view<Impl>::from_bytes(buffer_bytes(*self.buffer));
auto outer = root[&Impl::relations];
auto entry = outer.find(symbol);
if(!entry) {
return;
}
for(auto entry: *it->relations()) {
auto r = safe_cast<Relation>(entry->relation());
if(r->kind & kind) {
if(!callback(*r)) {
auto inner = entry->template get<1>();
const std::size_t count = inner.size();
for(std::size_t i = 0; i < count; ++i) {
auto rel_view = inner.at(i).template get<0>();
// Kind comes back as the wire uint32 via the type_adapter; rewrap it.
auto relation_kind =
RelationKind(static_cast<RelationKind::Kind>(rel_view[&Relation::kind]));
if(relation_kind & kind) {
auto range_view = rel_view[&Relation::range];
Relation relation{
.kind = relation_kind,
.padding = rel_view[&Relation::padding],
.range = LocalSourceRange{range_view[&LocalSourceRange::begin],
range_view[&LocalSourceRange::end]},
.target_symbol = rel_view[&Relation::target_symbol],
};
if(!callback(relation)) {
break;
}
}
@@ -516,25 +435,31 @@ bool MergedIndex::need_update(this const Self& self, llvm::ArrayRef<llvm::String
return false;
} else if(self.buffer) {
auto index = fbs::GetRoot<binary::MergedIndex>(self.buffer->getBufferStart());
if(index->compilation_contexts()->empty()) {
auto root = kfb::table_view<Impl>::from_bytes(buffer_bytes(*self.buffer));
auto contexts = root[&Impl::compilation_contexts];
if(contexts.empty()) {
return true;
}
auto context = *index->compilation_contexts()->begin();
auto context = contexts.at(0).template get<1>();
auto build_at = context[&CompilationContext::build_at];
auto include_locations = context[&CompilationContext::include_locations];
llvm::DenseSet<std::uint32_t> deps;
for(auto location: *context->include_locations()) {
auto [_, success] = deps.insert(location->path_id());
const std::size_t count = include_locations.size();
for(std::size_t i = 0; i < count; ++i) {
auto location = include_locations.at(i);
auto path_id = location[&IncludeLocation::path_id];
auto [_, success] = deps.insert(path_id);
if(success) {
fs::file_status status;
if(auto err = fs::status(path_mapping[location->path_id()], status)) {
if(auto err = fs::status(path_mapping[path_id], status)) {
return true;
}
auto time = std::chrono::duration_cast<std::chrono::milliseconds>(
status.getLastModificationTime().time_since_epoch());
if(time.count() > context->build_at()) {
if(time.count() > build_at) {
return true;
}
}
@@ -616,10 +541,9 @@ llvm::StringRef MergedIndex::content(this const Self& self) {
if(self.impl) {
return self.impl->content;
} else if(self.buffer) {
auto root = fbs::GetRoot<binary::MergedIndex>(self.buffer->getBufferStart());
if(root->content()) {
return root->content()->string_view();
}
auto root = kfb::table_view<Impl>::from_bytes(buffer_bytes(*self.buffer));
auto view = root[&Impl::content];
return llvm::StringRef(view.data(), view.size());
}
return {};
}

View File

@@ -1,9 +1,22 @@
#include "index/project_index.h"
#include "index/serialization.h"
#include <cassert>
#include <cstdint>
#include <span>
#include "index/kotatsu_adapters.h" // type_adapter specializations
#include "kota/codec/flatbuffers/deserializer.h"
#include "kota/codec/flatbuffers/serializer.h"
namespace clice::index {
namespace {
namespace kfb = kota::codec::flatbuffers;
} // namespace
llvm::SmallVector<std::uint32_t> ProjectIndex::merge(this ProjectIndex& self, TUIndex& index) {
auto& paths = index.graph.paths;
llvm::SmallVector<std::uint32_t> file_ids_map;
@@ -28,79 +41,22 @@ llvm::SmallVector<std::uint32_t> ProjectIndex::merge(this ProjectIndex& self, TU
}
void ProjectIndex::serialize(this ProjectIndex& self, llvm::raw_ostream& os) {
fbs::FlatBufferBuilder builder(1024);
llvm::SmallVector<char, 1024> buffer;
auto i = 0;
auto paths = transform(self.path_pool.paths, [&](llvm::StringRef path) {
auto entry =
binary::CreatePathEntry(builder, CreateString(builder, self.path_pool.paths[i]), i);
i += 1;
return entry;
});
auto indices = transform(self.indices, [&](auto&& value) {
auto&& [source, index] = value;
return binary::PathMapEntry(source, index);
});
auto symbols = transform(self.symbols, [&](auto&& value) {
auto& [symbol_id, symbol] = value;
buffer.clear();
buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false));
symbol.reference_files.write(buffer.data(), false);
return binary::CreateSymbolEntry(builder,
symbol_id,
binary::CreateSymbol(builder,
CreateString(builder, symbol.name),
symbol.kind.value(),
CreateVector(builder, buffer)));
});
auto project_index =
binary::CreateProjectIndex(builder,
CreateVector(builder, paths),
CreateStructVector<binary::PathMapEntry>(builder, indices),
CreateVector(builder, symbols));
builder.Finish(project_index);
os.write(safe_cast<const char>(builder.GetBufferPointer()), builder.GetSize());
auto bytes = kfb::to_flatbuffer(self);
assert(bytes && "ProjectIndex flatbuffer serialization failed");
os.write(reinterpret_cast<const char*>(bytes->data()), bytes->size());
}
ProjectIndex ProjectIndex::from(const void* data) {
auto root = fbs::GetRoot<binary::ProjectIndex>(data);
ProjectIndex ProjectIndex::from(const void* data, std::size_t size) {
ProjectIndex index;
auto& pool = index.path_pool;
pool.paths.resize(root->paths()->size());
for(auto entry: *root->paths()) {
// Normalize backslashes to forward slashes for cross-platform consistency
// (persisted index may contain native-separator paths from Windows).
llvm::SmallString<256> normalized(entry->path()->string_view());
std::replace(normalized.begin(), normalized.end(), '\\', '/');
auto k = pool.save(normalized.str());
pool.paths[entry->id()] = k;
pool.cache.try_emplace(k, entry->id());
if(data == nullptr || size == 0) {
return index;
}
for(auto entry: *root->indices()) {
index.indices.try_emplace(entry->source(), entry->index());
std::span<const std::uint8_t> bytes(static_cast<const std::uint8_t*>(data), size);
auto result = kfb::from_flatbuffer(bytes, index);
if(!result) {
return ProjectIndex();
}
for(auto entry: *root->symbols()) {
auto& symbol = index.symbols[entry->symbol_id()];
auto* fb_symbol = entry->symbol();
if(auto* name = fb_symbol->name()) {
symbol.name = name->str();
}
symbol.kind = SymbolKind(static_cast<std::uint8_t>(fb_symbol->kind()));
symbol.reference_files = read_bitmap(fb_symbol->refs());
}
return index;
}

View File

@@ -2,10 +2,14 @@
#include <algorithm>
#include <cstdint>
#include <string>
#include <vector>
#include "index/tu_index.h"
#include "kota/codec/arena/traits.h"
#include "kota/codec/detail/fwd.h"
#include "kota/support/expected_try.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
@@ -84,7 +88,71 @@ struct ProjectIndex {
void serialize(this ProjectIndex& self, llvm::raw_ostream& os);
static ProjectIndex from(const void* data);
static ProjectIndex from(const void* data, std::size_t size);
};
} // namespace clice::index
namespace kota::codec {
/// `PathPool` on the wire is a flat list of absolute paths; `id` is the
/// position in the vector. The allocator and reverse cache are runtime-only.
///
/// Streaming serialize: iterate `pool.paths` and allocate strings directly
/// into the builder, avoiding the double-copy that a value-mode
/// `wire_type = std::vector<std::string>` conversion would introduce.
template <typename S>
requires arena::arena_serializer_like<S>
struct serialize_traits<S, clice::index::PathPool> {
// Structural wire shape — declared so the flatbuffers proxy views
// a `PathPool` field as an `array_view<std::string>`.
using wire_type = std::vector<std::string>;
static auto serialize(S& s, const clice::index::PathPool& pool)
-> std::expected<typename S::vector_ref, typename S::error_type> {
std::vector<typename S::string_ref> offsets;
offsets.reserve(pool.paths.size());
for(const auto& path: pool.paths) {
auto r = s.alloc_string(std::string_view(path.data(), path.size()));
if(!r) {
return std::unexpected(r.error());
}
offsets.push_back(*r);
}
return s.alloc_string_vector(
std::span<const typename S::string_ref>(offsets.data(), offsets.size()));
}
};
/// Streaming deserialize: read each path out of the flatbuffer's
/// string-vector view directly, interning it into the pool's allocator
/// in-place. Avoids the transient `std::vector<std::string>` the
/// value-mode form would materialize.
template <typename D>
requires arena::arena_deserializer_like<D>
struct deserialize_traits<D, clice::index::PathPool> {
using wire_type = std::vector<std::string>;
static auto deserialize(const D& d,
typename D::TableView view,
typename D::slot_id sid,
clice::index::PathPool& out)
-> std::expected<void, typename D::error_type> {
if(!view.has(sid)) {
return {};
}
KOTA_EXPECTED_TRY_V(auto vec, d.get_string_vector(view, sid));
out.paths.resize(vec.size());
for(std::size_t i = 0; i < vec.size(); ++i) {
auto sv = vec[i];
llvm::SmallString<256> normalized(llvm::StringRef(sv.data(), sv.size()));
std::replace(normalized.begin(), normalized.end(), '\\', '/');
auto interned = out.save(normalized.str());
out.paths[i] = interned;
out.cache.try_emplace(interned, static_cast<std::uint32_t>(i));
}
return {};
}
};
} // namespace kota::codec

View File

@@ -1,173 +0,0 @@
namespace clice.index.binary;
struct Range {
begin : uint;
end : uint;
}
struct Occurrence {
range : Range;
target : ulong;
}
struct Relation {
kind : uint;
padding : uint;
range : Range;
target_symbol : ulong;
}
table CacheEntry {
sha256:
string;
canonical_id:
uint;
}
struct IncludeContext {
include_id : uint;
canonical_id : uint;
}
table HeaderContextEntry {
path_id:
uint;
version:
uint;
includes:
[IncludeContext];
}
struct IncludeLocation {
path_id : uint;
line : uint;
include_id : uint;
}
table CompilationContextEntry {
path_id:
uint;
version:
uint;
canonical_id:
uint;
build_at:
ulong;
include_locations:
[IncludeLocation];
}
table OccurrenceEntry {
occurrence:
Occurrence;
context:
[ubyte];
}
table RelationEntry {
relation:
Relation;
context:
[ubyte];
}
table SymbolRelationsEntry {
symbol:
ulong;
relations:
[RelationEntry];
}
table Symbol {
name:
string;
kind:
ubyte;
refs:
[ubyte];
}
table SymbolEntry {
symbol_id:
ulong;
symbol:
Symbol;
}
table MergedIndex {
max_canonical_id:
uint;
canonical_cache:
[CacheEntry];
header_contexts:
[HeaderContextEntry];
compilation_contexts:
[CompilationContextEntry];
occurrences:
[OccurrenceEntry];
relations:
[SymbolRelationsEntry];
removed:
[ubyte];
content:
string;
}
table TUFileRelationsEntry {
symbol:
ulong;
relations:
[Relation];
}
table TUFileIndexEntry {
file_id:
uint;
occurrences:
[Occurrence];
relations:
[TUFileRelationsEntry];
}
table TUIndex {
built_at:
ulong;
paths:
[string];
locations:
[IncludeLocation];
symbols:
[SymbolEntry];
file_indices:
[TUFileIndexEntry];
main_file_index:
TUFileIndexEntry;
}
table PathEntry {
path:
string;
id:
uint;
}
struct PathMapEntry {
source : uint;
index : uint;
}
table ProjectIndex {
paths:
[PathEntry];
indices:
[PathMapEntry];
symbols:
[SymbolEntry];
}

View File

@@ -1,79 +0,0 @@
#include <cstdint>
#include <ranges>
#include <type_traits>
#include "schema_generated.h"
#include "support/bitmap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
namespace clice::index {
namespace fbs = flatbuffers;
namespace {
template <typename Range>
concept sequence_range = std::ranges::input_range<Range> &&
!requires { typename Range::key_type; } && requires(const Range& r) {
r.data();
r.size();
};
template <typename T>
using Offsets = llvm::SmallVector<fbs::Offset<T>, 0>;
template <typename U, typename V>
const U* safe_cast(const V* v) {
static_assert(sizeof(U) == sizeof(V), "size mismatch");
static_assert(alignof(U) == alignof(V), "alignment mismatch");
static_assert(std::is_trivially_copyable_v<U> && std::is_trivially_copyable_v<V>,
"requires trivially copyable");
/// If aliasing issues arise, prefer copying into a temporary SmallVector<U>.
return reinterpret_cast<const U*>(v);
}
auto CreateString(fbs::FlatBufferBuilder& builder, llvm::StringRef string) {
return builder.CreateString(string.data(), string.size());
}
template <sequence_range Range>
auto CreateVector(fbs::FlatBufferBuilder& builder, const Range& range) {
return builder.CreateVector(range.data(), range.size());
}
auto CreateVector(fbs::FlatBufferBuilder& builder, const llvm::SmallVector<char, 1024>& range) {
return builder.CreateVector(reinterpret_cast<const std::uint8_t*>(range.data()), range.size());
}
template <typename U, sequence_range Range>
auto CreateStructVector(fbs::FlatBufferBuilder& builder, const Range& range) {
using V = std::ranges::range_value_t<Range>;
(void)sizeof(V);
return builder.CreateVectorOfStructs(safe_cast<U>(range.data()), range.size());
}
template <typename Range, typename Functor>
auto transform(const Range& range, const Functor& functor) {
using V = std::ranges::range_value_t<Range>;
using R = std::invoke_result_t<Functor, V>;
llvm::SmallVector<R, 0> result;
result.resize_for_overwrite(std::ranges::size(range));
auto i = 0;
for(auto&& v: range) {
result[i] = functor(v);
i += 1;
}
return result;
}
Bitmap read_bitmap(const fbs::Vector<uint8_t>* buffer) {
return Bitmap::read(reinterpret_cast<const char*>(buffer->data()), false);
}
} // namespace
} // namespace clice::index

View File

@@ -1,17 +1,24 @@
#include "index/tu_index.h"
#include <cassert>
#include <cstdint>
#include <span>
#include <tuple>
#include "index/serialization.h"
#include "index/kotatsu_adapters.h" // type_adapter specializations
#include "semantic/ast_utility.h"
#include "semantic/semantic_visitor.h"
#include "kota/codec/flatbuffers/deserializer.h"
#include "kota/codec/flatbuffers/serializer.h"
#include "llvm/Support/SHA256.h"
namespace clice::index {
namespace {
namespace kfb = kota::codec::flatbuffers;
class Builder : public SemanticVisitor<Builder> {
public:
Builder(TUIndex& result, CompilationUnitRef unit, bool interested_only) :
@@ -114,6 +121,8 @@ public:
void build() {
run();
auto interested = unit.interested_file();
for(auto& [fid, index]: result.file_indices) {
for(auto& [symbol_id, relations]: index.relations) {
std::ranges::sort(relations, [](const Relation& lhs, const Relation& rhs) {
@@ -144,13 +153,19 @@ public:
return lhs.range == rhs.range && lhs.target == rhs.target;
});
index.occurrences.erase(range.begin(), range.end());
if(fid == unit.interested_file()) {
result.main_file_index = std::move(index);
}
}
result.file_indices.erase(unit.interested_file());
// Populate main_file_index (interested file) and path_file_indices
// (keyed by path_id) for serialization. `file_indices` itself is
// `skip`-marked (runtime-only, keyed by clang::FileID) and retained
// for in-memory consumers/tests that need FileID access.
for(auto& [fid, index]: result.file_indices) {
if(fid == interested) {
result.main_file_index = index;
} else {
result.path_file_indices[result.graph.path_id(fid)] = index;
}
}
}
private:
@@ -198,119 +213,23 @@ TUIndex TUIndex::build(CompilationUnitRef unit, bool interested_only) {
return index;
}
void TUIndex::serialize(llvm::raw_ostream& os) const {
fbs::FlatBufferBuilder builder(4096);
llvm::SmallVector<char, 1024> buffer;
auto paths =
transform(graph.paths, [&](const std::string& p) { return builder.CreateString(p); });
auto syms = transform(symbols, [&](auto&& value) {
auto& [symbol_id, symbol] = value;
buffer.clear();
buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false));
symbol.reference_files.write(buffer.data(), false);
return binary::CreateSymbolEntry(builder,
symbol_id,
binary::CreateSymbol(builder,
CreateString(builder, symbol.name),
symbol.kind.value(),
CreateVector(builder, buffer)));
});
/// Serialize a single FileIndex into a TUFileIndexEntry.
auto serialize_file_index = [&](std::uint32_t fid, const FileIndex& index) {
auto occs = CreateStructVector<binary::Occurrence>(builder, index.occurrences);
auto rels = transform(index.relations, [&](auto&& value) {
auto& [symbol_id, relations] = value;
return binary::CreateTUFileRelationsEntry(
builder,
symbol_id,
CreateStructVector<binary::Relation>(builder, relations));
});
return binary::CreateTUFileIndexEntry(builder, fid, occs, CreateVector(builder, rels));
};
/// Convert FileID-keyed file_indices to path_id-keyed entries.
llvm::SmallVector<fbs::Offset<binary::TUFileIndexEntry>> file_idx_vec;
for(auto& [fid, index]: file_indices) {
auto pid = graph.path_id(fid);
file_idx_vec.push_back(serialize_file_index(pid, index));
}
/// Main file is the last path in graph.paths (convention from IncludeGraph).
auto main_idx =
serialize_file_index(static_cast<std::uint32_t>(graph.paths.size() - 1), main_file_index);
auto tu_index =
binary::CreateTUIndex(builder,
static_cast<std::uint64_t>(built_at.count()),
CreateVector(builder, paths),
CreateStructVector<binary::IncludeLocation>(builder, graph.locations),
CreateVector(builder, syms),
builder.CreateVector(file_idx_vec.data(), file_idx_vec.size()),
main_idx);
builder.Finish(tu_index);
os.write(safe_cast<const char>(builder.GetBufferPointer()), builder.GetSize());
void TUIndex::serialize(llvm::raw_ostream& os) {
auto bytes = kfb::to_flatbuffer(*this);
assert(bytes && "TUIndex flatbuffer serialization failed");
os.write(reinterpret_cast<const char*>(bytes->data()), bytes->size());
}
TUIndex TUIndex::from(const void* data) {
auto root = fbs::GetRoot<binary::TUIndex>(data);
TUIndex TUIndex::from(const void* data, std::size_t size) {
TUIndex index;
index.built_at = std::chrono::milliseconds(root->built_at());
for(auto p: *root->paths()) {
index.graph.paths.emplace_back(p->str());
if(data == nullptr || size == 0) {
return index;
}
for(auto loc: *root->locations()) {
index.graph.locations.emplace_back(*safe_cast<IncludeLocation>(loc));
std::span<const std::uint8_t> bytes(static_cast<const std::uint8_t*>(data), size);
auto result = kfb::from_flatbuffer(bytes, index);
if(!result) {
return TUIndex();
}
for(auto entry: *root->symbols()) {
auto& symbol = index.symbols[entry->symbol_id()];
symbol.name = entry->symbol()->name()->str();
symbol.kind = SymbolKind(static_cast<std::uint8_t>(entry->symbol()->kind()));
symbol.reference_files = read_bitmap(entry->symbol()->refs());
}
/// Helper to deserialize a TUFileIndexEntry into a FileIndex.
auto deserialize_file_index = [](const binary::TUFileIndexEntry* entry) -> FileIndex {
FileIndex fi;
if(entry->occurrences()) {
fi.occurrences.reserve(entry->occurrences()->size());
for(auto o: *entry->occurrences()) {
fi.occurrences.emplace_back(*safe_cast<Occurrence>(o));
}
}
if(entry->relations()) {
for(auto rel_entry: *entry->relations()) {
auto& rels = fi.relations[rel_entry->symbol()];
if(rel_entry->relations()) {
rels.reserve(rel_entry->relations()->size());
for(auto r: *rel_entry->relations()) {
rels.emplace_back(*safe_cast<Relation>(r));
}
}
}
}
return fi;
};
/// Populate path_file_indices keyed by path_id (no clang::FileID needed).
if(root->file_indices()) {
for(auto entry: *root->file_indices()) {
index.path_file_indices[entry->file_id()] = deserialize_file_index(entry);
}
}
if(root->main_file_index()) {
index.main_file_index = deserialize_file_index(root->main_file_index());
}
return index;
}

View File

@@ -12,6 +12,7 @@
#include "semantic/symbol_kind.h"
#include "support/bitmap.h"
#include "kota/meta/annotation.h"
#include "llvm/Support/raw_ostream.h"
namespace clice::index {
@@ -35,6 +36,10 @@ struct Relation {
constexpr auto definition_range() {
return std::bit_cast<LocalSourceRange>(target_symbol);
}
friend bool operator==(const Relation&, const Relation&) = default;
friend auto operator<=>(const Relation&, const Relation&) = default;
};
struct Occurrence {
@@ -45,6 +50,8 @@ struct Occurrence {
SymbolHash target;
friend bool operator==(const Occurrence&, const Occurrence&) = default;
friend auto operator<=>(const Occurrence&, const Occurrence&) = default;
};
struct FileIndex {
@@ -77,19 +84,21 @@ struct TUIndex {
SymbolTable symbols;
llvm::DenseMap<clang::FileID, FileIndex> file_indices;
/// Runtime-only: keyed by AST-scoped `clang::FileID` during build; flushed
/// into `path_file_indices` (keyed by path id) before serialization.
kota::meta::skip<llvm::DenseMap<clang::FileID, FileIndex>> file_indices;
/// File indices keyed by path_id, populated by from() for deserialized data.
/// When built from AST, this is empty and file_indices (keyed by FileID) is used.
/// File indices keyed by path_id. Populated from `file_indices` at
/// serialize time, and directly from the wire on deserialize.
llvm::DenseMap<std::uint32_t, FileIndex> path_file_indices;
FileIndex main_file_index;
static TUIndex build(CompilationUnitRef unit, bool interested_only = false);
void serialize(llvm::raw_ostream& os) const;
void serialize(llvm::raw_ostream& os);
static TUIndex from(const void* data);
static TUIndex from(const void* data, std::size_t size);
};
} // namespace clice::index

View File

@@ -308,10 +308,6 @@ const clang::NamedDecl* decl_of_impl(const void* T) {
}
auto decl_of(clang::QualType type) -> const clang::NamedDecl* {
if(type.isNull()) {
return nullptr;
}
// Strip type-sugar that wraps the underlying type without adding a decl
// (e.g. ElaboratedType for "struct Foo" vs plain "Foo").
if(auto ET = type->getAs<clang::ElaboratedType>()) {

View File

@@ -71,6 +71,10 @@ constexpr bool operator==(RelationKind lhs, RelationKind rhs) {
return lhs.value() == rhs.value();
}
constexpr auto operator<=>(RelationKind lhs, RelationKind rhs) {
return lhs.value() <=> rhs.value();
}
constexpr bool operator&(RelationKind lhs, RelationKind rhs) {
return lhs.value() == rhs.value();
}

View File

@@ -122,11 +122,9 @@ void Compiler::init_compile_graph() {
auto result = co_await pool.send_stateless(bp);
if(!result.has_value() || !result.value().success) {
auto error_msg = result.has_value() ? result.value().error : result.error().message;
LOG_WARN("BuildPCM failed for module {}: {}", mod_it->second, error_msg);
peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("PCM build failed for module {}: {}", mod_it->second, error_msg)});
LOG_WARN("BuildPCM failed for module {}: {}",
mod_it->second,
result.has_value() ? result.value().error : result.error().message);
co_return false;
}
@@ -173,10 +171,6 @@ bool Compiler::fill_compile_args(llvm::StringRef path,
auto& cmd = results.front();
directory = cmd.resolved.directory.str();
arguments = cmd.to_string_argv();
LOG_DEBUG("fill_compile_args: CDB match for {} (dir={}, {} args)",
path,
directory,
arguments.size());
return true;
}
@@ -496,22 +490,6 @@ kota::task<bool> Compiler::ensure_pch(Session& session,
auto completion = std::make_shared<kota::event>();
workspace.pch_cache[path_id].building = completion;
if(workspace.config.project.cache_dir.empty()) {
LOG_WARN("PCH build skipped: cache_dir is not configured");
workspace.pch_cache[path_id].building.reset();
completion->set();
co_return false;
}
// Ensure the PCH cache directory exists.
auto pch_dir = path::join(workspace.config.project.cache_dir, "cache", "pch");
if(auto ec = llvm::sys::fs::create_directories(pch_dir)) {
LOG_WARN("Cannot create PCH cache dir {}: {}", pch_dir, ec.message());
workspace.pch_cache[path_id].building.reset();
completion->set();
co_return false;
}
// Build a new PCH via stateless worker.
worker::BuildParams bp;
bp.kind = worker::BuildKind::BuildPCH;
@@ -527,11 +505,9 @@ kota::task<bool> Compiler::ensure_pch(Session& session,
auto result = co_await pool.send_stateless(bp);
if(!result.has_value() || !result.value().success) {
auto error_msg = result.has_value() ? result.value().error : result.error().message;
LOG_WARN("PCH build failed for {}: {}", path, error_msg);
peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("PCH build failed for {}: {}", path, error_msg)});
LOG_WARN("PCH build failed for {}: {}",
path,
result.has_value() ? result.value().error : result.error().message);
workspace.pch_cache[path_id].building.reset();
completion->set();
co_return false;
@@ -738,10 +714,6 @@ kota::task<bool> Compiler::ensure_compiled(Session& session) {
params.version = sess->version;
params.text = sess->text;
if(!self->fill_compile_args(file_path, params.directory, params.arguments, sess)) {
LOG_WARN("ensure_compiled: no compile args for {}", uri_str);
self->peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("No compile arguments available for {}", file_path)});
finish_compile();
co_return;
}
@@ -749,9 +721,6 @@ kota::task<bool> Compiler::ensure_compiled(Session& session) {
if(!co_await self
->ensure_deps(*sess, params.directory, params.arguments, params.pch, params.pcms)) {
LOG_WARN("Dependency preparation failed for {}, skipping compile", uri_str);
self->peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("Dependency preparation failed for {}", file_path)});
finish_compile();
co_return;
}
@@ -783,9 +752,6 @@ kota::task<bool> Compiler::ensure_compiled(Session& session) {
if(!result.has_value()) {
LOG_WARN("Compile failed for {}: {}", uri_str, result.error().message);
self->peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Error,
std::format("Compilation failed for {}: {}", file_path, result.error().message)});
self->clear_diagnostics(uri_str);
finish_compile();
co_return;
@@ -797,7 +763,8 @@ kota::task<bool> Compiler::ensure_compiled(Session& session) {
// Store open file index from the stateful worker's TUIndex.
if(!result.value().tu_index_data.empty()) {
auto tu_index = index::TUIndex::from(result.value().tu_index_data.data());
auto tu_index = index::TUIndex::from(result.value().tu_index_data.data(),
result.value().tu_index_data.size());
OpenFileIndex ofi;
ofi.file_index = std::move(tu_index.main_file_index);
ofi.symbols = std::move(tu_index.symbols);
@@ -834,17 +801,11 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind,
auto text = session.text;
if(!co_await ensure_compiled(session)) {
LOG_WARN("forward_query: compilation failed for {}", path);
co_await kota::fail("Compilation failed");
co_return serde_raw{"null"};
}
auto sit = sessions.find(path_id);
if(sit == sessions.end()) {
LOG_WARN("forward_query: session lost after compile for {}", path);
co_await kota::fail("Document was closed during compilation");
}
if(sit->second.ast_dirty) {
LOG_DEBUG("forward_query: still dirty after compile for {} (concurrent edit)", path);
if(sit == sessions.end() || sit->second.ast_dirty) {
co_return serde_raw{"null"};
}
@@ -856,13 +817,8 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind,
if(position) {
auto offset = mapper.to_offset(*position);
if(!offset) {
LOG_WARN("forward_query: invalid position {}:{} for {}",
position->line,
position->character,
path);
co_await kota::fail("Invalid position: failed to convert to byte offset");
}
if(!offset)
co_return serde_raw{"null"};
wp.offset = *offset;
}
@@ -876,8 +832,7 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind,
auto result = co_await pool.send_stateful(path_id, wp);
if(!result.has_value()) {
LOG_WARN("forward_query: worker failed for {}: {}", path, result.error().message);
co_await kota::fail(result.error().message);
co_return serde_raw{};
}
co_return std::move(result.value());
}
@@ -896,36 +851,27 @@ Compiler::RawResult Compiler::forward_build(worker::BuildKind kind,
wp.version = session.version;
wp.text = session.text;
if(!fill_compile_args(path, wp.directory, wp.arguments, &session)) {
LOG_WARN("forward_build: compile args not available for {}", path);
co_await kota::fail("Compile arguments not available");
co_return serde_raw{};
}
if(!co_await ensure_deps(session, wp.directory, wp.arguments, wp.pch, wp.pcms)) {
LOG_WARN("forward_build: dependency preparation failed for {}", path);
co_await kota::fail("Dependency preparation failed");
co_return serde_raw{};
}
// After co_await, verify session still exists.
if(sessions.find(path_id) == sessions.end()) {
LOG_WARN("forward_build: session lost after co_await for {}", path);
co_await kota::fail("Document was closed during compilation");
co_return serde_raw{};
}
lsp::PositionMapper mapper(wp.text, lsp::PositionEncoding::UTF16);
auto offset = mapper.to_offset(position);
if(!offset) {
LOG_WARN("forward_build: invalid position {}:{} for {}",
position.line,
position.character,
path);
co_await kota::fail("Invalid position: failed to convert to byte offset");
}
if(!offset)
co_return serde_raw{"null"};
wp.offset = *offset;
auto result = co_await pool.send_stateless(wp);
if(!result.has_value()) {
LOG_WARN("forward_build: worker failed for {}: {}", path, result.error().message);
co_await kota::fail(result.error().message);
co_return serde_raw{};
}
co_return std::move(result.value().result_json);
}
@@ -943,10 +889,8 @@ Compiler::RawResult Compiler::handle_completion(const protocol::Position& positi
pctx.kind == CompletionContext::IncludeAngled) {
std::string directory;
std::vector<std::string> arguments;
if(!fill_compile_args(path, directory, arguments)) {
LOG_WARN("handle_completion: compile args not available for {}", path);
co_await kota::fail("Compile arguments not available for include completion");
}
if(!fill_compile_args(path, directory, arguments))
co_return serde_raw{"[]"};
std::vector<const char*> args_ptrs;
args_ptrs.reserve(arguments.size());

View File

@@ -14,7 +14,7 @@
#include "syntax/completion.h"
#include "kota/async/async.h"
#include "kota/codec/json/json.h"
#include "kota/codec/raw_value.h"
#include "kota/ipc/codec/json.h"
#include "kota/ipc/lsp/protocol.h"
#include "kota/ipc/peer.h"

View File

@@ -6,9 +6,8 @@
#include "support/glob_pattern.h"
#include "support/logging.h"
#include "kota/async/io/system.h"
#include "kota/codec/json/json.h"
#include "kota/codec/toml/toml.h"
#include "kota/codec/toml.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
@@ -66,10 +65,8 @@ void Config::apply_defaults(llvm::StringRef workspace_root) {
if(p.stateful_worker_count == 0)
p.stateful_worker_count = 2;
if(p.stateless_worker_count == 0) {
auto cores = kota::sys::parallelism();
p.stateless_worker_count = std::max(cores / 2, 2u);
}
if(p.stateless_worker_count == 0)
p.stateless_worker_count = 3;
if(p.worker_memory_limit == 0)
p.worker_memory_limit = 4ULL * 1024 * 1024 * 1024; // 4GB
@@ -168,7 +165,7 @@ std::optional<Config> Config::load_from_json(llvm::StringRef json, llvm::StringR
return config;
}
Config Config::load_from_workspace(llvm::StringRef workspace_root, std::string* warning) {
Config Config::load_from_workspace(llvm::StringRef workspace_root) {
if(!workspace_root.empty()) {
for(auto* name: {"clice.toml", ".clice/config.toml"}) {
auto config_path = path::join(workspace_root, name);
@@ -179,9 +176,6 @@ Config Config::load_from_workspace(llvm::StringRef workspace_root, std::string*
// Present but malformed: fall through to defaults, but surface
// the situation clearly so users know their config wasn't applied.
LOG_WARN("Falling back to default configuration because {} is invalid", config_path);
if(warning)
*warning = std::format("Configuration file {} is invalid, falling back to defaults",
config_path);
}
}

View File

@@ -73,10 +73,7 @@ struct Config {
/// Load config from the workspace, trying standard locations.
/// Returns a default config (with apply_defaults) if no file is found.
/// If `warning` is non-null and a config file was found but malformed,
/// the warning message is written there.
static Config load_from_workspace(llvm::StringRef workspace_root,
std::string* warning = nullptr);
static Config load_from_workspace(llvm::StringRef workspace_root);
};
} // namespace clice

View File

@@ -1,6 +1,5 @@
#include "server/indexer.h"
#include <algorithm>
#include <string>
#include <variant>
#include <vector>
@@ -26,7 +25,7 @@ namespace clice {
namespace lsp = kota::ipc::lsp;
void Indexer::merge(const void* tu_index_data, std::size_t size) {
auto tu_index = index::TUIndex::from(tu_index_data);
auto tu_index = index::TUIndex::from(tu_index_data, size);
if(tu_index.graph.paths.empty()) {
LOG_WARN("Ignoring TUIndex with empty path graph");
return;
@@ -145,7 +144,8 @@ void Indexer::load(llvm::StringRef index_dir) {
auto project_path = path::join(index_dir, "project.idx");
auto buf = llvm::MemoryBuffer::getFile(project_path);
if(buf) {
workspace.project_index = index::ProjectIndex::from((*buf)->getBufferStart());
workspace.project_index =
index::ProjectIndex::from((*buf)->getBufferStart(), (*buf)->getBufferSize());
LOG_INFO("Loaded ProjectIndex: {} symbols", workspace.project_index.symbols.size());
}
@@ -625,23 +625,6 @@ void Indexer::enqueue(std::uint32_t server_path_id) {
index_queue.push_back(server_path_id);
}
void Indexer::pause_indexing() {
++pause_depth;
if(pause_depth == 1) {
resume_event.reset();
LOG_DEBUG("Background indexing paused");
}
}
void Indexer::resume_indexing() {
if(pause_depth > 0)
--pause_depth;
if(pause_depth == 0) {
resume_event.set();
LOG_DEBUG("Background indexing resumed");
}
}
void Indexer::schedule() {
if(!*workspace.config.project.enable_indexing || indexing_active || indexing_scheduled)
return;
@@ -654,76 +637,6 @@ void Indexer::schedule() {
loop.schedule(run_background_indexing());
}
kota::task<> Indexer::index_one(std::uint32_t server_path_id) {
auto file_path = std::string(workspace.path_pool.resolve(server_path_id));
if(sessions.contains(server_path_id))
co_return;
if(!need_update(file_path))
co_return;
// For module interface units, compile their PCM (and transitive deps)
// first so the stateless worker has the artifacts it needs.
if(workspace.compile_graph && workspace.path_to_module.contains(server_path_id)) {
co_await workspace.compile_graph->compile(server_path_id);
}
worker::BuildParams params;
params.kind = worker::BuildKind::Index;
params.file = file_path;
if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr))
co_return;
workspace.fill_pcm_deps(params.pcms);
LOG_INFO("Background indexing: {}", file_path);
auto result = co_await pool.send_stateless(params);
if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) {
LOG_INFO("Background indexing got TUIndex for {}: {} bytes",
file_path,
result.value().tu_index_data.size());
merge(result.value().tu_index_data.data(), result.value().tu_index_data.size());
} else if(result.has_value() && !result.value().success) {
LOG_WARN("Background index failed for {}: {}", file_path, result.value().error);
} else if(result.has_value() && result.value().tu_index_data.empty()) {
LOG_WARN("Background index returned empty TUIndex for {}", file_path);
} else {
LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message);
}
}
kota::task<> Indexer::monitor_resources(std::uint32_t generation) {
while(generation == monitor_generation) {
co_await kota::sleep(std::chrono::milliseconds(3000), loop);
if(generation != monitor_generation)
break;
auto mem = kota::sys::memory();
if(mem.total == 0)
continue;
// Respect cgroup/container limits when present.
auto effective_total =
(mem.constrained > 0 && mem.constrained < mem.total) ? mem.constrained : mem.total;
auto ratio = static_cast<double>(mem.available) / static_cast<double>(effective_total);
if(ratio < 0.15 && max_concurrent > 1) {
--max_concurrent;
LOG_INFO("Index concurrency -> {} (memory pressure: {:.0f}% available)",
max_concurrent,
ratio * 100);
} else if(ratio > 0.30 && max_concurrent < baseline_concurrent) {
++max_concurrent;
LOG_DEBUG("Index concurrency -> {} (memory OK: {:.0f}% available)",
max_concurrent,
ratio * 100);
}
}
}
kota::task<> Indexer::run_background_indexing() {
if(index_idle_timer) {
co_await index_idle_timer->wait();
@@ -736,88 +649,48 @@ kota::task<> Indexer::run_background_indexing() {
}
indexing_active = true;
++monitor_generation;
loop.schedule(monitor_resources(monitor_generation));
std::size_t processed = 0;
// Put module interface units first so their PCMs are built before
// non-module files that might import them.
std::stable_partition(
index_queue.begin() + index_queue_pos,
index_queue.end(),
[this](std::uint32_t id) { return workspace.path_to_module.contains(id); });
while(index_queue_pos < index_queue.size()) {
auto server_path_id = index_queue[index_queue_pos];
index_queue_pos++;
auto batch = index_queue.size() - index_queue_pos;
std::size_t dispatched = 0;
std::size_t completed = 0;
finished = 0;
auto file_path = std::string(workspace.path_pool.resolve(server_path_id));
// Progress reporting via LSP $/progress.
std::optional<lsp::ProgressReporter<kota::ipc::JsonPeer>> progress;
if(peer) {
progress.emplace(*peer, protocol::ProgressToken(std::string("clice/backgroundIndex")));
auto create_result = co_await progress->create();
if(!create_result.has_error()) {
progress->begin("Indexing", std::format("0/{} files", batch), 0);
if(sessions.contains(server_path_id))
continue;
if(!need_update(file_path))
continue;
worker::BuildParams params;
params.kind = worker::BuildKind::Index;
params.file = file_path;
if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr))
continue;
workspace.fill_pcm_deps(params.pcms);
LOG_INFO("Background indexing: {}", file_path);
auto result = co_await pool.send_stateless(params);
if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) {
LOG_INFO("Background indexing got TUIndex for {}: {} bytes",
file_path,
result.value().tu_index_data.size());
merge(result.value().tu_index_data.data(), result.value().tu_index_data.size());
++processed;
} else if(result.has_value() && !result.value().success) {
LOG_WARN("Background index failed for {}: {}", file_path, result.value().error);
} else if(result.has_value() && result.value().tu_index_data.empty()) {
LOG_WARN("Background index returned empty TUIndex for {}", file_path);
} else {
progress.reset();
LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message);
}
}
while(index_queue_pos < index_queue.size() || inflight > 0) {
// Dispatch new tasks up to max_concurrent.
while(index_queue_pos < index_queue.size() && inflight < max_concurrent) {
// Wait if paused by a user request.
if(pause_depth > 0) {
co_await resume_event.wait();
}
auto server_path_id = index_queue[index_queue_pos++];
// Quick pre-filter: skip open files and fresh files without
// consuming a concurrency slot.
auto file_path = std::string(workspace.path_pool.resolve(server_path_id));
if(sessions.contains(server_path_id) || !need_update(file_path)) {
++completed;
continue;
}
++inflight;
++dispatched;
// Launch the index task. On completion it decrements
// inflight, bumps finished, and signals the event.
loop.schedule([](Indexer* self, std::uint32_t id, kota::event& done) -> kota::task<> {
co_await self->index_one(id);
--self->inflight;
++self->finished;
done.set();
}(this, server_path_id, completion_event));
}
if(inflight == 0)
break;
// Wait for at least one task to finish.
co_await completion_event.wait();
completion_event.reset();
// Drain all completions that occurred since last wake.
completed += std::exchange(finished, 0);
// Report progress.
if(progress) {
auto pct = batch > 0 ? static_cast<std::uint32_t>(completed * 100 / batch) : 100;
progress->report(std::format("{}/{} files", completed, batch), pct);
}
}
if(progress) {
progress->end(std::format("Indexed {} files", dispatched));
}
indexing_active = false;
++monitor_generation; // Stop the monitor coroutine.
LOG_INFO("Background indexing complete: {} files dispatched", dispatched);
LOG_INFO("Background indexing complete: {} files processed", processed);
save(workspace.config.project.index_dir);
}

View File

@@ -12,9 +12,7 @@
#include "server/workspace.h"
#include "kota/async/async.h"
#include "kota/ipc/codec/json.h"
#include "kota/ipc/lsp/position.h"
#include "kota/ipc/lsp/progress.h"
#include "kota/ipc/lsp/protocol.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
@@ -64,47 +62,6 @@ public:
loop(loop), workspace(workspace), sessions(sessions), pool(pool), compiler(compiler),
is_file_open(std::move(is_file_open)) {}
/// Set the LSP peer for progress reporting. Must be called before
/// schedule() if progress notifications are desired.
void set_peer(kota::ipc::JsonPeer* p) {
peer = p;
}
/// Temporarily pause background indexing to give priority to user
/// requests. Indexing tasks already dispatched to workers continue,
/// but no new tasks will be sent until resume_indexing() is called.
void pause_indexing();
/// Resume background indexing after a pause.
void resume_indexing();
/// RAII guard that pauses indexing for its lifetime.
struct [[nodiscard]] ScopedPause {
Indexer& indexer;
explicit ScopedPause(Indexer& idx) : indexer(idx) {
indexer.pause_indexing();
}
~ScopedPause() {
indexer.resume_indexing();
}
ScopedPause(const ScopedPause&) = delete;
ScopedPause& operator=(const ScopedPause&) = delete;
};
ScopedPause scoped_pause() {
return ScopedPause{*this};
}
/// Set the maximum number of concurrent index tasks.
/// Also sets the baseline that dynamic adjustment will restore to.
void set_max_concurrency(std::size_t n) {
max_concurrent = std::max<std::size_t>(n, 1);
baseline_concurrent = max_concurrent;
}
/// Add a file to the background indexing queue.
void enqueue(std::uint32_t server_path_id);
@@ -218,9 +175,6 @@ private:
/// server-path-id-keyed sessions map to project-level path_ids.
std::function<bool(std::uint32_t)> is_file_open;
/// LSP peer for progress reporting (optional, not owned).
kota::ipc::JsonPeer* peer = nullptr;
/// Background indexing queue and scheduling state.
std::vector<std::uint32_t> index_queue;
std::size_t index_queue_pos = 0;
@@ -228,30 +182,7 @@ private:
bool indexing_scheduled = false;
std::shared_ptr<kota::timer> index_idle_timer;
/// Concurrency control for background indexing.
std::size_t max_concurrent = 2;
std::size_t baseline_concurrent = 2;
std::size_t inflight = 0;
std::size_t finished = 0; ///< Incremented by each completed dispatch task.
/// Pause/resume: when paused, new index tasks wait on this event.
/// Uses a counter so nested pause/resume pairs work correctly.
std::size_t pause_depth = 0;
kota::event resume_event{true};
/// Completion event — signalled by each finished dispatch task so the
/// main loop can wake up. Must be a member (not local to the coroutine)
/// because inflight tasks capture it by reference and may outlive the
/// coroutine frame during server shutdown.
kota::event completion_event;
/// Generation counter — incremented each run so a stale monitor_resources
/// coroutine can detect that its owning run has ended.
std::uint32_t monitor_generation = 0;
kota::task<> run_background_indexing();
kota::task<> index_one(std::uint32_t server_path_id);
kota::task<> monitor_resources(std::uint32_t generation);
};
} // namespace clice

View File

@@ -56,9 +56,9 @@ MasterServer::MasterServer(kota::event_loop& loop,
MasterServer::~MasterServer() = default;
void MasterServer::load_workspace() {
kota::task<> MasterServer::load_workspace() {
if(workspace_root.empty())
return;
co_return;
auto& cfg = workspace.config.project;
@@ -125,10 +125,7 @@ void MasterServer::load_workspace() {
if(cdb_path.empty()) {
LOG_WARN("No compile_commands.json found in workspace {}", workspace_root);
peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("No compile_commands.json found in workspace {}", workspace_root)});
return;
co_return;
}
auto count = workspace.cdb.load(cdb_path);
@@ -286,18 +283,10 @@ void MasterServer::register_handlers() {
// any initializationOptions on top so fields not mentioned in the JSON
// keep the values from clice.toml — kotatsu's deserializer only touches
// fields that are present in the input.
std::string config_warning;
workspace.config = Config::load_from_workspace(workspace_root, &config_warning);
if(!config_warning.empty())
peer.send_notification(
protocol::LogMessageParams{protocol::MessageType::Warning, config_warning});
workspace.config = Config::load_from_workspace(workspace_root);
if(!init_options_json.empty()) {
if(auto ov = kota::codec::json::parse(init_options_json, workspace.config); !ov) {
LOG_WARN("Failed to apply initializationOptions: {}", ov.error().to_string());
peer.send_notification(protocol::LogMessageParams{
protocol::MessageType::Warning,
std::format("Failed to apply initializationOptions: {}",
ov.error().to_string())});
} else {
// Re-run apply_defaults so overridden strings get workspace
// substitution and `compiled_rules` is rebuilt if `rules`
@@ -333,8 +322,6 @@ void MasterServer::register_handlers() {
pool_opts.log_dir = session_log_dir;
if(!pool.start(pool_opts)) {
LOG_ERROR("Failed to start worker pool");
peer.send_notification(protocol::LogMessageParams{protocol::MessageType::Error,
"Failed to start worker pool"});
return;
}
@@ -344,10 +331,7 @@ void MasterServer::register_handlers() {
indexer.schedule();
};
indexer.set_peer(&peer);
indexer.set_max_concurrency(cfg.stateless_worker_count.value);
load_workspace();
loop.schedule(load_workspace());
});
peer.on_request(
@@ -501,7 +485,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::Hover,
sit->second,
params.text_document_position_params.position);
@@ -513,7 +497,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::SemanticTokens, sit->second);
});
@@ -523,7 +507,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::InlayHints,
sit->second,
{},
@@ -536,7 +520,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::FoldingRange, sit->second);
});
@@ -546,7 +530,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::DocumentSymbol, sit->second);
});
@@ -556,7 +540,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
auto& session = sit->second;
auto result = co_await compiler.forward_query(worker::QueryKind::DocumentLink, session);
if(!result.has_value())
@@ -589,7 +573,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::CodeAction, sit->second);
});
@@ -644,7 +628,7 @@ void MasterServer::register_handlers() {
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
co_return serde_raw{"null"};
co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition,
sit->second,
pos);
@@ -686,33 +670,28 @@ void MasterServer::register_handlers() {
/// Feature requests — stateless forwarding.
peer.on_request(
[this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult {
auto path = uri_to_path(params.text_document_position_params.text_document.uri);
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
auto pause = indexer.scoped_pause();
auto result =
co_await compiler.handle_completion(params.text_document_position_params.position,
sit->second);
co_return std::move(result);
});
peer.on_request([this](RequestContext& ctx,
const protocol::SignatureHelpParams& params) -> RawResult {
const protocol::CompletionParams& params) -> RawResult {
auto path = uri_to_path(params.text_document_position_params.text_document.uri);
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_await kota::fail("Document not open");
auto pause = indexer.scoped_pause();
auto result = co_await compiler.forward_build(worker::BuildKind::SignatureHelp,
co_return serde_raw{"null"};
co_return co_await compiler.handle_completion(params.text_document_position_params.position,
sit->second);
});
peer.on_request(
[this](RequestContext& ctx, const protocol::SignatureHelpParams& params) -> RawResult {
auto path = uri_to_path(params.text_document_position_params.text_document.uri);
auto path_id = workspace.path_pool.intern(path);
auto sit = sessions.find(path_id);
if(sit == sessions.end())
co_return serde_raw{"null"};
co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp,
params.text_document_position_params.position,
sit->second);
co_return std::move(result);
});
});
/// Hierarchy queries — index-based.
@@ -738,8 +717,10 @@ void MasterServer::register_handlers() {
const protocol::CallHierarchyIncomingCallsParams& params) -> RawResult {
auto info = resolve_item(params.item.uri, params.item.range, params.item.data);
if(!info)
co_await kota::fail("Failed to resolve call hierarchy item");
co_return serde_raw{"null"};
auto results = indexer.find_incoming_calls(info->hash);
if(results.empty())
co_return serde_raw{"null"};
co_return to_raw(results);
});
@@ -748,8 +729,10 @@ void MasterServer::register_handlers() {
const protocol::CallHierarchyOutgoingCallsParams& params) -> RawResult {
auto info = resolve_item(params.item.uri, params.item.range, params.item.data);
if(!info)
co_await kota::fail("Failed to resolve call hierarchy item");
co_return serde_raw{"null"};
auto results = indexer.find_outgoing_calls(info->hash);
if(results.empty())
co_return serde_raw{"null"};
co_return to_raw(results);
});
@@ -776,8 +759,10 @@ void MasterServer::register_handlers() {
const protocol::TypeHierarchySupertypesParams& params) -> RawResult {
auto info = resolve_item(params.item.uri, params.item.range, params.item.data);
if(!info)
co_await kota::fail("Failed to resolve type hierarchy item");
co_return serde_raw{"null"};
auto results = indexer.find_supertypes(info->hash);
if(results.empty())
co_return serde_raw{"null"};
co_return to_raw(results);
});
@@ -786,14 +771,18 @@ void MasterServer::register_handlers() {
const protocol::TypeHierarchySubtypesParams& params) -> RawResult {
auto info = resolve_item(params.item.uri, params.item.range, params.item.data);
if(!info)
co_await kota::fail("Failed to resolve type hierarchy item");
co_return serde_raw{"null"};
auto results = indexer.find_subtypes(info->hash);
if(results.empty())
co_return serde_raw{"null"};
co_return to_raw(results);
});
peer.on_request(
[this](RequestContext& ctx, const protocol::WorkspaceSymbolParams& params) -> RawResult {
auto results = indexer.search_symbols(params.query);
if(results.empty())
co_return serde_raw{"null"};
co_return to_raw(results);
});

View File

@@ -12,7 +12,7 @@
#include "server/workspace.h"
#include "kota/async/async.h"
#include "kota/codec/json/json.h"
#include "kota/codec/raw_value.h"
#include "kota/ipc/peer.h"
#include "llvm/ADT/DenseMap.h"
@@ -73,7 +73,7 @@ private:
std::string session_log_dir;
std::string init_options_json; ///< Raw JSON from initializationOptions, consumed once.
void load_workspace();
kota::task<> load_workspace();
using RawResult = kota::task<kota::codec::RawValue, kota::ipc::Error>;
};

View File

@@ -9,7 +9,7 @@
#include "syntax/token.h"
#include "kota/codec/json/json.h"
#include "kota/codec/raw_value.h"
#include "kota/ipc/lsp/protocol.h"
#include "kota/ipc/protocol.h"

View File

@@ -94,7 +94,6 @@ class StatefulWorker {
kota::task<kota::codec::RawValue> with_ast(llvm::StringRef path, F&& fn) {
auto it = documents.find(path);
if(it == documents.end()) {
LOG_WARN("with_ast: document not found: {}", path.str());
co_return kota::codec::RawValue{"null"};
}
@@ -106,10 +105,8 @@ class StatefulWorker {
co_await doc->strand.lock();
auto result = co_await kota::queue([&]() -> kota::codec::RawValue {
if(!doc->has_ast || (!doc->unit.completed() && !doc->unit.fatal_error())) {
LOG_WARN("with_ast: AST not available for {}", path.str());
if(!doc->has_ast || (!doc->unit.completed() && !doc->unit.fatal_error()))
return kota::codec::RawValue{"null"};
}
return fn(*doc);
});

View File

@@ -15,22 +15,6 @@
namespace clice {
/// RAII guard that lowers the current process's scheduling priority and
/// restores it on destruction.
struct ScopedNice {
int saved;
explicit ScopedNice(int increment = 10) {
auto p = kota::sys::priority();
saved = p ? *p : 0;
kota::sys::set_priority(saved + increment);
}
~ScopedNice() {
kota::sys::set_priority(saved);
}
};
using kota::ipc::RequestResult;
using RequestContext = kota::ipc::BincodePeer::RequestContext;
@@ -244,8 +228,6 @@ static worker::BuildResult handle_completion(const worker::BuildParams& params)
cp.completion = {params.file, params.offset};
auto items = feature::code_complete(cp);
if(items.empty())
LOG_DEBUG("Completion: no items returned for {}:{}", params.file, params.offset);
LOG_DEBUG("Completion done: {} items, {}ms", items.size(), timer.ms());
worker::BuildResult result;
@@ -269,7 +251,7 @@ static worker::BuildResult handle_signature_help(const worker::BuildParams& para
cp.completion = {params.file, params.offset};
auto help = feature::signature_help(cp);
LOG_DEBUG("SignatureHelp done: {} signatures, {}ms", help.signatures.size(), timer.ms());
LOG_DEBUG("SignatureHelp done: {}ms", timer.ms());
worker::BuildResult result;
result.result_json = to_raw(help);
@@ -301,10 +283,7 @@ int run_stateless_worker_mode(const std::string& worker_name, const std::string&
switch(params.kind) {
case K::BuildPCH: return handle_build_pch(params);
case K::BuildPCM: return handle_build_pcm(params);
case K::Index: {
ScopedNice guard;
return handle_index(params);
}
case K::Index: return handle_index(params);
case K::Completion: return handle_completion(params);
case K::SignatureHelp: return handle_signature_help(params);
}

View File

@@ -8,7 +8,8 @@
#include "compile/compilation.h"
#include "kota/codec/json/json.h"
#include "kota/codec/json/serializer.h"
#include "kota/codec/raw_value.h"
#include "kota/ipc/codec/json.h"
namespace clice {

View File

@@ -13,13 +13,14 @@ namespace {
/// Coroutine that drains a worker's stderr pipe.
/// Workers write their own log files, so this only captures unexpected output
/// (crash stacktraces, assertion failures, sanitizer reports, etc.).
/// (crash stacktraces, assertion failures, etc.) that bypasses spdlog.
kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) {
std::string buffer;
while(true) {
auto result = co_await stderr_pipe.read();
if(!result.has_value())
if(!result.has_value()) {
break;
}
auto& chunk = result.value();
if(chunk.empty())
break;
@@ -33,7 +34,7 @@ kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) {
break;
auto line = buffer.substr(pos, nl - pos);
if(!line.empty()) {
LOG_WARN("{} {}", prefix, line);
LOG_DEBUG("{} {}", prefix, line);
}
pos = nl + 1;
}
@@ -41,7 +42,7 @@ kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) {
}
if(!buffer.empty()) {
LOG_WARN("{} {}", prefix, buffer);
LOG_DEBUG("{} {}", prefix, buffer);
}
}
@@ -107,29 +108,24 @@ bool WorkerPool::spawn_worker(const std::string& self_path,
});
auto& w = workers.back();
w.alive = true;
++alive_count_;
loop.schedule(w.peer->run());
return true;
}
bool WorkerPool::start(const WorkerPoolOptions& options) {
options_ = options;
log_dir_ = options.log_dir;
for(std::uint32_t i = 0; i < options.stateless_count; ++i) {
if(!spawn_worker(options.self_path, false, 0)) {
return false;
}
loop.schedule(monitor_worker(stateless_workers.size() - 1, false));
}
for(std::uint32_t i = 0; i < options.stateful_count; ++i) {
if(!spawn_worker(options.self_path, true, options.worker_memory_limit)) {
return false;
}
loop.schedule(monitor_worker(stateful_workers.size() - 1, true));
}
// Register evicted notification handler for each stateful worker
@@ -149,24 +145,29 @@ bool WorkerPool::start(const WorkerPoolOptions& options) {
kota::task<> WorkerPool::stop() {
LOG_INFO("WorkerPool stopping...");
shutting_down_ = true;
// Close output pipes to signal workers to exit gracefully.
for(auto& w: stateless_workers)
// Close output pipes to signal workers to exit gracefully
for(auto& w: stateless_workers) {
w.peer->close_output();
for(auto& w: stateful_workers)
}
for(auto& w: stateful_workers) {
w.peer->close_output();
}
// Send SIGTERM. monitor_worker coroutines handle the wait.
for(auto& w: stateless_workers)
// Send SIGTERM to all workers
for(auto& w: stateless_workers) {
w.proc.kill(SIGTERM);
for(auto& w: stateful_workers)
}
for(auto& w: stateful_workers) {
w.proc.kill(SIGTERM);
}
// Wait until all monitor_worker coroutines have finished.
if(alive_count_ > 0) {
all_exited_.reset();
co_await all_exited_.wait();
// Wait for all worker processes to exit
for(auto& w: stateless_workers) {
co_await w.proc.wait();
}
for(auto& w: stateful_workers) {
co_await w.proc.wait();
}
LOG_INFO("WorkerPool stopped");
@@ -197,10 +198,7 @@ std::size_t WorkerPool::assign_worker(std::uint32_t path_id) {
std::size_t WorkerPool::pick_least_loaded() {
std::size_t best = 0;
for(std::size_t i = 1; i < stateful_workers.size(); ++i) {
if(!stateful_workers[i].alive)
continue;
if(!stateful_workers[best].alive ||
stateful_workers[i].owned_documents < stateful_workers[best].owned_documents) {
if(stateful_workers[i].owned_documents < stateful_workers[best].owned_documents) {
best = i;
}
}
@@ -235,127 +233,4 @@ void WorkerPool::clear_owner(std::size_t worker_index) {
}
}
kota::task<> WorkerPool::monitor_worker(std::size_t index, bool stateful) {
auto& workers = stateful ? stateful_workers : stateless_workers;
auto& w = workers[index];
auto name = std::string(stateful ? "SF-" : "SL-") + std::to_string(index);
auto result = co_await w.proc.wait();
w.alive = false;
--alive_count_;
if(shutting_down_) {
if(alive_count_ == 0)
all_exited_.set();
co_return;
}
if(result.has_value()) {
auto& exit = result.value();
if(exit.term_signal != 0) {
LOG_ERROR("Worker {} killed by signal {} (restarts: {})",
name,
exit.term_signal,
w.restart_count);
} else {
LOG_ERROR("Worker {} exited with code {} (restarts: {})",
name,
exit.status,
w.restart_count);
}
} else {
LOG_ERROR("Worker {} lost: {} (restarts: {})",
name,
result.error().message(),
w.restart_count);
}
if(stateful)
clear_owner(index);
constexpr unsigned max_restarts = 5;
if(w.restart_count >= max_restarts) {
LOG_ERROR("Worker {} exceeded max restarts ({}), giving up", name, max_restarts);
co_return;
}
if(!respawn_worker(index, stateful)) {
LOG_ERROR("Worker {} respawn failed", name);
}
}
bool WorkerPool::respawn_worker(std::size_t index, bool stateful) {
auto& workers = stateful ? stateful_workers : stateless_workers;
auto old_restart_count = workers[index].restart_count + 1;
auto worker_name = std::string(stateful ? "SF-" : "SL-") + std::to_string(index);
// Close the old peer and retire it so its coroutines (run/write_loop)
// can finish naturally before the object is destroyed.
if(workers[index].peer) {
workers[index].peer->close();
retired_peers.push_back(std::move(workers[index].peer));
}
kota::process::options opts;
opts.file = options_.self_path;
if(stateful) {
opts.args = {options_.self_path,
"--mode",
"stateful-worker",
"--worker-memory-limit",
std::to_string(options_.worker_memory_limit)};
} else {
opts.args = {options_.self_path, "--mode", "stateless-worker"};
}
opts.args.push_back("--worker-name");
opts.args.push_back(worker_name);
if(!log_dir_.empty()) {
opts.args.push_back("--log-dir");
opts.args.push_back(log_dir_);
}
opts.streams = {
kota::process::stdio::pipe(true, false),
kota::process::stdio::pipe(false, true),
kota::process::stdio::pipe(false, true),
};
auto result = kota::process::spawn(opts, loop);
if(!result) {
LOG_ERROR("Failed to respawn worker {}: {}", worker_name, result.error().message());
return false;
}
auto& spawn = *result;
auto transport = std::make_unique<kota::ipc::StreamTransport>(std::move(spawn.stdout_pipe),
std::move(spawn.stdin_pipe));
auto peer = std::make_unique<kota::ipc::BincodePeer>(loop, std::move(transport));
std::string prefix = "[" + worker_name + "]";
loop.schedule(drain_stderr(std::move(spawn.stderr_pipe), prefix));
workers[index] = WorkerProcess{
.proc = std::move(spawn.proc),
.peer = std::move(peer),
.owned_documents = 0,
.alive = true,
.restart_count = old_restart_count,
};
auto& w = workers[index];
++alive_count_;
loop.schedule(w.peer->run());
if(stateful) {
w.peer->on_notification([this](const worker::EvictedParams& params) {
if(on_evicted)
on_evicted(params.path);
});
}
loop.schedule(monitor_worker(index, stateful));
LOG_INFO("Worker {} restarted (attempt {})", worker_name, old_restart_count);
return true;
}
} // namespace clice

View File

@@ -64,8 +64,6 @@ private:
kota::process proc;
std::unique_ptr<kota::ipc::BincodePeer> peer;
std::size_t owned_documents = 0;
bool alive = true;
unsigned restart_count = 0;
};
kota::event_loop& loop;
@@ -82,19 +80,8 @@ private:
void clear_owner(std::size_t worker_index);
std::size_t pick_least_loaded();
bool shutting_down_ = false;
std::size_t alive_count_ = 0;
kota::event all_exited_{true}; // Signalled when alive_count_ reaches 0.
WorkerPoolOptions options_;
std::string log_dir_;
/// Peers moved here during respawn so their coroutines can finish
/// before the object is destroyed.
llvm::SmallVector<std::unique_ptr<kota::ipc::BincodePeer>> retired_peers;
bool spawn_worker(const std::string& self_path, bool stateful, std::uint64_t memory_limit);
bool respawn_worker(std::size_t index, bool stateful);
kota::task<> monitor_worker(std::size_t index, bool stateful);
};
template <typename Params>
@@ -104,10 +91,11 @@ RequestResult<Params> WorkerPool::send_stateful(std::uint32_t path_id,
if(stateful_workers.empty()) {
co_return kota::outcome_error(kota::ipc::Error{"No stateful workers available"});
}
// No timeout: compile tasks run as detached tasks (loop.schedule) that
// are immune to LSP $/cancelRequest. Adding a timeout here would use
// kotatsu's with_token/when_any which has a spurious-cancellation bug
// that kills requests within milliseconds instead of the configured period.
auto idx = assign_worker(path_id);
if(!stateful_workers[idx].alive) {
co_return kota::outcome_error(kota::ipc::Error{"Assigned stateful worker is down"});
}
co_return co_await stateful_workers[idx].peer->send_request(params, opts);
}
@@ -117,16 +105,9 @@ RequestResult<Params> WorkerPool::send_stateless(const Params& params,
if(stateless_workers.empty()) {
co_return kota::outcome_error(kota::ipc::Error{"No stateless workers available"});
}
// Round-robin, skipping dead workers.
auto start = next_stateless;
for(std::size_t i = 0; i < stateless_workers.size(); ++i) {
auto idx = (start + i) % stateless_workers.size();
if(stateless_workers[idx].alive) {
next_stateless = (idx + 1) % stateless_workers.size();
co_return co_await stateless_workers[idx].peer->send_request(params, opts);
}
}
co_return kota::outcome_error(kota::ipc::Error{"All stateless workers are down"});
auto idx = next_stateless;
next_stateless = (next_stateless + 1) % stateless_workers.size();
co_return co_await stateless_workers[idx].peer->send_request(params, opts);
}
template <typename Params>
@@ -134,8 +115,6 @@ void WorkerPool::notify_stateful(std::uint32_t path_id, const Params& params) {
auto it = owner.find(path_id);
if(it == owner.end())
return;
if(!stateful_workers[it->second].alive)
return;
stateful_workers[it->second].peer->send_notification(params);
}

View File

@@ -46,6 +46,8 @@ struct LocalSourceRange {
constexpr bool operator==(const LocalSourceRange& other) const = default;
constexpr auto operator<=>(const LocalSourceRange& other) const = default;
constexpr std::uint32_t length() const {
return end - begin;
}

View File

@@ -10,14 +10,6 @@ import pytest
from tests.integration.utils.client import CliceClient
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""Store test outcome so fixtures can detect failures."""
outcome = yield
rep = outcome.get_result()
setattr(item, f"rep_{rep.when}", rep)
def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption(
"--executable",
@@ -83,8 +75,7 @@ def workspace(request: pytest.FixtureRequest, test_data_dir: Path) -> Path | Non
"""
marker = request.node.get_closest_marker("workspace")
if marker is None:
yield None
return
return None
if not marker.args or not isinstance(marker.args[0], str):
raise pytest.UsageError(
"@pytest.mark.workspace requires a string argument, e.g. "
@@ -97,10 +88,7 @@ def workspace(request: pytest.FixtureRequest, test_data_dir: Path) -> Path | Non
clice_dir = path / ".clice"
if clice_dir.exists():
shutil.rmtree(clice_dir)
yield path
# Post-test cleanup: remove cache generated during the test.
if clice_dir.exists():
shutil.rmtree(clice_dir)
return path
@pytest.fixture
@@ -122,20 +110,12 @@ async def client(
if workspace is not None:
init_options_marker = request.node.get_closest_marker("init_options")
init_options = dict(init_options_marker.args[0]) if init_options_marker else {}
# Force cache_dir into the workspace so .clice/ cleanup prevents stale PCH.
project = dict(init_options.get("project", {}))
project.setdefault("cache_dir", str(workspace / ".clice"))
init_options["project"] = project
init_options = init_options_marker.args[0] if init_options_marker else None
await c.initialize(workspace, initialization_options=init_options)
yield c
test_failed = (
getattr(request.node, "rep_call", None) is not None
and request.node.rep_call.failed
)
await _shutdown_client(c, verbose=test_failed)
await _shutdown_client(c)
def generate_cdb(workspace: Path) -> None:
@@ -168,12 +148,8 @@ async def make_client(executable: Path, workspace: Path) -> CliceClient:
return c
async def _shutdown_client(c: CliceClient, *, verbose: bool = False) -> None:
"""Gracefully shut down a client, force-kill if needed.
When verbose=True (typically on test failure), dump collected log messages
and server stderr to help diagnose the failure.
"""
async def _shutdown_client(c: CliceClient) -> None:
"""Gracefully shut down a client, force-kill if needed."""
try:
await asyncio.wait_for(c.shutdown_async(None), timeout=3.0)
except Exception:
@@ -189,25 +165,15 @@ async def _shutdown_client(c: CliceClient, *, verbose: bool = False) -> None:
try:
server = getattr(c, "_server", None)
if server:
if server.returncode is not None:
print(f"[server] exit code: {server.returncode}", flush=True)
if server.stderr:
stderr_data = await asyncio.wait_for(server.stderr.read(), timeout=2.0)
if stderr_data:
for line in stderr_data.decode(
"utf-8", errors="replace"
).splitlines():
if "[warn]" in line or "[error]" in line or "Sanitizer" in line:
print(f"[server] {line}", flush=True)
if server and server.stderr:
stderr_data = await asyncio.wait_for(server.stderr.read(), timeout=2.0)
if stderr_data:
for line in stderr_data.decode("utf-8", errors="replace").splitlines():
if "[warn]" in line or "[error]" in line:
print(f"[server] {line}", flush=True)
except Exception:
pass
if verbose and c.log_messages:
for msg in c.log_messages:
level = {1: "ERROR", 2: "WARN", 3: "INFO", 4: "LOG"}.get(msg.type, "?")
print(f"[logMessage/{level}] {msg.message}", flush=True)
try:
c._stop_event.set()
for task in c._async_tasks:

View File

@@ -16,7 +16,6 @@ from lsprotocol.types import (
from tests.conftest import make_client, shutdown_client
from tests.integration.utils import write_cdb, doc
from tests.integration.utils.wait import MTIME_GRANULARITY, SETTLE_TIME
from tests.integration.utils.cache import (
list_pch_files,
list_pcm_files,
@@ -101,7 +100,7 @@ async def test_pch_reused_on_close_reopen(client, tmp_path):
# Close.
client.text_document_did_close(DidCloseTextDocumentParams(text_document=doc(uri)))
await asyncio.sleep(SETTLE_TIME)
await asyncio.sleep(0.5)
# Clear diagnostics so we can wait for fresh ones.
client.diagnostics.pop(uri, None)
@@ -228,7 +227,7 @@ async def test_pch_rebuilt_on_header_change(client, tmp_path):
assert len(pch_before) >= 1
# Modify header — changes preamble content hash.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").write_text("#pragma once\nstruct V2 { int b; };\n")
# Also update main.cpp to use V2 so it compiles cleanly.
(tmp_path / "main.cpp").write_text(
@@ -237,7 +236,7 @@ async def test_pch_rebuilt_on_header_change(client, tmp_path):
# Close and reopen to get fresh preamble.
client.text_document_did_close(DidCloseTextDocumentParams(text_document=doc(uri)))
await asyncio.sleep(SETTLE_TIME)
await asyncio.sleep(0.5)
client.diagnostics.pop(uri, None)
uri2, _ = await client.open_and_wait(tmp_path / "main.cpp")

View File

@@ -21,7 +21,7 @@ from lsprotocol.types import (
)
from tests.integration.utils import write_cdb, doc
from tests.integration.utils.wait import MTIME_GRANULARITY, wait_for_recompile
from tests.integration.utils.wait import wait_for_recompile
from tests.integration.utils.assertions import assert_clean_compile, assert_has_errors
@@ -42,7 +42,7 @@ async def test_header_change_invalidates_ast(client, tmp_path):
# Modify header on disk — introduce an error.
# Ensure mtime advances past filesystem granularity (1s on some FSes).
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").write_text(
"inline int value() { return }\n"
) # syntax error
@@ -71,7 +71,7 @@ async def test_header_change_invalidates_pch(client, tmp_path):
# Modify header — rename struct field.
# Ensure mtime advances past filesystem granularity (1s on some FSes).
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").write_text(
"#pragma once\nstruct Foo { int y; };\n" # x -> y
)
@@ -115,22 +115,16 @@ async def test_touch_without_content_change_skips_recompile(client, tmp_path):
assert_clean_compile(client, uri)
# Touch the header — mtime changes but content stays the same.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
original_content = (tmp_path / "header.h").read_text()
(tmp_path / "header.h").write_text(original_content)
# Hover triggers ensure_compiled which runs deps_changed.
# Layer 2 hash confirms nothing actually changed → cached AST reused.
# The first hover may see ast_dirty=true (mtime changed, hash check in progress),
# so retry to let the hash check complete.
hover = None
for _ in range(3):
hover = await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=1, character=4))
)
if hover is not None:
break
await asyncio.sleep(SETTLE_TIME)
# Hover on "main" (line 1, col 4) which should be hoverable.
hover = await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=1, character=4))
)
assert hover is not None
# No new diagnostics should appear — the file is still clean.
@@ -151,7 +145,7 @@ async def test_header_replaced_with_different_content(client, tmp_path):
assert_clean_compile(client, uri)
# Replace header — delete and recreate with a breaking change.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").unlink()
(tmp_path / "header.h").write_text("inline int renamed_value() { return 1; }\n")
@@ -176,7 +170,7 @@ async def test_fix_error_clears_diagnostics(client, tmp_path):
assert_has_errors(client, uri, "Expected diagnostics from broken header")
# Fix the header.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").write_text("inline int value() { return 1; }\n")
# Hover triggers recompilation — diagnostics should clear.
@@ -204,7 +198,7 @@ async def test_multiple_files_share_header(client, tmp_path):
assert_clean_compile(client, uri_b)
# Break the shared header.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "shared.h").write_text("inline int shared() { return }\n")
# Both files should get diagnostics after hover.
@@ -229,7 +223,7 @@ async def test_transitive_header_change(client, tmp_path):
assert_clean_compile(client, uri)
# Modify the transitive dep (base.h).
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "base.h").write_text("inline int base() { return }\n") # broken
await wait_for_recompile(client, uri)
@@ -316,7 +310,7 @@ async def test_didclose_then_reopen(client, tmp_path):
client.text_document_did_close(DidCloseTextDocumentParams(text_document=doc(uri)))
# Modify on disk while closed.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "main.cpp").write_text("int main() { return }\n") # broken
# Reopen — should compile the new (broken) content from disk.
@@ -327,7 +321,7 @@ async def test_didclose_then_reopen(client, tmp_path):
async def test_didclose_clears_hover(client, tmp_path):
"""After didClose, hover on the closed file should return an error."""
"""After didClose, hover on the closed file should return None."""
(tmp_path / "main.cpp").write_text("int main() { return 0; }\n")
write_cdb(tmp_path, ["main.cpp"])
await client.initialize(tmp_path)
@@ -336,10 +330,10 @@ async def test_didclose_clears_hover(client, tmp_path):
client.text_document_did_close(DidCloseTextDocumentParams(text_document=doc(uri)))
with pytest.raises(Exception, match="Document not open"):
await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=0, character=4))
)
hover = await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=0, character=4))
)
assert hover is None, "Hover on closed file should return None"
async def test_didsave_triggers_recompile_for_dependents(client, tmp_path):
@@ -355,7 +349,7 @@ async def test_didsave_triggers_recompile_for_dependents(client, tmp_path):
assert_clean_compile(client, uri)
# Modify header on disk and send didSave.
await asyncio.sleep(MTIME_GRANULARITY)
await asyncio.sleep(1.1)
(tmp_path / "header.h").write_text("inline int value() { return }\n") # broken
client.text_document_did_save(
DidSaveTextDocumentParams(

View File

@@ -10,7 +10,6 @@ from lsprotocol.types import (
)
from tests.integration.utils import doc
from tests.integration.utils.wait import SETTLE_TIME
from tests.integration.utils.workspace import did_change
@@ -71,7 +70,7 @@ async def test_semantic_token_modifier_legend(client, workspace):
@pytest.mark.workspace("hello_world")
async def test_did_open_close_cycle(client, workspace):
uri, _ = client.open(workspace / "main.cpp")
await asyncio.sleep(SETTLE_TIME)
await asyncio.sleep(0.5)
client.close(uri)
@@ -84,8 +83,8 @@ async def test_shutdown_exit(client, workspace):
async def test_feature_requests_after_close(client, workspace):
uri, _ = client.open(workspace / "main.cpp")
client.close(uri)
with pytest.raises(Exception, match="Document not open"):
await client.hover_at(uri, 0, 0)
result = await client.hover_at(uri, 0, 0)
assert result is None
@pytest.mark.workspace("hello_world")
@@ -95,7 +94,7 @@ async def test_incremental_change(client, workspace):
content += f"\n// change {i}"
did_change(client, uri, i + 1, content)
await asyncio.sleep(0.05)
await asyncio.sleep(SETTLE_TIME * 2)
await asyncio.sleep(1)
client.close(uri)
@@ -192,23 +191,23 @@ async def test_rapid_changes_stress(client, workspace):
for i in range(20):
content += f"\n// stress change {i}\n"
did_change(client, uri, i + 1, content)
await asyncio.sleep(SETTLE_TIME * 2)
await asyncio.sleep(2)
client.close(uri)
@pytest.mark.workspace("hello_world")
async def test_save_notification(client, workspace):
uri, _ = client.open(workspace / "main.cpp")
await asyncio.sleep(SETTLE_TIME)
await asyncio.sleep(0.5)
client.text_document_did_save(DidSaveTextDocumentParams(text_document=doc(uri)))
await asyncio.sleep(SETTLE_TIME)
await asyncio.sleep(0.5)
client.close(uri)
@pytest.mark.workspace("hello_world")
async def test_hover_on_unknown_file(client, workspace):
with pytest.raises(Exception, match="Document not open"):
await client.hover_at("file:///nonexistent/fake.cpp", 0, 0)
result = await client.hover_at("file:///nonexistent/fake.cpp", 0, 0)
assert result is None
@pytest.mark.workspace("hello_world")

View File

@@ -13,14 +13,13 @@ from lsprotocol.types import (
)
from tests.integration.utils import doc
from tests.integration.utils.wait import IDLE_TIMEOUT
from tests.integration.utils.workspace import did_change
@pytest.mark.workspace("hello_world")
async def test_did_open(client, workspace):
client.open(workspace / "main.cpp")
await asyncio.sleep(IDLE_TIMEOUT)
await asyncio.sleep(5)
@pytest.mark.workspace("hello_world")
@@ -30,13 +29,13 @@ async def test_did_change(client, workspace):
content += "\n"
await asyncio.sleep(0.2)
did_change(client, uri, i + 1, content)
await asyncio.sleep(IDLE_TIMEOUT)
await asyncio.sleep(5)
@pytest.mark.workspace("clang_tidy")
async def test_clang_tidy(client, workspace):
client.open(workspace / "main.cpp")
await asyncio.sleep(IDLE_TIMEOUT)
await asyncio.sleep(5)
@pytest.mark.workspace("hello_world")
@@ -57,7 +56,7 @@ async def test_hover_save_close(client, workspace):
)
)
client.text_document_did_close(DidCloseTextDocumentParams(text_document=doc(uri)))
with pytest.raises(Exception, match="Document not open"):
await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=0, character=0))
)
closed_hover = await client.text_document_hover_async(
HoverParams(text_document=doc(uri), position=Position(line=0, character=0))
)
assert closed_hover is None

View File

@@ -14,7 +14,6 @@ from lsprotocol.types import (
)
from tests.integration.utils.assertions import assert_clean_compile, assert_has_errors
from tests.integration.utils.wait import IDLE_TIMEOUT
@pytest.mark.workspace("modules/single_module_no_deps")
@@ -268,7 +267,7 @@ async def test_circular_module_dependency(client, workspace):
the server remains responsive by opening a non-cyclic file afterwards.
"""
client.open(workspace / "cycle_a.cppm")
await asyncio.sleep(IDLE_TIMEOUT)
await asyncio.sleep(5.0)
uri_ok, _ = await client.open_and_wait(workspace / "ok.cppm")
diags = client.diagnostics.get(uri_ok, [])

View File

@@ -10,7 +10,6 @@ from lsprotocol.types import (
)
from tests.integration.utils import doc
from tests.integration.utils.wait import SETTLE_TIME
from tests.integration.utils.workspace import did_change
@@ -54,7 +53,7 @@ async def test_rapid_edits_with_hover(client, workspace):
await asyncio.sleep(0.02) # ~20ms between edits
# Wait a moment for in-flight requests to settle.
await asyncio.sleep(SETTLE_TIME * 2)
await asyncio.sleep(1.0)
# Final hover must succeed and return correct result.
final_hover = await asyncio.wait_for(

View File

@@ -1,6 +1,6 @@
"""Diagnostic and log message assertion helpers for integration tests."""
"""Diagnostic assertion helpers for integration tests."""
from lsprotocol.types import Diagnostic, DiagnosticSeverity, MessageType
from lsprotocol.types import Diagnostic, DiagnosticSeverity
def get_errors(diagnostics: list[Diagnostic]) -> list[Diagnostic]:
@@ -48,23 +48,3 @@ def assert_clean_compile(client, uri: str) -> None:
"""Assert the file compiled without any diagnostics at all."""
diags = client.diagnostics.get(uri, [])
assert len(diags) == 0, f"Expected clean compile, got: {diags}"
def has_log_message(
client, substring: str, *, severity: MessageType | None = None
) -> bool:
"""Check if any log message contains the given substring."""
for msg in client.log_messages:
if severity is not None and msg.type != severity:
continue
if substring in msg.message:
return True
return False
def assert_no_log_errors(client) -> None:
"""Assert that no error-level log messages were received."""
errors = [m for m in client.log_messages if m.type == MessageType.Error]
assert len(errors) == 0, (
f"Expected no log errors, got: {[e.message for e in errors]}"
)

View File

@@ -7,7 +7,6 @@ from urllib.parse import unquote
from lsprotocol.types import (
PROGRESS,
TEXT_DOCUMENT_PUBLISH_DIAGNOSTICS,
WINDOW_LOG_MESSAGE,
WINDOW_WORK_DONE_PROGRESS_CREATE,
ClientCapabilities,
CodeActionContext,
@@ -25,7 +24,6 @@ from lsprotocol.types import (
InitializeParams,
InitializeResult,
InitializedParams,
LogMessageParams,
Position,
ProgressParams,
PublishDiagnosticsParams,
@@ -50,7 +48,6 @@ class CliceClient(BaseLanguageClient):
super().__init__("clice-test-client", "0.1.0")
self.diagnostics: dict[str, list[Diagnostic]] = {}
self.diagnostics_events: dict[str, asyncio.Event] = {}
self.log_messages: list[LogMessageParams] = []
self.progress_tokens: list[str] = []
self.progress_events: list[dict] = []
self.init_result: InitializeResult | None = None
@@ -67,10 +64,6 @@ class CliceClient(BaseLanguageClient):
if key in self.diagnostics_events:
self.diagnostics_events[key].set()
@self.feature(WINDOW_LOG_MESSAGE)
def on_log_message(params: LogMessageParams) -> None:
self.log_messages.append(params)
@self.feature(WINDOW_WORK_DONE_PROGRESS_CREATE)
def on_create_progress(params: WorkDoneProgressCreateParams) -> None:
token = str(params.token) if isinstance(params.token, int) else params.token

View File

@@ -9,11 +9,6 @@ from lsprotocol.types import (
WorkspaceSymbolParams,
)
# Standard timing constants — use these instead of hardcoded sleep values.
MTIME_GRANULARITY = 1.1 # Filesystem mtime precision (1s on many FSes, +0.1 margin)
SETTLE_TIME = 0.5 # Time for server to stabilize after an operation
IDLE_TIMEOUT = 5.0 # Time to wait for server idle in lifecycle tests
async def wait_for_recompile(client, uri: str, *, timeout: float = 60.0) -> None:
"""Trigger recompilation via hover and wait for fresh diagnostics.

View File

@@ -13,9 +13,6 @@ import re
import signal
import sys
import time
# Force line-buffered stdout so CI sees output immediately.
sys.stdout.reconfigure(line_buffering=True)
from pathlib import Path
from urllib.parse import quote, unquote
@@ -112,9 +109,7 @@ async def write_lsp_message(writer: asyncio.StreamWriter, payload: str):
await writer.drain()
async def replay_one(
trace_path: Path, clice_bin: Path, timeout: int, wall_timeout: int = 300
) -> bool | None:
async def replay_one(trace_path: Path, clice_bin: Path, timeout: int) -> bool | None:
"""Replay a single trace. Returns True=PASS, False=FAIL, None=SKIP."""
records = load_trace(trace_path)
if not records:
@@ -184,21 +179,8 @@ async def replay_one(
last_method = None
sent_count = 0
wall_deadline = wall_start + wall_timeout
def remaining_wall():
return max(0, wall_deadline - time.monotonic())
try:
for i, rec in enumerate(records):
if remaining_wall() <= 0:
elapsed = time.monotonic() - wall_start
print(
f" result: TIMEOUT (wall-clock {wall_timeout}s exceeded, {elapsed:.1f}s)"
)
success = False
break
if i > 0:
delay = rec["ts"] - records[i - 1]["ts"]
if delay > 0:
@@ -214,7 +196,7 @@ async def replay_one(
try:
await asyncio.wait_for(
asyncio.gather(*pending.values(), return_exceptions=True),
timeout=min(timeout, remaining_wall()),
timeout=timeout,
)
except asyncio.TimeoutError:
elapsed = time.monotonic() - wall_start
@@ -228,19 +210,7 @@ async def replay_one(
if msg_id is not None and method is not None:
pending[msg_id] = asyncio.get_event_loop().create_future()
try:
await asyncio.wait_for(
write_lsp_message(proc.stdin, rec["msg"]),
timeout=min(30, remaining_wall()),
)
except asyncio.TimeoutError:
elapsed = time.monotonic() - wall_start
print(
f" result: HANG (write blocked at {last_method},"
f" sent={sent_count}/{len(records)}, {elapsed:.1f}s)"
)
success = False
break
await write_lsp_message(proc.stdin, rec["msg"])
sent_count = i + 1
except (ConnectionError, BrokenPipeError):
@@ -261,7 +231,7 @@ async def replay_one(
try:
await asyncio.wait_for(
asyncio.gather(*pending.values(), return_exceptions=True),
timeout=min(timeout, remaining_wall()),
timeout=timeout,
)
except asyncio.TimeoutError:
elapsed = time.monotonic() - wall_start
@@ -324,7 +294,7 @@ async def async_main(args):
print(f"SKIP: {trace} (not found)")
skipped += 1
continue
result = await replay_one(trace, args.clice, args.timeout, args.wall_timeout)
result = await replay_one(trace, args.clice, args.timeout)
if result is None:
skipped += 1
elif result:
@@ -347,16 +317,7 @@ def main():
p.add_argument("traces", nargs="+", type=Path, help="JSONL trace files")
p.add_argument("--clice", required=True, type=Path, help="Path to clice binary")
p.add_argument(
"--timeout",
type=int,
default=120,
help="Per-request timeout in seconds (default: 120)",
)
p.add_argument(
"--wall-timeout",
type=int,
default=300,
help="Max wall-clock time per trace in seconds (default: 300)",
"--timeout", type=int, default=120, help="Timeout in seconds (default: 120)"
)
args = p.parse_args()
sys.exit(asyncio.run(async_main(args)))

View File

@@ -21,6 +21,11 @@ void run(llvm::StringRef source, llvm::StringRef standard = "-std=c++17") {
links = feature::document_links(*unit, feature::PositionEncoding::UTF8);
}
auto to_local_range(const protocol::Range& range) -> LocalSourceRange {
feature::PositionMapper converter(unit->interested_content(), feature::PositionEncoding::UTF8);
return LocalSourceRange(*converter.to_offset(range.start), *converter.to_offset(range.end));
}
void EXPECT_LINK(std::size_t index, llvm::StringRef name, llvm::StringRef path) {
auto& link = links[index];
auto expected = range(name, "main.cpp");

View File

@@ -37,10 +37,19 @@ void run(llvm::StringRef code) {
}
auto to_local_range(const protocol::FoldingRange& range) -> LocalSourceRange {
return Tester::to_local_range(protocol::Range{
.start = {.line = range.start_line, .character = range.start_character.value_or(0)},
.end = {.line = range.end_line, .character = range.end_character.value_or(0) },
});
feature::PositionMapper converter(unit->interested_content(), feature::PositionEncoding::UTF8);
auto start = protocol::Position{
.line = range.start_line,
.character = range.start_character.value_or(0),
};
auto end = protocol::Position{
.line = range.end_line,
.character = range.end_character.value_or(0),
};
return LocalSourceRange(*converter.to_offset(start), *converter.to_offset(end));
}
void EXPECT_FOLDING(std::uint32_t index,

View File

@@ -128,7 +128,7 @@ TEST_CASE(SerializationRoundTrip) {
project.serialize(os);
// Deserialize.
auto restored = index::ProjectIndex::from(buf.data());
auto restored = index::ProjectIndex::from(buf.data(), buf.size());
// Path pools should match.
ASSERT_EQ(project.path_pool.paths.size(), restored.path_pool.paths.size());
@@ -190,7 +190,7 @@ TEST_CASE(NameSurvivesRoundTrip) {
llvm::SmallString<4096> buf;
llvm::raw_svector_ostream os(buf);
project.serialize(os);
auto restored = index::ProjectIndex::from(buf.data());
auto restored = index::ProjectIndex::from(buf.data(), buf.size());
// Verify names survive round-trip.
for(auto& [hash, symbol]: project.symbols) {

View File

@@ -6,7 +6,7 @@
#include "support/filesystem.h"
#include "kota/codec/json/json.h"
#include "kota/codec/toml/toml.h"
#include "kota/codec/toml.h"
namespace clice::testing {
@@ -148,7 +148,7 @@ TEST_CASE(ApplyDefaults) {
EXPECT_EQ(*config.project.idle_timeout_ms, 3000);
EXPECT_EQ(config.project.max_active_file.value, 8);
EXPECT_EQ(config.project.stateful_worker_count.value, 2u);
EXPECT_GE(config.project.stateless_worker_count.value, 2u);
EXPECT_EQ(config.project.stateless_worker_count.value, 3u);
EXPECT_FALSE(config.project.cache_dir.empty());
EXPECT_FALSE(config.project.index_dir.empty());
EXPECT_FALSE(config.project.logging_dir.empty());

View File

@@ -5,7 +5,7 @@
#include "server/protocol.h"
#include "server/worker_test_helpers.h"
#include "kota/codec/json/json.h"
#include "kota/codec/raw_value.h"
namespace clice::testing {

View File

@@ -6,6 +6,7 @@
#include "server/worker_test_helpers.h"
#include "kota/codec/bincode/bincode.h"
#include "kota/codec/raw_value.h"
namespace clice::testing {

View File

@@ -8,7 +8,6 @@
#include "test/test.h"
#include "command/command.h"
#include "compile/compilation.h"
#include "feature/feature.h"
#include "support/logging.h"
namespace clice::testing {
@@ -83,12 +82,6 @@ struct Tester {
LocalSourceRange range(llvm::StringRef name = "", llvm::StringRef file = "");
LocalSourceRange to_local_range(const kota::ipc::protocol::Range& range) {
feature::PositionMapper converter(unit->interested_content(),
feature::PositionEncoding::UTF8);
return LocalSourceRange(*converter.to_offset(range.start), *converter.to_offset(range.end));
}
void clear();
};