diff --git a/src/feature/semantic_tokens.cpp b/src/feature/semantic_tokens.cpp index 6a7dd10d..4f81ae82 100644 --- a/src/feature/semantic_tokens.cpp +++ b/src/feature/semantic_tokens.cpp @@ -12,6 +12,7 @@ #include "clang/AST/Attr.h" #include "clang/Basic/IdentifierTable.h" +#include "clang/Basic/Module.h" namespace clice::feature { @@ -168,6 +169,7 @@ public: auto collect() -> std::vector { highlight_lexical(unit.interested_file()); run(); + highlight_modules(); merge_tokens(); return std::move(tokens); } @@ -291,6 +293,58 @@ private: }); } + void highlight_modules() { + auto interested = unit.interested_file(); + + auto directives_it = unit.directives().find(interested); + if(directives_it != unit.directives().end()) { + for(const auto& import: directives_it->second.imports) { + add_token(import.location, SymbolKind::Keyword, 0); + for(auto loc: import.name_locations) { + add_token(loc, SymbolKind::Module, 0); + } + } + } + + auto* mod = unit.context().getCurrentNamedModule(); + if(!mod) { + return; + } + + auto def_loc = mod->DefinitionLoc; + if(!def_loc.isValid() || !def_loc.isFileID()) { + return; + } + + auto [fid, offset] = unit.decompose_location(def_loc); + if(fid != interested) { + return; + } + + auto content = unit.file_content(fid); + auto& lang_opts = unit.lang_options(); + Lexer lexer(content.substr(offset), false, &lang_opts); + + auto module_token = lexer.advance(); + if(module_token.is_identifier()) { + auto range = LocalSourceRange(offset + module_token.range.begin, + offset + module_token.range.end); + tokens.push_back({.range = range, .kind = SymbolKind::Keyword, .modifiers = 0}); + } + + // Scan for identifiers (module name parts) until semicolon/eof. + while(true) { + auto token = lexer.advance(); + if(token.is_eof() || token.kind == clang::tok::semi) { + break; + } + if(token.is_identifier()) { + auto range = LocalSourceRange(offset + token.range.begin, offset + token.range.end); + tokens.push_back({.range = range, .kind = SymbolKind::Module, .modifiers = 0}); + } + } + } + void highlight_lexical(clang::FileID fid) { auto content = unit.file_content(fid); auto& lang_opts = unit.lang_options(); @@ -345,10 +399,17 @@ private: } static void resolve_conflict(RawToken& last, const RawToken& current) { - (void)current; if(last.kind == SymbolKind::Conflict) { return; } + // Directive is a low-priority lexical kind; semantic tokens override it. + if(last.kind == SymbolKind::Directive) { + last = current; + return; + } + if(current.kind == SymbolKind::Directive) { + return; + } last.kind = SymbolKind::Conflict; } diff --git a/src/semantic/semantic_visitor.h b/src/semantic/semantic_visitor.h index 93cb1988..74729195 100644 --- a/src/semantic/semantic_visitor.h +++ b/src/semantic/semantic_visitor.h @@ -131,33 +131,6 @@ public: } } } - - // if(auto module = unit.context().getCurrentNamedModule()) { - // auto keyword = module->DefinitionLoc; - // auto begin = TB.spelledTokenContaining(keyword); - // // assert(begin->kind() == clang::tok::identifier && begin->text(SM) == "module" && - // // "Invalid module declaration"); - // - // begin += 1; - // auto end = TB.spelledTokens(unit.file_id(keyword)).end(); - // - // for(auto iter = begin; iter != end; ++iter) { - // if(iter->kind() == clang::tok::identifier) { - // if(auto next = iter + 1; next != end && (next->kind() == clang::tok::period || - // next->kind() == clang::tok::colon)) { - // iter += 1; - // continue; - // } - // - // end = iter + 1; - // break; - // } - // - // std::unreachable(); - // } - // - // handleModuleOccurrence(keyword, llvm::ArrayRef(begin, end)); - //} } public: diff --git a/tests/unit/feature/semantic_tokens_tests.cpp b/tests/unit/feature/semantic_tokens_tests.cpp index 039da375..41fa73ea 100644 --- a/tests/unit/feature/semantic_tokens_tests.cpp +++ b/tests/unit/feature/semantic_tokens_tests.cpp @@ -423,6 +423,165 @@ cd*/ ASSERT_EQ(comments[1].length, 4); } +TEST_CASE(ModuleDeclaration) { + add_main("main.cpp", R"cpp( +export @kw[module] @mod[foo]; +)cpp"); + ASSERT_TRUE(compile("-std=c++20")); + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("kw", SymbolKind::Keyword); + EXPECT_TOKEN("mod", SymbolKind::Module); +} + +TEST_CASE(ModuleDeclarationDotted) { + add_main("main.cpp", R"cpp( +export @kw[module] @m0[foo].@m1[bar]; +)cpp"); + ASSERT_TRUE(compile("-std=c++20")); + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("kw", SymbolKind::Keyword); + EXPECT_TOKEN("m0", SymbolKind::Module); + EXPECT_TOKEN("m1", SymbolKind::Module); +} + +TEST_CASE(ModuleImport) { + auto pcm_path = fs::createTemporaryFile("test-mod", "pcm"); + ASSERT_TRUE(pcm_path.has_value()); + + { + Tester mod; + mod.add_main("mod.cppm", "export module foo;\nexport int x = 42;\n"); + mod.prepare("-std=c++20"); + mod.params.kind = CompilationKind::ModuleInterface; + mod.params.output_file = *pcm_path; + auto built = clice::compile(mod.params); + ASSERT_TRUE(built.completed()); + } + + add_main("main.cpp", R"cpp( +@kw[import] @mod[foo]; +int y = x; +)cpp"); + prepare("-std=c++20"); + auto fmodule_arg = std::string("-fmodule-file=foo=") + *pcm_path; + owned_args.push_back(fmodule_arg); + params.arguments.clear(); + for(auto& arg: owned_args) { + params.arguments.push_back(arg.c_str()); + } + + auto built = clice::compile(params); + ASSERT_TRUE(built.completed()); + unit.emplace(std::move(built)); + + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("kw", SymbolKind::Keyword); + EXPECT_TOKEN("mod", SymbolKind::Module); + + fs::remove(*pcm_path); +} + +TEST_CASE(ModulePartition) { + add_main("main.cpp", R"cpp( +export module @m0[foo]:@m1[bar]; +)cpp"); + ASSERT_TRUE(compile("-std=c++20")); + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("m0", SymbolKind::Module); + EXPECT_TOKEN("m1", SymbolKind::Module); +} + +TEST_CASE(ModuleReexport) { + auto pcm_path = fs::createTemporaryFile("test-mod", "pcm"); + ASSERT_TRUE(pcm_path.has_value()); + + { + Tester mod; + mod.add_main("mod.cppm", "export module foo;\nexport int x = 42;\n"); + mod.prepare("-std=c++20"); + mod.params.kind = CompilationKind::ModuleInterface; + mod.params.output_file = *pcm_path; + auto built = clice::compile(mod.params); + ASSERT_TRUE(built.completed()); + } + + add_main("main.cppm", R"cpp( +export module bar; +export @kw[import] @mod[foo]; +)cpp"); + prepare("-std=c++20"); + auto fmodule_arg = std::string("-fmodule-file=foo=") + *pcm_path; + owned_args.push_back(fmodule_arg); + params.arguments.clear(); + for(auto& arg: owned_args) { + params.arguments.push_back(arg.c_str()); + } + params.kind = CompilationKind::ModuleInterface; + + auto built = clice::compile(params); + ASSERT_TRUE(built.completed()); + unit.emplace(std::move(built)); + + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("kw", SymbolKind::Keyword); + EXPECT_TOKEN("mod", SymbolKind::Module); + + fs::remove(*pcm_path); +} + +TEST_CASE(GlobalModuleFragment) { + add_main("main.cpp", R"cpp( +module; +export module @mod[foo]; +)cpp"); + ASSERT_TRUE(compile("-std=c++20")); + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("mod", SymbolKind::Module); +} + +TEST_CASE(PrivateModuleFragment) { + add_main("main.cpp", R"cpp( +export module @mod[foo]; +module :private; +int x = 1; +)cpp"); + ASSERT_TRUE(compile("-std=c++20")); + tokens = feature::semantic_tokens(*unit, feature::PositionEncoding::UTF8); + decoded = decode_utf8_tokens(unit->interested_content(), tokens); + + EXPECT_TOKEN("mod", SymbolKind::Module); +} + +TEST_CASE(ModuleKeywordAsIdentifier) { + run_utf8(R"cpp( +void f() { + struct @s0[module] {}; + @s1[module] @v0[m]; + int @v1[import] = 1; + int @v2[module] = 2; +} +)cpp"); + + auto definition = modifier_mask({SymbolModifiers::Definition}); + EXPECT_TOKEN("s0", SymbolKind::Struct, definition); + EXPECT_TOKEN("s1", SymbolKind::Struct); + EXPECT_TOKEN("v0", SymbolKind::Variable, definition); + EXPECT_TOKEN("v1", SymbolKind::Variable, definition); + EXPECT_TOKEN("v2", SymbolKind::Variable, definition); +} + }; // TEST_SUITE(SemanticTokens) } // namespace