diff --git a/.gitignore b/.gitignore index bdeef599..f54bc82d 100644 --- a/.gitignore +++ b/.gitignore @@ -31,11 +31,11 @@ *.out *.app -build* -notes/ temp/ +*build*/ .cache/ .llvm*/ .xmake/ .vscode/ +unittests/Local/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 10dfaeaa..6c13a974 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,9 +40,9 @@ else() endif() if(CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fno-exceptions -g -O0 -fsanitize=address -Wno-deprecated-declarations") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fuse-ld=lld -fsanitize=address") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fno-exceptions -g -O0 -fsanitize=address -fsanitize=undefined -Wno-deprecated-declarations") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fuse-ld=lld -fsanitize=address -fsanitize=undefined") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld -fsanitize=address -fsanitize=undefined") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fno-exceptions -O3 -Wno-deprecated-declarations") if(WIN32) @@ -71,10 +71,6 @@ file(GLOB_RECURSE CLICE_SOURCES ) add_library(clice-core "${CLICE_BUILD_TYPE}" "${CLICE_SOURCES}") -target_precompile_headers(clice-core PRIVATE - "${CMAKE_SOURCE_DIR}/include/Compiler/Clang.h" -) - # set llvm include and lib path target_include_directories(clice-core PUBLIC "${LLVM_INSTALL_PATH}/include") target_link_directories(clice-core PUBLIC "${LLVM_INSTALL_PATH}/lib") diff --git a/include/Basic/Lifecycle.h b/include/Basic/Lifecycle.h index 268a24ac..53b71461 100644 --- a/include/Basic/Lifecycle.h +++ b/include/Basic/Lifecycle.h @@ -74,7 +74,22 @@ struct ServerCapabilities { /// defining each notification or for backwards compatibility the /// TextDocumentSyncKind number. If omitted it defaults to /// `TextDocumentSyncKind.None`. - TextDocumentSyncKind textDocumentSync = TextDocumentSyncKind::Incremental; + TextDocumentSyncKind textDocumentSync = TextDocumentSyncKind::None; + + /// The server provides go to declaration support. + DeclarationOptions declarationProvider = {}; + + /// The server provides goto definition support. + DefinitionOptions definitionProvider = {}; + + /// The server provides goto type definition support. + TypeDefinitionOptions typeDefinitionProvider = {}; + + /// The server provides goto implementation support. + ImplementationOptions implementationProvider = {}; + + /// The server provides find references support. + ReferenceOptions referencesProvider = {}; /// The server provides semantic tokens support. SemanticTokensOptions semanticTokensProvider; diff --git a/include/Basic/Location.h b/include/Basic/Location.h index 0fa5d8d5..914345e6 100644 --- a/include/Basic/Location.h +++ b/include/Basic/Location.h @@ -51,4 +51,3 @@ struct TextEdit { } // namespace clice::proto - diff --git a/include/Basic/SourceCode.h b/include/Basic/SourceCode.h index 03223dc5..70a3fac2 100644 --- a/include/Basic/SourceCode.h +++ b/include/Basic/SourceCode.h @@ -1,9 +1,19 @@ #pragma once +#include "llvm/ADT/FunctionExtras.h" +#include "clang/Lex/Token.h" #include "clang/Basic/SourceLocation.h" namespace clice { +struct LocalSourceRange { + /// The begin position offset to the source file. + uint32_t begin; + + /// The end position offset to the source file. + uint32_t end; +}; + /// Get the content of the file with the given file ID. llvm::StringRef getFileContent(const clang::SourceManager& SM, clang::FileID fid); @@ -16,4 +26,13 @@ std::uint32_t getTokenLength(const clang::SourceManager& SM, clang::SourceLocati /// Get the spelling of the token at the given location. llvm::StringRef getTokenSpelling(const clang::SourceManager& SM, clang::SourceLocation location); +/// @brief Run `clang::Lexer` in raw mode and tokenize the content. +/// @param content The content to tokenize. +/// @param callback The callback to call for each token. Return false to break. +/// @param langOpts The language options to use. If not provided, lastest C++ standard is used. +void tokenize(llvm::StringRef content, + llvm::unique_function callback, + bool ignoreComments = true, + const clang::LangOptions* langOpts = nullptr); + } // namespace clice diff --git a/include/Basic/SymbolKind.h b/include/Basic/SymbolKind.h index e6b0d65d..097c9e8e 100644 --- a/include/Basic/SymbolKind.h +++ b/include/Basic/SymbolKind.h @@ -57,7 +57,15 @@ struct SymbolKind : refl::Enum { static SymbolKind from(const clang::tok::TokenKind kind); }; -struct SymbolModifiers : refl::Enum {}; +struct SymbolModifiers : refl::Enum { + enum Kind { + Declaration = 0, + Definition, + Reference, + }; + + using Enum::Enum; +}; } // namespace clice diff --git a/include/Compiler/AST.h b/include/Compiler/AST.h index f1cb818d..e98e663f 100644 --- a/include/Compiler/AST.h +++ b/include/Compiler/AST.h @@ -65,11 +65,19 @@ public: return instance->getASTContext().getTranslationUnitDecl(); } + /// The interested file ID. For file without header context, it is the main file ID. + /// For file with header context, it is the file ID of header file. + clang::FileID getInterestedFile() { + return interested; + } + + /// All files involved in building the AST. + const llvm::DenseSet& files(); + std::vector deps(); private: - /// The interested file ID. For file without header context, it is the main file ID. - /// For file with header context, it is the file ID of header file. + /// The interested file ID. clang::FileID interested; /// The frontend action used to build the AST. @@ -87,6 +95,8 @@ private: /// All diretive information collected during the preprocessing. llvm::DenseMap m_directives; + + llvm::DenseSet allFiles; }; } // namespace clice diff --git a/include/Compiler/Semantic.h b/include/Compiler/Semantic.h index 5072ac13..04fa6266 100644 --- a/include/Compiler/Semantic.h +++ b/include/Compiler/Semantic.h @@ -61,7 +61,7 @@ public: /// its own implementation to avoid infinite recursion. if constexpr(!std::same_as) { - getDerived().handleDeclOccurrence(decl, location, kind); + getDerived().handleDeclOccurrence(decl, kind, location); } } @@ -75,7 +75,7 @@ public: if constexpr(!std::same_as) { - getDerived().handleMacroOccurrence(def, location, kind); + getDerived().handleMacroOccurrence(def, kind, location); } } @@ -283,7 +283,7 @@ public: handleRelation(decl, RelationKind::Definition, decl, decl->getLocation()); handleRelation(decl, RelationKind::TypeDefinition, - declForType(decl->getType()), + llvm::cast(decl->getDeclContext()), decl->getLocation()); return true; } @@ -605,10 +605,10 @@ public: /// ^~~~ reference VISIT_TYPELOC(DependentNameTypeLoc) { auto location = loc.getNameLoc(); - for(auto decl: resolver.lookup(loc.getTypePtr())) { - handleDeclOccurrence(decl, RelationKind::WeakReference, location); - handleRelation(decl, RelationKind::WeakReference, decl, location); - } + // for(auto decl: resolver.lookup(loc.getTypePtr())) { + // handleDeclOccurrence(decl, RelationKind::WeakReference, location); + // handleRelation(decl, RelationKind::WeakReference, decl, location); + // } return true; } @@ -616,10 +616,10 @@ public: /// ^~~~ reference VISIT_TYPELOC(DependentTemplateSpecializationTypeLoc) { auto location = loc.getTemplateNameLoc(); - for(auto decl: resolver.lookup(loc.getTypePtr())) { - handleDeclOccurrence(decl, RelationKind::WeakReference, location); - handleRelation(decl, RelationKind::WeakReference, decl, location); - } + // for(auto decl: resolver.lookup(loc.getTypePtr())) { + // handleDeclOccurrence(decl, RelationKind::WeakReference, location); + // handleRelation(decl, RelationKind::WeakReference, decl, location); + // } return true; } diff --git a/include/Feature/Lookup.h b/include/Feature/Lookup.h index bb1dbc81..5058ea32 100644 --- a/include/Feature/Lookup.h +++ b/include/Feature/Lookup.h @@ -2,22 +2,37 @@ namespace clice::proto { +struct WorkDoneProgressOptions { + /// Report on work done progress. + bool workDoneProgress = false; +}; + +using DeclarationOptions = WorkDoneProgressOptions; + using DeclarationParams = TextDocumentPositionParams; using DeclarationResult = std::vector; +using DefinitionOptions = DeclarationParams; + using DefinitionParams = TextDocumentPositionParams; using DefinitionResult = std::vector; +using TypeDefinitionOptions = WorkDoneProgressOptions; + using TypeDefinitionParams = TextDocumentPositionParams; using TypeDefinitionResult = std::vector; +using ImplementationOptions = WorkDoneProgressOptions; + using ImplementationParams = TextDocumentPositionParams; using ImplementationResult = std::vector; +using ReferenceOptions = WorkDoneProgressOptions; + using ReferenceParams = TextDocumentPositionParams; using ReferenceResult = std::vector; diff --git a/include/Feature/SemanticTokens.h b/include/Feature/SemanticTokens.h index 5bfbb988..59851487 100644 --- a/include/Feature/SemanticTokens.h +++ b/include/Feature/SemanticTokens.h @@ -2,7 +2,9 @@ #include "Basic/Document.h" #include "Basic/SymbolKind.h" -#include "Index/FeatureIndex.h" +#include "Basic/SourceCode.h" +#include "Basic/SourceConverter.h" +#include "Index/Shared.h" namespace clice { @@ -50,22 +52,24 @@ struct SemanticTokens { namespace feature { struct SemanticToken { - uint32_t line; - uint32_t column; - uint32_t length; + LocalSourceRange range; SymbolKind kind; SymbolModifiers modifiers; }; /// Generate semantic tokens for all files. -index::SharedIndex> semanticTokens(ASTInfo& info); +index::Shared> semanticTokens(ASTInfo& info); /// Translate semantic tokens to LSP format. proto::SemanticTokens toSemanticTokens(llvm::ArrayRef tokens, + SourceConverter& SC, + llvm::StringRef content, const config::SemanticTokensOption& option); /// Generate semantic tokens for main file and translate to LSP format. -proto::SemanticTokens semanticTokens(ASTInfo& info, const config::SemanticTokensOption& option); +proto::SemanticTokens semanticTokens(ASTInfo& info, + SourceConverter& SC, + const config::SemanticTokensOption& option); } // namespace feature diff --git a/include/Index/Shared.h b/include/Index/Shared.h new file mode 100644 index 00000000..6d4a58cc --- /dev/null +++ b/include/Index/Shared.h @@ -0,0 +1,11 @@ +#pragma once + +#include "llvm/ADT/DenseMap.h" +#include "clang/Basic/SourceLocation.h" + +namespace clice::index { + +template +using Shared = llvm::DenseMap; + +} diff --git a/include/Index/SymbolIndex.h b/include/Index/SymbolIndex.h index eb306b3a..b49bf733 100644 --- a/include/Index/SymbolIndex.h +++ b/include/Index/SymbolIndex.h @@ -1,32 +1,34 @@ #pragma once +#include "Shared.h" #include "ArrayView.h" -#include "Basic/RelationKind.h" +#include "Basic/SourceCode.h" #include "Basic/SymbolKind.h" -#include "Compiler/Compilation.h" +#include "Basic/RelationKind.h" #include "Support/JSON.h" +namespace clice { + +class ASTInfo; + +} + namespace clice::index { -struct LocalSourceRange { - /// The begin position offset to the source file. - uint32_t begin; - - /// The end position offset to the source file. - uint32_t end; -}; - class SymbolIndex { public: - SymbolIndex(void* base, std::size_t size) : base(base), size(size) {} + SymbolIndex(void* base, std::size_t size, bool own = true) : base(base), size(size), own(own) {} - SymbolIndex(SymbolIndex&& other) : base(other.base), size(other.size) { + SymbolIndex(SymbolIndex&& other) noexcept : base(other.base), size(other.size), own(other.own) { other.base = nullptr; other.size = 0; + other.own = false; } ~SymbolIndex() { - std::free(base); + if(own) { + std::free(base); + } } struct Symbol; @@ -78,15 +80,16 @@ public: void locateSymbols(uint32_t position, llvm::SmallVectorImpl& symbols) const; /// Locate symbol with the given id(usually from another index). - Symbol locateSymbol(SymbolID ID) const; + std::optional locateSymbol(uint64_t id, llvm::StringRef name) const; json::Value toJSON() const; public: void* base; std::size_t size; + bool own; }; -llvm::DenseMap test(ASTInfo& info); +Shared index(ASTInfo& info); } // namespace clice::index diff --git a/include/Server/Async.h b/include/Server/Async.h index 98198796..092b02e1 100644 --- a/include/Server/Async.h +++ b/include/Server/Async.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "Support/JSON.h" @@ -136,11 +137,11 @@ public: } public: - core_handle handle() const noexcept { + coroutine_handle handle() const noexcept { return core; } - core_handle release() noexcept { + coroutine_handle release() noexcept { auto handle = core; core = nullptr; return handle; @@ -197,30 +198,29 @@ auto suspend(Callback&& callback) { return suspend_awaiter{std::forward(callback)}; } +struct none {}; + +template ::value_type> +using task_value_t = std::conditional_t, none, V>; + template -auto gather(Tasks&&... tasks) - -> Task::value_type...>> { - bool all_done = (tasks.done() && ...); +auto gather [[gnu::noinline]] (Tasks&&... tasks) -> Task...>> { + /// FIXME: If remove noinline, the program crashes. Figure out in the future. + (async::schedule(tasks.handle()), ...); - if(!all_done) { - llvm::SmallVector handles = {tasks.handle()...}; - bool started = false; - - if(!started) { - for(auto handle: handles) { - async::schedule(handle); - } - started = true; - } - - while(!all_done) { - co_await async::suspend([](core_handle handle) { async::schedule(handle); }); - all_done = (tasks.done() && ...); - } + while(!(tasks.done() && ...)) { + co_await async::suspend([](core_handle handle) { async::schedule(handle); }); } /// If all tasks are done, return the results. - co_return std::tuple{tasks.await_resume()...}; + auto getResult = [](Task& task) { + if constexpr(std::is_void_v) { + return none{}; + } else { + return task.await_resume(); + } + }; + co_return std::tuple{getResult(tasks)...}; } /// Run the tasks in parallel and return the results. @@ -385,8 +385,7 @@ inline auto wait_for(std::chrono::milliseconds duration) { } struct Stats { - using time_point = std::chrono::time_point; - time_point mtime; + std::chrono::milliseconds mtime; }; namespace awaiter { @@ -408,8 +407,8 @@ struct stat { auto callback = [](uv_fs_t* fs) { auto transform = [](uv_timespec_t& mtime) { using namespace std::chrono; - return system_clock::time_point(duration_cast( - seconds(mtime.tv_sec) + nanoseconds(mtime.tv_nsec))); + return milliseconds(duration_cast(seconds(mtime.tv_sec) + + nanoseconds(mtime.tv_nsec))); }; auto& awaiter = *static_cast(fs->data); @@ -418,6 +417,10 @@ struct stat { awaiter.stats.mtime = transform(fs->statbuf.st_mtim); async::schedule(awaiter.waiting); + + uv_fs_t close_req; + uv_fs_close(fs->loop, &close_req, fs->result, nullptr); + uv_fs_req_cleanup(fs); }; uv_fs_stat(uv_default_loop(), &fs, path.c_str(), callback); @@ -428,6 +431,73 @@ struct stat { } }; +struct write { + uv_fs_t fs; + std::string path; + char* data; + size_t size; + core_handle waiting; + + bool await_ready() const noexcept { + return false; + } + + void await_suspend(core_handle waiting) noexcept { + fs.data = this; + this->waiting = waiting; + + auto callback = [](uv_fs_t* fs) { + auto& awaiter = *static_cast(fs->data); + async::schedule(awaiter.waiting); + + uv_fs_t close_req; + uv_fs_close(fs->loop, &close_req, fs->result, nullptr); + uv_fs_req_cleanup(fs); + }; + + uv_fs_open(uv_default_loop(), + &fs, + path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC, + 0666, + nullptr); + + uv_buf_t buf[1] = {uv_buf_init(data, size)}; + uv_fs_write(uv_default_loop(), &fs, fs.result, buf, 1, 0, callback); + } + + void await_resume() noexcept {} +}; + +struct read { + uv_fs_t fs; + std::string path; + core_handle waiting; + + bool await_ready() const noexcept { + return false; + } + + void await_suspend(core_handle waiting) noexcept { + fs.data = this; + this->waiting = waiting; + + auto callback = [](uv_fs_t* fs) { + auto& awaiter = *static_cast(fs->data); + async::schedule(awaiter.waiting); + + uv_fs_t close_req; + uv_fs_close(fs->loop, &close_req, fs->result, nullptr); + uv_fs_req_cleanup(fs); + }; + + uv_fs_open(uv_default_loop(), &fs, path.c_str(), O_RDONLY, 0, nullptr); + + uv_buf_t buf[1] = {}; + uv_fs_read(uv_default_loop(), &fs, fs.result, buf, 1, 0, callback); + } +}; + } // namespace awaiter /// Get the file status asynchronously. @@ -435,6 +505,11 @@ inline auto stat(llvm::StringRef path) { return awaiter::stat{{}, path.str(), {}, {}}; } +/// Write the data to the file asynchronously. +inline auto write(llvm::StringRef path, char* data, size_t size) { + return awaiter::write{{}, path.str(), data, size, {}}; +} + using Callback = llvm::unique_function(json::Value)>; /// Listen on stdin/stdout, callback is called when there is a LSP message available. diff --git a/include/Server/Database.h b/include/Server/Database.h index 4048289d..dfa13485 100644 --- a/include/Server/Database.h +++ b/include/Server/Database.h @@ -11,10 +11,13 @@ namespace clice { class CompilationDatabase { public: /// Update the compile commands with the given file. - void update(llvm::StringRef file); + void updateCommands(llvm::StringRef file); + + /// Update the compile commands with the given file and compile command. + void updateCommand(llvm::StringRef file, llvm::StringRef command); /// Update the module map with the given file and module name. - void update(llvm::StringRef file, llvm::StringRef name); + void updateModule(llvm::StringRef file, llvm::StringRef name); /// Lookup the compile commands of the given file. llvm::StringRef getCommand(llvm::StringRef file); @@ -22,6 +25,18 @@ public: /// Lookup the module interface unit file path of the given module name. llvm::StringRef getModuleFile(llvm::StringRef name); + auto size() const { + return commands.size(); + } + + auto begin() { + return commands.begin(); + } + + auto end() { + return commands.end(); + } + private: /// A map between file path and compile commands. llvm::StringMap commands; diff --git a/include/Server/Indexer.h b/include/Server/Indexer.h new file mode 100644 index 00000000..407c5aa5 --- /dev/null +++ b/include/Server/Indexer.h @@ -0,0 +1,142 @@ +#pragma once + +#include "Async.h" +#include "Config.h" +#include "Database.h" +#include "Protocol.h" +#include "Basic/RelationKind.h" + +namespace clice { + +class ASTInfo; + +class Indexer { +public: + Indexer(const config::IndexOptions& options, CompilationDatabase& database) : + options(options), database(database) {} + + ~Indexer(); + + struct TranslationUnit; + + struct Context { + /// The index file path(not include suffix, e.g. `.sidx` and `.fidx`). + std::string indexPath; + + /// The include chain that introduces this context. + uint32_t include; + }; + + struct IncludeLocation { + /// The location of the include directive. + uint32_t line; + + /// The index of the file that includes this header. + uint32_t include = -1; + + /// The file name of the header. + std::string filename; + }; + + struct Header { + /// The path of the header file. + std::string srcPath; + + /// All header contexts of this header. + llvm::DenseMap> contexts; + }; + + struct TranslationUnit { + /// The source file path. + std::string srcPath; + + /// The index file path(not include suffix, e.g. `.sidx` and `.fidx`). + std::string indexPath; + + /// All headers included by this translation unit. + llvm::DenseSet headers; + + /// The time when this translation unit is indexed. Used to determine + /// whether the index file is outdated. + std::chrono::milliseconds mtime; + + /// All include locations introduced by this translation unit. + /// Note that if a file has guard macro or pragma once, we will + /// record it at most once. + std::vector locations; + }; + + /// Check whether the index file is outdated. + async::Task needUpdate(TranslationUnit* tu); + + /// Try to merge the index file with same content for the given header. + async::Task<> merge(Header* header); + + async::Task<> merge(llvm::StringRef header); + + async::Task<> mergeAll(); + + /// Index the given file(for unopened file). + async::Task<> index(llvm::StringRef file); + + /// Index the given file(for opened file). + async::Task<> index(llvm::StringRef file, ASTInfo& info); + + async::Task<> indexAll(); + + /// Generate the index file path based on time and file name. + std::string getIndexPath(llvm::StringRef file); + + /// Dump the index information to JSON. + json::Value dumpToJSON(); + + /// Dump all index information of the given file for test. + void dumpForTest(llvm::StringRef file); + + void lookupHeaderContexts(llvm::StringRef file); + + /// Save the index information to disk. + void saveToDisk(); + + /// Load the index information from disk. + void loadFromDisk(); + +private: + struct SymbolID { + uint64_t id; + std::string name; + }; + + async::Task> read(llvm::StringRef path); + + async::Task<> lookup(llvm::ArrayRef ids, + RelationKind kind, + llvm::StringRef srcPath, + llvm::StringRef content, + std::string indexPath, + std::vector& result); + +public: + async::Task> + lookup(const proto::TextDocumentPositionParams& params, RelationKind kind); + + async::Task + incomingCalls(const proto::CallHierarchyIncomingCallsParams& params); + + async::Task + outgoingCalls(const proto::CallHierarchyOutgoingCallsParams& params); + + async::Task + supertypes(const proto::TypeHierarchySupertypesParams& params); + + async::Task + subtypes(const proto::TypeHierarchySubtypesParams& params); + +private: + const config::IndexOptions& options; + CompilationDatabase& database; + llvm::StringMap headers; + llvm::StringMap tus; +}; + +} // namespace clice diff --git a/include/Server/Server.h b/include/Server/Server.h index 99215133..e166b22e 100644 --- a/include/Server/Server.h +++ b/include/Server/Server.h @@ -2,6 +2,7 @@ #include "Async.h" #include "Config.h" +#include "Indexer.h" #include "Protocol.h" #include "Database.h" #include "Scheduler.h" @@ -145,6 +146,10 @@ private: /// Extension /// ============================================================================ + async::Task<> onIndexCurrent(const proto::TextDocumentIdentifier& params); + + async::Task<> onIndexAll(const proto::None&); + async::Task<> onContextCurrent(const proto::TextDocumentIdentifier& params); async::Task<> onContextAll(const proto::TextDocumentIdentifier& params); @@ -153,6 +158,7 @@ private: SourceConverter converter; CompilationDatabase database; + Indexer indexer; Scheduler scheduler; }; diff --git a/include/Support/Assert.h b/include/Support/Assert.h new file mode 100644 index 00000000..80d6d236 --- /dev/null +++ b/include/Support/Assert.h @@ -0,0 +1,17 @@ +#pragma once + +#include "Format.h" + +namespace clice { + +#ifndef NDEBUG +#define ASSERT(expr, message, ...) \ + if(!(expr)) { \ + llvm::errs() << "ASSERT FAIL: " << std::format(message, ##__VA_ARGS__); \ + std::abort(); \ + } +#else +#define ASSERT(expr, message, ...) +#endif + +} // namespace clice diff --git a/include/Support/Enum.h b/include/Support/Enum.h index 56d0d43d..80ea2b43 100644 --- a/include/Support/Enum.h +++ b/include/Support/Enum.h @@ -215,6 +215,10 @@ public: return Enum(m_Value & (1 << underlying_value(kind))); } + constexpr Enum operator& (Enum e) const { + return Enum(m_Value & e.value()); + } + template Kind> constexpr Enum& operator|= (Kind kind) { m_Value |= (1 << underlying_value(kind)); diff --git a/include/Support/FileSystem.h b/include/Support/FileSystem.h index da0f9085..9da0fb90 100644 --- a/include/Support/FileSystem.h +++ b/include/Support/FileSystem.h @@ -1,5 +1,8 @@ #pragma once +#include + +#include "Assert.h" #include "llvm/Support/Path.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/VirtualFileSystem.h" @@ -18,6 +21,15 @@ std::string join(Args&&... args) { return path.str().str(); } +/// Get the real path of the given file. The file must exist. If the file does not exist, + +inline std::string real_path(llvm::StringRef file) { + llvm::SmallString<128> path; + auto error = llvm::sys::fs::real_path(file, path); + ASSERT(!error, "Failed to get real path of {0}, because {1}", file, error.message()); + return path.str().str(); +} + } // namespace path namespace fs { diff --git a/include/Support/Format.h b/include/Support/Format.h index c0dd8037..ac42e002 100644 --- a/include/Support/Format.h +++ b/include/Support/Format.h @@ -68,6 +68,21 @@ struct std::formatter : std::formatter { } }; +template <> +struct std::formatter : std::formatter { + using Base = std::formatter; + + template + constexpr auto parse(ParseContext& ctx) { + return Base::parse(ctx); + } + + template + auto format(const std::error_code& e, FormatContext& ctx) const { + return Base::format(e.message(), ctx); + } +}; + template <> struct std::formatter : std::formatter { using Base = std::formatter; diff --git a/include/Test/CTest.h b/include/Test/CTest.h index 8ca50809..9b2beaa5 100644 --- a/include/Test/CTest.h +++ b/include/Test/CTest.h @@ -1,77 +1,12 @@ #pragma once -#include "gtest/gtest.h" +#include "Test.h" #include "Basic/Location.h" #include "Compiler/Compilation.h" #include "Support/Support.h" -namespace clice::test { - -llvm::StringRef source_dir(); - -llvm::StringRef resource_dir(); - -} // namespace clice::test - namespace clice::testing { -#undef EXPECT_EQ -#undef EXPECT_NE - -inline void EXPECT_FAILURE(std::string msg, - std::source_location current = std::source_location::current()) { - ::testing::internal::AssertHelper(::testing ::TestPartResult ::kNonFatalFailure, - current.file_name(), - current.line(), - msg.c_str()) = ::testing ::Message(); -} - -template -inline void EXPECT_EQ(const LHS& lhs, - const RHS& rhs, - std::source_location current = std::source_location::current()) { - if(!refl::equal(lhs, rhs)) { - std::string expect; - if constexpr(requires { sizeof(json::Serde); }) { - llvm::raw_string_ostream(expect) << json::serialize(lhs); - } else { - expect = "cannot dump value"; - } - - std::string actual; - if constexpr(requires { sizeof(json::Serde); }) { - llvm::raw_string_ostream(actual) << json::serialize(rhs); - } else { - actual = "cannot dump value"; - } - - EXPECT_FAILURE(std::format("expect: {}, actual: {}\n", expect, actual), current); - } -} - -template -inline void EXPECT_NE(const LHS& lhs, - const RHS& rhs, - std::source_location current = std::source_location::current()) { - if(refl::equal(lhs, rhs)) { - std::string expect; - if constexpr(requires { sizeof(json::Serde); }) { - llvm::raw_string_ostream(expect) << json::serialize(lhs); - } else { - expect = "cannot dump value"; - } - - std::string actual; - if constexpr(requires { sizeof(json::Serde); }) { - llvm::raw_string_ostream(actual) << json::serialize(rhs); - } else { - actual = "cannot dump value"; - } - - EXPECT_FAILURE(std::format("expect: {}, actual: {}\n", expect, actual), current); - } -} - struct Tester { CompilationParams params; std::optional info; diff --git a/include/Test/IndexTester.h b/include/Test/IndexTester.h index 2e309410..6f971b1a 100644 --- a/include/Test/IndexTester.h +++ b/include/Test/IndexTester.h @@ -11,14 +11,15 @@ struct IndexTester : Tester { IndexTester& run(const char* standard = "-std=c++20") { Tester::run(standard); - indices = index::test(*info); + indices = index::index(*info); return *this; } IndexTester& GotoDefinition(llvm::StringRef cursor, llvm::StringRef target, std::source_location current = std::source_location::current()) { - auto offset = offsets[cursor]; + SourceConverter converter; + auto offset = converter.toOffset(sources[0], pos(cursor)); llvm::SmallVector symbols; indices.begin()->second.locateSymbols(offset, symbols); diff --git a/include/Test/Test.h b/include/Test/Test.h index 27190da9..68620a7e 100644 --- a/include/Test/Test.h +++ b/include/Test/Test.h @@ -7,6 +7,8 @@ namespace clice::testing { +llvm::StringRef test_dir(); + #undef EXPECT_EQ #undef EXPECT_NE diff --git a/scripts/build-llvm-debug.py b/scripts/build-llvm-debug.py new file mode 100644 index 00000000..89b27cea --- /dev/null +++ b/scripts/build-llvm-debug.py @@ -0,0 +1,30 @@ +import os +import subprocess +import shutil + +# CMake build commands +cmake_commands = [ + ["cmake", "-G", "Ninja", "-S", "./llvm", "-B", "build-debug", + "-DLLVM_USE_LINKER=lld", "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", "-DCMAKE_BUILD_TYPE=Debug", + "-DBUILD_SHARED_LIBS=ON", "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_SANITIZER=Address", "-DLLVM_ENABLE_PROJECTS=clang", + "-DCMAKE_INSTALL_PREFIX=./build-debug-install"], + ["cmake", "--build", "build-debug"], + ["cmake", "--install", "build-debug"] +] + +# Execute CMake commands +for command in cmake_commands: + subprocess.run(command, check=True) + +# File copy section +src = "./clang/lib/Sema/" +dst = "./build-debug-install/include/clang/Sema/" +files = ["CoroutineStmtBuilder.h", "TypeLocBuilder.h", "TreeTransform.h"] + +for file in files: + src_file = os.path.join(src, file) + dst_file = os.path.join(dst, file) + print(f"Copying {src_file} to {dst_file}") + shutil.copy(src_file, dst_file) diff --git a/scripts/build-llvm-debug.sh b/scripts/build-llvm-debug.sh new file mode 100755 index 00000000..b9f62a2d --- /dev/null +++ b/scripts/build-llvm-debug.sh @@ -0,0 +1,24 @@ +cmake \ + -G Ninja -S ./llvm \ + -B build-debug \ + -DLLVM_USE_LINKER=lld \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=ON \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_USE_SANITIZER=Address \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_INSTALL_PREFIX=./build-debug-install +cmake --build build-debug +cmake --install build-debug + +src="./clang/lib/Sema/" +dst="./build-debug-install/include/clang/Sema/" + +files=("CoroutineStmtBuilder.h" "TypeLocBuilder.h" "TreeTransform.h") + +for file in "${files[@]}"; do + echo "Copying ${src}${file} to ${dst}${file}" + cp "${src}${file}" "${dst}${file}" +done diff --git a/scripts/build-llvm-runtime.sh b/scripts/build-llvm-runtime.sh new file mode 100644 index 00000000..627759e5 --- /dev/null +++ b/scripts/build-llvm-runtime.sh @@ -0,0 +1,11 @@ +cmake -B build-runtime -S ./runtimes -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libc;libcxx;libcxxabi;libunwind" \ + -DLIBCXX_ENABLE_SHARED=ON \ + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -w" \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_USE_LINKER=lld +cmake --build build-runtime +cmake --build build-runtime --target install diff --git a/src/Basic/SourceCode.cpp b/src/Basic/SourceCode.cpp index 73a23f3d..490e8b90 100644 --- a/src/Basic/SourceCode.cpp +++ b/src/Basic/SourceCode.cpp @@ -1,4 +1,6 @@ #include "Basic/SourceCode.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Lex/Lexer.h" namespace clice { @@ -14,4 +16,33 @@ llvm::StringRef getTokenSpelling(const clang::SourceManager& SM, clang::SourceLo return llvm::StringRef(SM.getCharacterData(location), getTokenLength(SM, location)); } +void tokenize(llvm::StringRef content, + llvm::unique_function callback, + bool ignoreComments, + const clang::LangOptions* langOpts) { + clang::LangOptions defaultLangOpts; + defaultLangOpts.CPlusPlus = 1; + defaultLangOpts.CPlusPlus26 = 1; + defaultLangOpts.LineComment = !ignoreComments; + + clang::Lexer lexer(clang::SourceLocation::getFromRawEncoding(1), + langOpts ? *langOpts : defaultLangOpts, + content.begin(), + content.begin(), + content.end()); + lexer.SetCommentRetentionState(!ignoreComments); + + clang::Token token; + while(true) { + bool end = lexer.LexFromRawLexer(token); + if(!callback(token)) { + break; + } + + if(end || token.is(clang::tok::eof)) { + break; + } + } +} + } // namespace clice diff --git a/src/Basic/SourceConverter.cpp b/src/Basic/SourceConverter.cpp index 664c3eb5..2acf9b9e 100644 --- a/src/Basic/SourceConverter.cpp +++ b/src/Basic/SourceConverter.cpp @@ -4,6 +4,7 @@ #include "Support/Support.h" #include "clang/Basic/SourceManager.h" +#include "Compiler/Clang.h" namespace clice { diff --git a/src/Compiler/AST.cpp b/src/Compiler/AST.cpp index 561839e1..806d8403 100644 --- a/src/Compiler/AST.cpp +++ b/src/Compiler/AST.cpp @@ -2,6 +2,21 @@ namespace clice { +const llvm::DenseSet& ASTInfo::files() { + if(allFiles.empty()) { + /// FIXME: handle preamble and embed file id. + for(auto& [fid, diretive]: directives()) { + for(auto& include: diretive.includes) { + if(include.fid.isValid()) { + allFiles.insert(include.fid); + } + } + } + allFiles.insert(srcMgr().getMainFileID()); + } + return allFiles; +} + std::vector ASTInfo::deps() { llvm::StringSet<> deps; diff --git a/src/Compiler/Preamble.cpp b/src/Compiler/Preamble.cpp index 0850004a..53ba2051 100644 --- a/src/Compiler/Preamble.cpp +++ b/src/Compiler/Preamble.cpp @@ -1,4 +1,5 @@ #include "Compiler/Preamble.h" +#include "clang/Lex/Lexer.h" namespace clice { diff --git a/src/Compiler/Resolver.cpp b/src/Compiler/Resolver.cpp index 4ea99f56..734be2dc 100644 --- a/src/Compiler/Resolver.cpp +++ b/src/Compiler/Resolver.cpp @@ -706,7 +706,10 @@ public: } auto NNS = TransformNestedNameSpecifierLoc(TL.getQualifierLoc()).getNestedNameSpecifier(); - + if(!NNS){ + return clang::QualType(); + } + /// FIXME: figure out here. clang::TemplateArgumentListInfo info; using iterator = clang::TemplateArgumentLocContainerIterator< diff --git a/src/Compiler/Utility.cpp b/src/Compiler/Utility.cpp index 7f2ec31c..8bfaa491 100644 --- a/src/Compiler/Utility.cpp +++ b/src/Compiler/Utility.cpp @@ -140,7 +140,7 @@ const clang::NamedDecl* declForType(clang::QualType type) { if(type.isNull()) { return nullptr; } - + if(auto RT = type->getAs()) { return RT->getDecl(); } diff --git a/src/Driver/unit_tests.cc b/src/Driver/unit_tests.cc index a34012b8..ec0f9d29 100644 --- a/src/Driver/unit_tests.cc +++ b/src/Driver/unit_tests.cc @@ -16,17 +16,13 @@ llvm::cl::opt resource_dir("resource-dir", llvm::cl::desc("Resource } // namespace cl -namespace test { +namespace testing { -llvm::StringRef source_dir() { - return cl::test_dir.getValue(); +llvm::StringRef test_dir() { + return cl::test_dir; } -llvm::StringRef resource_dir() { - return cl::resource_dir.c_str(); -} - -} // namespace test +} // namespace testing } // namespace clice diff --git a/src/Feature/SemanticTokens.cpp b/src/Feature/SemanticTokens.cpp index c23b1653..f46873ff 100644 --- a/src/Feature/SemanticTokens.cpp +++ b/src/Feature/SemanticTokens.cpp @@ -1,6 +1,6 @@ -// #include "Compiler/ParsedAST.h" -#include "Feature/SemanticTokens.h" +#include "Index/Shared.h" #include "Compiler/Semantic.h" +#include "Feature/SemanticTokens.h" namespace clice::feature { @@ -8,26 +8,72 @@ namespace { class HighlightBuilder : public SemanticVisitor { public: - HighlightBuilder(ASTInfo& compiler) : SemanticVisitor(compiler, true) {} + HighlightBuilder(ASTInfo& info, bool emitForIndex) : + emitForIndex(emitForIndex), SemanticVisitor(info, true) {} - void run() { - TraverseAST(sema.getASTContext()); + void addToken(clang::FileID fid, const clang::Token& token, SymbolKind kind) { + auto fake = clang::SourceLocation::getFromRawEncoding(1); + LocalSourceRange range = { + token.getLocation().getRawEncoding() - fake.getRawEncoding(), + token.getEndLoc().getRawEncoding() - fake.getRawEncoding(), + }; + + auto& tokens = emitForIndex ? sharedResult[fid] : result; + tokens.emplace_back(SemanticToken{ + .range = range, + .kind = kind, + .modifiers = {}, + }); } - proto::SemanticTokens build() { - /// Collect semantic from spelled tokens. - auto mainFileID = srcMgr.getMainFileID(); - auto spelledTokens = tokBuf.spelledTokens(mainFileID); - for(auto& token: spelledTokens) { - SymbolKind type = SymbolKind::Invalid; - // llvm::outs() << clang::tok::getTokenName(token.kind()) << " " - // << pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts()) - // << "\n"; + void addToken(clang::SourceLocation location, SymbolKind kind, SymbolModifiers modifiers) { + auto& SM = srcMgr; + /// Always use spelling location. + auto spelling = SM.getSpellingLoc(location); + auto [fid, offset] = SM.getDecomposedLoc(spelling); + + /// If the spelling location is not in the interested file and not for index, skip it. + if(fid != SM.getMainFileID() && !emitForIndex) { + return; + } + + auto& tokens = emitForIndex ? sharedResult[fid] : result; + auto length = getTokenLength(SM, spelling); + tokens.emplace_back(SemanticToken{ + .range = {offset, offset + length}, + .kind = kind, + .modifiers = modifiers, + }); + } + + /// Render semantic tokens from lexer. Note that we only render literal, + /// directive, keyword, and comment tokens. + void highlightFromLexer(clang::FileID fid) { + auto& SM = srcMgr; + auto content = getFileContent(SM, fid); + auto& langOpts = pp.getLangOpts(); + + /// Whether the token is after `#`. + bool isAfterHash = false; + /// Whether the token is in the header name. + bool isInHeader = false; + /// Whether the token is in the directive line. + bool isInDirectiveLine = false; + + /// Use to distinguish whether the token is in a keyword. + clang::IdentifierTable identifierTable(pp.getLangOpts()); + + auto callback = [&](const clang::Token& token) -> bool { + SymbolKind kind = SymbolKind::Invalid; + + switch(token.getKind()) { + case clang::tok::comment: { + kind = SymbolKind::Comment; + break; + } - auto kind = token.kind(); - switch(kind) { case clang::tok::numeric_constant: { - type = SymbolKind::Number; + kind = SymbolKind::Number; break; } @@ -36,90 +82,162 @@ public: case clang::tok::utf8_char_constant: case clang::tok::utf16_char_constant: case clang::tok::utf32_char_constant: { - type = SymbolKind::Character; + kind = SymbolKind::Character; + break; + } + + case clang::tok::string_literal: { + if(isInHeader) { + isInHeader = false; + kind = SymbolKind::Header; + } else { + kind = SymbolKind::String; + } break; } - case clang::tok::string_literal: case clang::tok::wide_string_literal: case clang::tok::utf8_string_literal: case clang::tok::utf16_string_literal: case clang::tok::utf32_string_literal: { - type = SymbolKind::String; + kind = SymbolKind::String; break; } - case clang::tok::ampamp: /// and - case clang::tok::ampequal: /// and_eq - case clang::tok::amp: /// bitand - case clang::tok::pipe: /// bitor - case clang::tok::tilde: /// compl - case clang::tok::exclaim: /// not - case clang::tok::exclaimequal: /// not_eq - case clang::tok::pipepipe: /// or - case clang::tok::pipeequal: /// or_eq - case clang::tok::caret: /// xor - case clang::tok::caretequal: { /// xor_eq - /// Clang will lex above keywords as corresponding operators. But we want to - /// highlight them as keywords. So check whether their text is same as the - /// operator spelling. If not, it indicates that they are keywords. - if(token.text(srcMgr) != clang::tok::getPunctuatorSpelling(kind)) { - type = SymbolKind::Operator; + case clang::tok::hash: { + if(token.isAtStartOfLine()) { + isAfterHash = true; + isInDirectiveLine = true; + kind = SymbolKind::Directive; + } + break; + } + + case clang::tok::less: { + if(isInHeader) { + kind = SymbolKind::Header; + } + break; + } + + case clang::tok::greater: { + if(isInHeader) { + isInHeader = false; + kind = SymbolKind::Header; + } + break; + } + + case clang::tok::raw_identifier: { + auto spelling = token.getRawIdentifier(); + if(isAfterHash) { + isAfterHash = false; + isInHeader = (spelling == "include"); + kind = SymbolKind::Directive; + } else if(isInHeader) { + kind = SymbolKind::Header; + } else if(isInDirectiveLine) { + if(spelling == "defined") { + kind = SymbolKind::Directive; + } + } else { + /// Check whether the identifier is a keyword. + if(auto& II = identifierTable.get(spelling); II.isKeyword(langOpts)) { + kind = SymbolKind::Keyword; + } } } default: { - if(pp.getIdentifierInfo(token.text(srcMgr))->isKeyword(pp.getLangOpts())) { - type = SymbolKind::Keyword; - break; - } + break; } } - if(type != SymbolKind::Invalid) {} - } + /// Clear the directive line flag. + if(token.isAtStartOfLine() && token.isNot(clang::tok::hash)) { + isInDirectiveLine = false; + } - /// Collect semantic tokens from AST. - TraverseAST(sema.getASTContext()); + if(kind != SymbolKind::Invalid) { + addToken(fid, token, kind); + } - proto::SemanticTokens result; - std::ranges::sort(tokens, [](const SemanticToken& lhs, const SemanticToken& rhs) { - return lhs.line < rhs.line || (lhs.line == rhs.line && lhs.column < rhs.column); - }); + return true; + }; - std::size_t lastLine = 0; - std::size_t lastColumn = 0; - - for(auto& token: tokens) { - result.data.push_back(token.line - lastLine); - result.data.push_back(token.line == lastLine ? token.column - lastColumn - : token.column); - result.data.push_back(token.length); - // result.data.push_back(llvm::to_underlying(token.column)); - // result.data.push_back(0); - - lastLine = token.line; - lastColumn = token.column; - } - - return result; + tokenize(content, callback, false, &langOpts); } - std::vector tokens; + void handleDeclOccurrence(const clang::NamedDecl* decl, + RelationKind kind, + clang::SourceLocation location) { + /// FIXME: Add modifiers. + addToken(location, SymbolKind::from(decl), {}); + } + + void handleMacroOccurrence(const clang::MacroInfo* def, + RelationKind kind, + clang::SourceLocation location) { + /// FIXME: Add modifiers. + addToken(location, SymbolKind::Macro, {}); + } + + /// FIXME: handle module name. + + void merge(std::vector& tokens) { + ranges::sort(tokens, refl::less, [](const auto& token) { return token.range; }); + /// FIXME: Resolve the overlapped tokens. + } + + auto buildForFile() { + highlightFromLexer(info.getInterestedFile()); + run(); + merge(result); + return std::move(result); + } + + auto buildForIndex() { + for(auto fid: info.files()) { + highlightFromLexer(fid); + } + + run(); + + for(auto& [fid, tokens]: sharedResult) { + merge(tokens); + } + + return std::move(sharedResult); + } + +private: + std::vector result; + index::Shared> sharedResult; + bool emitForIndex; }; } // namespace -index::SharedIndex> semanticTokens(ASTInfo& info) { - return index::SharedIndex>{}; +index::Shared> semanticTokens(ASTInfo& info) { + return HighlightBuilder(info, true).buildForIndex(); } proto::SemanticTokens toSemanticTokens(llvm::ArrayRef tokens, + SourceConverter& SC, + llvm::StringRef content, const config::SemanticTokensOption& option) { + + std::size_t lastLine = 0; + std::size_t lastColumn = 0; + + for(auto& token: tokens) {} + return {}; } -proto::SemanticTokens semanticTokens(ASTInfo& info, const config::SemanticTokensOption& option) { +proto::SemanticTokens semanticTokens(ASTInfo& info, + SourceConverter& SC, + const config::SemanticTokensOption& option) { return {}; } diff --git a/src/Index/Deserialization.cpp b/src/Index/Deserialization.cpp index 6d54ec64..d4e862c8 100644 --- a/src/Index/Deserialization.cpp +++ b/src/Index/Deserialization.cpp @@ -145,16 +145,17 @@ void SymbolIndex::locateSymbols(uint32_t position, } } -/// Locate symbol with the given id(usually from another index). -SymbolIndex::Symbol SymbolIndex::locateSymbol(SymbolIndex::SymbolID ID) const { +std::optional SymbolIndex::locateSymbol(uint64_t id, + llvm::StringRef name) const { auto index = static_cast(base); auto symbols = index->getSymbols(); - auto iter = std::ranges::lower_bound(symbols, ID.id(), {}, [&](const auto& symbol) { - return symbol.id; - }); + auto range = + std::ranges::equal_range(symbols, id, {}, [&](const auto& symbol) { return symbol.id; }); - if(iter != symbols.end() && iter->id == ID.id() && index->getString(iter->name) == ID.name()) { - return {base, &*iter}; + for(auto& symbol: range) { + if(index->getString(symbol.name) == name) { + return SymbolIndex::Symbol{base, &symbol}; + } } return {}; diff --git a/src/Index/SymbolIndex.cpp b/src/Index/SymbolIndex.cpp index dabeae9c..febae476 100644 --- a/src/Index/SymbolIndex.cpp +++ b/src/Index/SymbolIndex.cpp @@ -110,12 +110,16 @@ public: std::ranges::sort(file.occurrences, refl::less, [&](const auto& occurrence) { return file.ranges[occurrence.location]; }); + + /// Remove duplicate occurrences. + auto range = ranges::unique(file.occurrences, refl::equal); + file.occurrences.erase(range.begin(), range.end()); } public: void handleDeclOccurrence(const clang::NamedDecl* decl, - clang::SourceLocation location, - RelationKind kind) { + RelationKind kind, + clang::SourceLocation location) { decl = normalize(decl); /// We always use spelling location for occurrence. @@ -130,13 +134,12 @@ public: } void handleMacroOccurrence(const clang::MacroInfo* def, - clang::SourceLocation location, - RelationKind kind) { - auto spelling = srcMgr.getSpellingLoc(location); - clang::FileID id = srcMgr.getFileID(spelling); - assert(id.isValid() && "Invalid file id"); - auto& file = files[id]; - + RelationKind kind, + clang::SourceLocation location) { + /// auto spelling = srcMgr.getSpellingLoc(location); + /// clang::FileID id = srcMgr.getFileID(spelling); + /// assert(id.isValid() && "Invalid file id"); + /// auto& file = files[id]; /// TODO: } @@ -151,6 +154,11 @@ public: auto [begin, end] = range; auto expansion = srcMgr.getExpansionLoc(begin); + if(srcMgr.isWrittenInBuiltinFile(expansion) || + srcMgr.isWrittenInCommandLineFile(expansion)) { + return; + } + assert(expansion.isValid() && expansion.isFileID() && "Invalid expansion location"); clang::FileID id = srcMgr.getFileID(expansion); assert(id.isValid() && "Invalid file id"); @@ -207,7 +215,7 @@ private: } // namespace -llvm::DenseMap test(ASTInfo& info) { +Shared index(ASTInfo& info) { SymbolIndexBuilder collector(info); return collector.build(); } diff --git a/src/Server/Async.cpp b/src/Server/Async.cpp index ad7f8dc6..c15a07c0 100644 --- a/src/Server/Async.cpp +++ b/src/Server/Async.cpp @@ -38,14 +38,20 @@ static void event_loop(uv_idle_t* handle) { } void schedule(std::coroutine_handle<> core) { - assert(core && !core.done() && "schedule: invalid coroutine handle"); - tasks.emplace_back(core); + uv_async_t* async = new uv_async_t; + async->data = core.address(); + uv_async_init(loop, async, [](uv_async_t* handle) { + auto core = std::coroutine_handle<>::from_address(handle->data); + core.resume(); + uv_close((uv_handle_t*)handle, [](uv_handle_t* handle) { delete (uv_async_t*)handle; }); + }); + uv_async_send(async); } void run() { - uv_idle_t idle; - uv_idle_init(loop, &idle); - uv_idle_start(&idle, event_loop); + // uv_idle_t idle; + // uv_idle_init(loop, &idle); + // uv_idle_start(&idle, event_loop); uv_run(loop, UV_RUN_DEFAULT); } diff --git a/src/Server/Database.cpp b/src/Server/Database.cpp index 1e88978e..23bdaba2 100644 --- a/src/Server/Database.cpp +++ b/src/Server/Database.cpp @@ -6,7 +6,10 @@ namespace clice { /// Update the compile commands with the given file. -void CompilationDatabase::update(llvm::StringRef filename) { +void CompilationDatabase::updateCommands(llvm::StringRef filename) { + auto path = path::real_path(filename); + filename = path; + /// Read the compile commands from the file. json::Value json = nullptr; @@ -78,28 +81,27 @@ void CompilationDatabase::update(llvm::StringRef filename) { commands.size()); /// Scan all files to build module map. - CompilationParams params; - for(auto& [path, command]: commands) { - params.srcPath = path; - params.command = command; - auto info = scanModule(params); - if(!info) { - log::warn("Failed to scan module from {0}, because {1}", path, info.takeError()); - continue; - } - - if(info->isInterfaceUnit) { - assert(!info->name.empty() && "module name is empty"); - moduleMap[info->name] = path; - } - } + // CompilationParams params; + // for(auto& [path, command]: commands) { + // params.srcPath = path; + // params.command = command; + // + // auto name = scanModuleName(params); + // if(!name.empty()) { + // moduleMap[name] = path; + // } + //} log::info("Successfully built module map, total {0} modules", moduleMap.size()); } +void CompilationDatabase::updateCommand(llvm::StringRef file, llvm::StringRef command) { + commands[path::real_path(file)] = command; +} + /// Update the module map with the given file and module name. -void CompilationDatabase::update(llvm::StringRef file, llvm::StringRef name) { - moduleMap[name] = file; +void CompilationDatabase::updateModule(llvm::StringRef file, llvm::StringRef name) { + moduleMap[path::real_path(file)] = file; } /// Lookup the compile commands of the given file. diff --git a/src/Server/Extension.cpp b/src/Server/Extension.cpp index 5008fcaa..308f32bf 100644 --- a/src/Server/Extension.cpp +++ b/src/Server/Extension.cpp @@ -2,6 +2,16 @@ namespace clice { +async::Task<> Server::onIndexCurrent(const proto::TextDocumentIdentifier& params) { + auto path = SourceConverter::toPath(params.uri); + co_await indexer.index(path); +} + +async::Task<> Server::onIndexAll(const proto::None& params) { + co_await indexer.indexAll(); + co_return; +} + async::Task<> Server::onContextCurrent(const proto::TextDocumentIdentifier& params) { co_return; } diff --git a/src/Server/Feature.cpp b/src/Server/Feature.cpp index 058cb686..66267e0f 100644 --- a/src/Server/Feature.cpp +++ b/src/Server/Feature.cpp @@ -4,25 +4,36 @@ namespace clice { async::Task<> Server::onGotoDeclaration(json::Value id, const proto::DeclarationParams& params) { - co_return; + proto::DeclarationResult result = + co_await indexer.lookup(params, + RelationKind(RelationKind::Declaration, RelationKind::Definition)); + co_await response(std::move(id), json::serialize(result)); } async::Task<> Server::onGotoDefinition(json::Value id, const proto::DefinitionParams& params) { - co_return; + proto::DefinitionResult result = co_await indexer.lookup(params, RelationKind::Definition); + co_await response(std::move(id), json::serialize(result)); } async::Task<> Server::onGotoTypeDefinition(json::Value id, const proto::TypeDefinitionParams& params) { - co_return; + proto::TypeDefinitionResult result = + co_await indexer.lookup(params, RelationKind::TypeDefinition); + co_await response(std::move(id), json::serialize(result)); } async::Task<> Server::onGotoImplementation(json::Value id, const proto::ImplementationParams& params) { - co_return; + proto::ImplementationResult result = + co_await indexer.lookup(params, RelationKind::Implementation); + co_await response(std::move(id), json::serialize(result)); } async::Task<> Server::onFindReferences(json::Value id, const proto::ReferenceParams& params) { - co_return; + proto::ReferenceResult result = co_await indexer.lookup( + params, + RelationKind(RelationKind::Declaration, RelationKind::Definition, RelationKind::Reference)); + co_await response(std::move(id), json::serialize(result)); } async::Task<> Server::onPrepareCallHierarchy(json::Value id, diff --git a/src/Server/Indexer.cpp b/src/Server/Indexer.cpp new file mode 100644 index 00000000..a9e8d97c --- /dev/null +++ b/src/Server/Indexer.cpp @@ -0,0 +1,584 @@ +#include + +#include "Support/Assert.h" +#include "Compiler/Compilation.h" +#include "Index/SymbolIndex.h" +#include "Server/Logger.h" +#include "Server/Indexer.h" + +namespace clice { + +static uint32_t addIncludeChain(std::vector& locations, + llvm::DenseMap& files, + clang::SourceManager& SM, + clang::FileID fid) { + auto [iter, success] = files.try_emplace(fid, locations.size()); + if(!success) { + return iter->second; + } + + auto index = iter->second; + locations.emplace_back(); + auto entry = SM.getFileEntryRefForID(fid); + assert(entry && "Invalid file entry"); + locations[index].filename = path::real_path(path::real_path(entry->getName())); + + if(auto presumed = SM.getPresumedLoc(SM.getIncludeLoc(fid), false); presumed.isValid()) { + locations[index].line = presumed.getLine(); + auto include = addIncludeChain(locations, files, SM, presumed.getFileID()); + locations[index].include = include; + } + + return index; +} + +Indexer::~Indexer() { + for(auto& [_, header]: headers) { + delete header; + } + + for(auto& [_, tu]: tus) { + delete tu; + } +} + +async::Task Indexer::needUpdate(TranslationUnit* tu) { + auto stats = co_await async::stat(tu->srcPath); + /// If the file is modified, we need to update the index. + if(stats.mtime > tu->mtime) { + co_return true; + } + + /// Check all headers. + for(auto& header: tu->headers) { + auto stats = co_await async::stat(header->srcPath); + if(stats.mtime > tu->mtime) { + co_return true; + } + } + + co_return false; +} + +async::Task<> Indexer::merge(Header* header) { + co_await async::submit([header] { + llvm::StringMap> indices; + + for(auto& [tu, contexts]: header->contexts) { + for(auto& context: contexts) { + if(context.indexPath.empty() || indices.contains(context.indexPath)) { + continue; + } + + auto file = llvm::MemoryBuffer::getFile(context.indexPath + ".sidx"); + if(!file) { + log::warn("Failed to open index file: {}", context.indexPath); + continue; + } + + bool merged = false; + for(auto& [indexPath, value]: indices) { + if(file->get()->getBuffer() == value->getBuffer()) { + auto error = fs::remove(context.indexPath + ".sidx"); + log::info("Merged index file: {} -> {}", context.indexPath, indexPath); + context.indexPath = indexPath; + merged = true; + break; + } + } + + if(!merged) { + indices.try_emplace(context.indexPath, std::move(file.get())); + } + } + } + }); + + co_return; +} + +async::Task<> Indexer::merge(llvm::StringRef header) { + if(auto iter = headers.find(header); iter != headers.end()) { + co_await merge(iter->second); + } +} + +async::Task<> Indexer::mergeAll() { + std::vector> tasks; + for(auto& [_, header]: headers) { + tasks.emplace_back(merge(header)); + } + + for(auto& task: tasks) { + co_await task; + } +} + +async::Task<> Indexer::index(llvm::StringRef file) { + auto real_path = path::real_path(file); + file = real_path; + + TranslationUnit* tu = nullptr; + bool needIndex = false; + + if(auto iter = tus.find(file); iter != tus.end()) { + /// If no need to update, just return. + if(!co_await needUpdate(iter->second)) { + log::info("No need to update index for file: {}", file); + co_return; + } + + /// Otherwise, we need to update the translation unit. + /// Clear all old header contexts associated with this translation unit. + tu = iter->second; + needIndex = true; + for(auto& header: tu->headers) { + header->contexts[tu].clear(); + } + } else { + tu = new TranslationUnit{ + .srcPath = file.str(), + .indexPath = "", + }; + needIndex = true; + tus.try_emplace(file, tu); + } + + if(!needIndex) { + log::info("No need to update index for file: {}", file); + co_return; + } + + auto command = database.getCommand(file); + if(command.empty()) { + log::warn("No command found for file: {}", file); + co_return; + } + + CompilationParams params; + params.command = command; + auto info = co_await async::submit([¶ms] { return compile(params); }); + + if(!info) { + log::warn("Failed to compile {}: {}", file, info.takeError()); + co_return; + } + + /// Update all headers and translation units. + auto& SM = info->srcMgr(); + std::vector locations; + llvm::DenseMap files; + + for(auto& [fid, directive]: info->directives()) { + for(auto& include: directive.includes) { + /// If the include is invalid, it indicates that the file is skipped because of + /// include guard, or `#pragma once`. Such file cannot provide header context. + /// So we just skip it. + if(include.fid.isInvalid()) { + continue; + } + + /// Add all include chains. + addIncludeChain(locations, files, SM, include.fid); + } + } + + /// Update the translation unit. + tu->mtime = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + tu->locations = std::move(locations); + + /// Update the header context. + for(auto& [fid, include]: files) { + if(fid == SM.getMainFileID()) { + continue; + } + + auto entry = SM.getFileEntryRefForID(fid); + assert(entry && "Invalid file entry"); + auto name = path::real_path(entry->getName()); + + Header* header = nullptr; + if(auto iter = headers.find(name); iter != headers.end()) { + header = iter->second; + } else { + header = new Header; + header->srcPath = name; + headers.try_emplace(name, header); + } + + header->contexts[tu].emplace_back(Context{ + .indexPath = "", + .include = include, + }); + } + + auto indices = co_await async::submit([&info] { return index::index(*info); }); + + /// Write binary index to file. + for(auto& [fid, index]: indices) { + if(fid == SM.getMainFileID()) { + if(tu->indexPath.empty()) { + tu->indexPath = getIndexPath(tu->srcPath); + } + co_await async::write(tu->indexPath + ".sidx", + static_cast(index.base), + index.size); + continue; + } + + auto entry = SM.getFileEntryRefForID(fid); + if(!entry) { + continue; + } + assert(entry && "Invalid file entry"); + auto name = path::real_path(entry->getName()); + auto include = files[fid]; + assert(headers.contains(name) && "Header not found"); + for(auto& context: headers[name]->contexts[tu]) { + if(context.include == include) { + if(context.indexPath.empty()) { + context.indexPath = getIndexPath(name); + } + co_await async::write(context.indexPath + ".sidx", + static_cast(index.base), + index.size); + } + } + } + + co_return; +} + +async::Task<> Indexer::index(llvm::StringRef file, ASTInfo& info) { + co_return; +} + +async::Task<> Indexer::indexAll() { + auto total = database.size(); + auto count = 0; + + std::vector> tasks; + + for(auto& [file, command]: database) { + tasks.emplace_back(index(file)); + + if(tasks.size() == 4) { + co_await async::gather(tasks[0], tasks[1], tasks[2], tasks[3]); + tasks.clear(); + count += 4; + log::info("Indexing progress: {0}/{1}", count, total); + } + } + + for(auto& task: tasks) { + co_await task; + } + tasks.clear(); + + co_await mergeAll(); +} + +std::string Indexer::getIndexPath(llvm::StringRef file) { + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1000); + return path::join(options.dir, path::filename(file) + "." + llvm::Twine(ms + dis(gen))); +} + +json::Value Indexer::dumpToJSON() { + json::Array headers; + for(auto& [_, header]: this->headers) { + json::Array contexts; + for(auto& [tu, context]: header->contexts) { + contexts.emplace_back(json::Object{ + {"tu", tu->srcPath }, + {"contexts", json::serialize(context)}, + }); + } + + headers.emplace_back(json::Object{ + {"srcPath", header->srcPath }, + {"contexts", std::move(contexts)}, + }); + } + + json::Array tus; + for(auto& [_, tu]: this->tus) { + tus.emplace_back(json::Object{ + {"srcPath", tu->srcPath }, + {"indexPath", tu->indexPath }, + {"mtime", tu->mtime.count() }, + {"locations", json::serialize(tu->locations)}, + }); + } + + return json::Object{ + {"headers", std::move(headers)}, + {"tus", std::move(tus) }, + }; +} + +void Indexer::dumpForTest(llvm::StringRef file) { + llvm::StringSet<> files; + + if(auto iter = headers.find(file); iter != headers.end()) { + for(auto& [tu, contexts]: iter->second->contexts) { + for(auto& context: contexts) { + if(files.contains(context.indexPath)) { + continue; + } + + if(auto buffer = llvm::MemoryBuffer::getFile(context.indexPath + ".sidx")) { + index::SymbolIndex index(const_cast(buffer.get()->getBufferStart()), + buffer.get()->getBufferSize(), + false); + println("Include {}, Value: {}", context.include, index.toJSON()); + files.insert(context.indexPath); + } + } + } + } +} + +void Indexer::saveToDisk() { + auto path = path::join(options.dir, "index.json"); + std::error_code ec; + llvm::raw_fd_ostream os(path, ec); + if(ec) { + log::warn("Failed to open index file: {} Beacuse {}", path, ec.message()); + return; + } + + os << dumpToJSON(); + + if(os.has_error()) { + log::warn("Failed to write index file: {}", os.error().message()); + } else { + log::info("Successfully saved index to disk"); + } +} + +void Indexer::loadFromDisk() { + auto path = path::join(options.dir, "index.json"); + auto file = llvm::MemoryBuffer::getFile(path); + if(!file) { + log::warn("Failed to open index file: {} Beacuse {}", path, file.getError()); + return; + } + + auto json = json::parse(file.get()->getBuffer()); + ASSERT(json, "Failed to parse index file: {}", path); + + for(auto& value: *json->getAsObject()->getArray("tus")) { + auto object = value.getAsObject(); + auto tu = new TranslationUnit{ + .srcPath = object->getString("srcPath")->str(), + .indexPath = object->getString("indexPath")->str(), + .mtime = std::chrono::milliseconds(*object->getInteger("mtime")), + .locations = json::deserialize>(*object->get("locations")), + }; + tus.try_emplace(tu->srcPath, tu); + } + + /// All headers must be already initialized. + for(auto& value: *json->getAsObject()->getArray("headers")) { + auto object = value.getAsObject(); + llvm::StringRef srcPath = *object->getString("srcPath"); + + Header* header = nullptr; + if(auto iter = headers.find(srcPath); iter != headers.end()) { + header = iter->second; + } else { + header = new Header; + header->srcPath = srcPath; + headers.try_emplace(srcPath, header); + } + + for(auto& value: *object->getArray("contexts")) { + auto object = value.getAsObject(); + auto tu = tus[*object->getString("tu")]; + header->contexts[tu] = + json::deserialize>(*object->get("contexts")); + tu->headers.insert(header); + } + } + + log::info("Successfully loaded index from disk"); + + return; +} + +async::Task> Indexer::read(llvm::StringRef path) { + co_return co_await async::submit([path] { + auto file = llvm::MemoryBuffer::getFile(path); + ASSERT(file, "Failed to open file: {}, because: {}", path, file.getError()); + return std::move(file.get()); + }); +} + +async::Task<> Indexer::lookup(llvm::ArrayRef ids, + RelationKind kind, + llvm::StringRef srcPath, + llvm::StringRef content, + std::string indexPath, + std::vector& result) { + auto indexFile = llvm::MemoryBuffer::getFile(indexPath); + ASSERT(indexFile, + "Failed to open index file: {}, Beacuse: {}", + indexPath, + indexFile.getError()); + index::SymbolIndex index(const_cast(indexFile.get()->getBufferStart()), + indexFile.get()->getBufferSize(), + false); + + for(auto& id: ids) { + if(auto symbol = index.locateSymbol(id.id, id.name)) { + for(auto relation: symbol->relations()) { + if(relation.kind() & kind) { + auto range = relation.range(); + auto begin = SourceConverter().toPosition(content, range->begin); + auto end = SourceConverter().toPosition(content, range->end); + result.emplace_back(proto::Location{ + .uri = SourceConverter::toURI(srcPath), + .range = proto::Range{.start = begin, .end = end}, + }); + } + } + } + } + + co_return; +} + +async::Task> + Indexer::lookup(const proto::TextDocumentPositionParams& params, RelationKind kind) { + auto srcPath = SourceConverter::toPath(params.textDocument.uri); + llvm::StringRef indexPathPrefix = ""; + + if(auto iter = tus.find(srcPath); iter != tus.end()) { + indexPathPrefix = iter->second->indexPath; + } else if(auto iter = headers.find(srcPath); iter != headers.end()) { + /// FIXME: Indexer should use a variable to indicate know + /// which context of the header is active. And use it to + /// determine the index path prefix. Currently, we just + /// use the first context. + for(auto& [_, contexts]: iter->second->contexts) { + for(auto& context: contexts) { + if(!context.indexPath.empty()) { + indexPathPrefix = context.indexPath; + break; + } + } + + if(!indexPathPrefix.empty()) { + break; + } + } + } + + if(indexPathPrefix.empty()) { + /// FIXME: If such index file does not exist, we should + /// wait for the index task to complete. + co_return proto::DefinitionResult{}; + } + + proto::DefinitionResult result; + std::string indexPath = (indexPathPrefix + ".sidx").str(); + + llvm::SmallVector ids; + + /// Lookup Target index first + { + auto srcFile = co_await read(srcPath); + auto content = srcFile->getBuffer(); + auto offset = SourceConverter().toOffset(content, params.position); + + auto indexFile = co_await read(indexPath); + index::SymbolIndex index(const_cast(indexFile.get()->getBufferStart()), + indexFile.get()->getBufferSize(), + false); + llvm::SmallVector symbols; + index.locateSymbols(offset, symbols); + + for(auto& symbol: symbols) { + ids.emplace_back(SymbolID{symbol.id(), symbol.name().str()}); + + for(auto relation: symbol.relations()) { + if(relation.kind() & kind) { + auto range = relation.range(); + auto begin = SourceConverter().toPosition(content, range->begin); + auto end = SourceConverter().toPosition(content, range->end); + result.emplace_back(proto::Location{ + .uri = SourceConverter::toURI(srcPath), + .range = proto::Range{.start = begin, .end = end}, + }); + } + } + } + } + + for(auto& [path, tu]: tus) { + if(path == srcPath || tu->indexPath.empty()) { + continue; + } + + auto srcPath = path.str(); + auto srcFile = co_await read(srcPath); + co_await lookup(ids, kind, srcPath, srcFile->getBuffer(), tu->indexPath + ".sidx", result); + } + + llvm::StringSet<> visited; + for(auto& [path, header]: headers) { + if(path == srcPath) { + continue; + } + + auto srcPath = path.str(); + auto srcFile = co_await read(srcPath); + auto content = srcFile->getBuffer(); + + for(auto& [_, contexts]: header->contexts) { + for(auto& context: contexts) { + if(context.indexPath.empty() || visited.contains(context.indexPath)) { + continue; + } + + visited.insert(context.indexPath); + co_await lookup(ids, kind, srcPath, content, context.indexPath + ".sidx", result); + } + } + } + + co_await async::submit([&] { + ranges::sort(result, refl::less); + auto [first, last] = ranges::unique(result, refl::equal); + result.erase(first, last); + }); + + co_return result; +} + +async::Task + Indexer::incomingCalls(const proto::CallHierarchyIncomingCallsParams& params) { + co_return proto::CallHierarchyIncomingCallsResult{}; +} + +async::Task + Indexer::outgoingCalls(const proto::CallHierarchyOutgoingCallsParams& params) { + co_return proto::CallHierarchyOutgoingCallsResult{}; +} + +async::Task + Indexer::supertypes(const proto::TypeHierarchySupertypesParams& params) { + co_return proto::TypeHierarchySupertypesResult{}; +} + +async::Task + Indexer::subtypes(const proto::TypeHierarchySubtypesParams& params) { + co_return proto::TypeHierarchySubtypesResult{}; +} + +} // namespace clice diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index 085e4b19..de0a3f78 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -37,7 +37,7 @@ async::Task<> Server::onInitialize(json::Value id, const proto::InitializeParams for(auto& dir: config::server.compile_commands_dirs) { llvm::SmallString<128> path = {dir}; path::append(path, "compile_commands.json"); - database.update(path); + database.updateCommands(path); } } diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index 12050a66..d025c08c 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -3,7 +3,7 @@ namespace clice { -Server::Server() : scheduler(database, {}) { +Server::Server() : indexer(config::index, database), scheduler(database, {}) { addMethod("initialize", &Server::onInitialize); addMethod("initialized", &Server::onInitialized); addMethod("shutdown", &Server::onShutdown); @@ -41,6 +41,8 @@ Server::Server() : scheduler(database, {}) { addMethod("workspace/didChangeWatchedFiles", &Server::onDidChangeWatchedFiles); + addMethod("index/current", &Server::onIndexCurrent); + addMethod("index/all", &Server::onIndexAll); addMethod("context/current", &Server::onContextCurrent); addMethod("context/switch", &Server::onContextSwitch); addMethod("context/all", &Server::onContextAll); diff --git a/tests/indexer/foo.cpp b/tests/indexer/foo.cpp new file mode 100644 index 00000000..0d377044 --- /dev/null +++ b/tests/indexer/foo.cpp @@ -0,0 +1,7 @@ +#include "foo.h" + +int foo(); + +int foo() { + return 42; +} \ No newline at end of file diff --git a/tests/indexer/foo.h b/tests/indexer/foo.h new file mode 100644 index 00000000..eacbb8d7 --- /dev/null +++ b/tests/indexer/foo.h @@ -0,0 +1,3 @@ +#pragma once + +int foo(); diff --git a/tests/indexer/macro.h b/tests/indexer/macro.h new file mode 100644 index 00000000..125a0d44 --- /dev/null +++ b/tests/indexer/macro.h @@ -0,0 +1,5 @@ +#ifdef TEST +struct A {}; +#else +struct B {}; +#endif diff --git a/tests/indexer/main.cpp b/tests/indexer/main.cpp new file mode 100644 index 00000000..aa2e31eb --- /dev/null +++ b/tests/indexer/main.cpp @@ -0,0 +1,12 @@ +#include "foo.h" + +#include "macro.h" +#define TEST +#include "macro.h" + +int main() { + int v = foo(); + A a = {}; + B b = {}; + return 0; +} diff --git a/unittests/Basic/SourceCode.cpp b/unittests/Basic/SourceCode.cpp new file mode 100644 index 00000000..3ab015ff --- /dev/null +++ b/unittests/Basic/SourceCode.cpp @@ -0,0 +1,68 @@ +#include "Test/Test.h" +#include "Basic/SourceCode.h" + +namespace clice::testing { + +TEST(SourceCode, tokenize) { + std::size_t count = 0; + + /// Test breaking the loop. + tokenize("int x = 1;", [&](const clang::Token& token) { + count++; + return false; + }); + + EXPECT_EQ(count, 1); + + /// Test all tokens. + count = 0; + + std::vector kinds = { + clang::tok::raw_identifier, + clang::tok::raw_identifier, + clang::tok::equal, + clang::tok::numeric_constant, + clang::tok::semi, + }; + + tokenize("int x = 1; // comment", [&](const clang::Token& token) { + if(token.is(clang::tok::eof)) [[unlikely]] { + return false; + } + + EXPECT_EQ(token.getKind(), kinds[count]); + count++; + return true; + }); + + EXPECT_EQ(count, 5); + + /// Test retain comments. + count = 0; + + kinds = { + clang::tok::raw_identifier, + clang::tok::raw_identifier, + clang::tok::equal, + clang::tok::numeric_constant, + clang::tok::semi, + clang::tok::comment, + }; + + tokenize( + "int x = 1; /// comment", + [&](const clang::Token& token) { + if(token.is(clang::tok::eof)) [[unlikely]] { + return false; + } + + EXPECT_EQ(token.getKind(), kinds[count]); + count++; + return true; + }, + false); + + EXPECT_EQ(count, 6); +} + +} // namespace clice::testing diff --git a/unittests/Feature/SemanticTokens.cpp b/unittests/Feature/SemanticTokens.cpp index 5641fa64..6abadb1e 100644 --- a/unittests/Feature/SemanticTokens.cpp +++ b/unittests/Feature/SemanticTokens.cpp @@ -5,7 +5,38 @@ namespace clice::testing { namespace { -TEST(Feature, SemanticTokens) {} +struct SemanticTokens : ::testing::Test { + std::optional tester; + index::Shared> result; + + struct InputCase { + std::uint32_t offset; + SymbolKind kind; + std::source_location current; + }; + + std::vector cases; + + void run(llvm::StringRef code) { + tester.emplace("main.cpp", code); + tester->run(); + auto& info = tester->info; + result = feature::semanticTokens(*info); + } + + void EXPECT_TOKEN(llvm::StringRef pos, + SymbolKind kind, + std::source_location current = std::source_location::current()) {} + + void test() {} +}; + +TEST_F(SemanticTokens, Include) { + run("#include "); + + auto& tokens = result[tester->info->getInterestedFile()]; + print("Tokens: {}\n", dump(tokens)); +} } // namespace diff --git a/unittests/Index/Serialization.cpp b/unittests/Index/Serialization.cpp new file mode 100644 index 00000000..b6a70113 --- /dev/null +++ b/unittests/Index/Serialization.cpp @@ -0,0 +1,71 @@ +#include "Test/IndexTester.h" + +namespace clice::testing { + +namespace { + +TEST(Index, locateSymbol) { + const char* code = R"cpp( +void $(1)foo(); + +void $(2)foo() {} +)cpp"; + + IndexTester tester{"main.cpp", code}; + tester.run(); + auto& info = *tester.info; + auto indices = index::index(info); + + ASSERT_EQ(indices.size(), 1); + auto& index = indices.begin()->second; + + llvm::SmallVector symbols; + index.locateSymbols(tester.offsets["1"], symbols); + ASSERT_EQ(symbols.size(), 1); + auto symbol = symbols[0]; +} + +TEST(Index, Serialization) { + const char* code = R"cpp( +void $(1)foo(); + +void $(2)foo() {} +)cpp"; + + IndexTester tester{"main.cpp", code}; + tester.run(); + auto& info = *tester.info; + auto indices = index::index(info); + + ASSERT_EQ(indices.size(), 1); + auto& index = indices.begin()->second; + + auto json = index.toJSON(); + + llvm::SmallString<128> path; + auto error = fs::createTemporaryFile("index", ".sidx", path); + ASSERT_FALSE(error); + + { + std::error_code ec; + llvm::raw_fd_ostream file(path, ec); + ASSERT_FALSE(ec); + file.write(static_cast(index.base), index.size); + } + + { + auto file = llvm::MemoryBuffer::getFile(path); + ASSERT_TRUE(bool(file)); + auto& buffer = file.get(); + auto size = buffer->getBufferSize(); + auto base = std::malloc(size); + std::memcpy(base, buffer->getBufferStart(), size); + index::SymbolIndex index(base, size); + EXPECT_EQ(json, index.toJSON()); + } +} + +} // namespace + +} // namespace clice::testing + diff --git a/unittests/Index/Template.cpp b/unittests/Index/Template.cpp index 57759ddf..97d3f0bf 100644 --- a/unittests/Index/Template.cpp +++ b/unittests/Index/Template.cpp @@ -4,42 +4,6 @@ namespace clice::testing { namespace { -TEST(Index, Test) { - - const char* code = R"cpp( -void $(1)foo(); - -void $(2)foo() {} -)cpp"; - - IndexTester tester{"main.cpp", code}; - tester.run(); - auto indices = index::test(*tester.info); - - std::size_t total = 0; - llvm::SmallVector symbols; - tester.GotoDefinition("1", "2"); - - for(auto& [id, index]: indices) { - // auto& srcMgr = tester.info.srcMgr(); - // auto entry = srcMgr.getFileEntryRefForID(id); - // if(!entry) { - // continue; - // } - // llvm::SmallString<128> path; - // auto err = fs::real_path(entry->getName(), path); - // print("File: {}, Size: {}k\n", path.str(), index.size / 1024); - // total += index.size; - // - // if(path == "/usr/include/c++/13/bits/stl_vector.h") { - // llvm::raw_fd_ostream os("stdlib.json", err); - // os << index.toJSON(); - //} - } - - print("Total size: {}k\n", total / 1024); -} - TEST(Index, ClassTemplate) { const char* code = R"cpp( template @@ -198,5 +162,5 @@ TEST(Index, Concept) { } // namespace -} // namespace clice +} // namespace clice::testing diff --git a/unittests/Server/Async.cpp b/unittests/Server/Async.cpp index d4e5e407..76fd6941 100644 --- a/unittests/Server/Async.cpp +++ b/unittests/Server/Async.cpp @@ -1,9 +1,67 @@ #include "Test/Test.h" #include "Server/Async.h" +#include namespace clice::testing { -namespace {} // namespace +namespace { -} // namespace clice +TEST(Async, Task) { + auto task = []() -> async::Task { + co_return 1; + }(); + + auto task2 = []() -> async::Task { + co_return 2; + }(); + + auto task3 = []() -> async::Task { + co_return 3; + }(); + + auto result = async::run(task, task2, task3); + + EXPECT_EQ(task.done(), true); + EXPECT_EQ(task2.done(), true); + EXPECT_EQ(task3.done(), true); + EXPECT_EQ(result, std::tuple{1, 2, 3}); +} + +TEST(Async, Submit) { + auto task = []() -> async::Task { + co_return co_await async::submit([]() { + std::this_thread::sleep_for(std::chrono::seconds(1)); + return std::this_thread::get_id(); + }); + }(); + + auto task2 = []() -> async::Task { + co_return co_await async::submit([]() { + std::this_thread::sleep_for(std::chrono::seconds(1)); + return std::this_thread::get_id(); + }); + }(); + + auto task3 = []() -> async::Task { + co_return co_await async::submit([]() { + std::this_thread::sleep_for(std::chrono::seconds(1)); + return std::this_thread::get_id(); + }); + }(); + + auto result = async::run(task, task2, task3); + + EXPECT_EQ(task.done(), true); + EXPECT_EQ(task2.done(), true); + EXPECT_EQ(task3.done(), true); + + auto [id1, id2, id3] = result; + EXPECT_NE(id1, id2); + EXPECT_NE(id2, id3); + EXPECT_NE(id1, id3); +} + +} // namespace + +} // namespace clice::testing diff --git a/unittests/Server/Indexer.cpp b/unittests/Server/Indexer.cpp new file mode 100644 index 00000000..4b7bf2d4 --- /dev/null +++ b/unittests/Server/Indexer.cpp @@ -0,0 +1,47 @@ +#include "Test/Test.h" +#include "Server/Indexer.h" + +namespace clice::testing { + +TEST(Indexer, Basic) { + config::IndexOptions options; + options.dir = path::join(".", "temp"); + auto error = fs::create_directories(options.dir); + + CompilationDatabase database; + auto prefix = path::join(test_dir(), "indexer"); + auto foo = path::real_path(path::join(prefix, "foo.cpp")); + auto main = path::real_path(path::join(prefix, "main.cpp")); + database.updateCommand(foo, std::format("clang++ {}", foo)); + database.updateCommand(main, std::format("clang++ {}", main)); + + Indexer indexer(options, database); + indexer.loadFromDisk(); + + auto p1 = indexer.index(main); + auto p2 = indexer.index(foo); + async::run(p1, p2); + + auto kind = + RelationKind(RelationKind::Reference, RelationKind::Definition, RelationKind::Declaration); + proto::DeclarationParams params{ + .textDocument = {.uri = SourceConverter::toURI(foo)}, + .position = {2, 5} + }; + auto lookup = indexer.lookup(params, kind); + auto&& [result] = async::run(lookup); + + indexer.saveToDisk(); + + Indexer indexer2(options, database); + indexer2.loadFromDisk(); + + auto lookup2 = indexer2.lookup(params, kind); + auto&& [result2] = async::run(lookup2); + + print("Result: {}\n", json::serialize(result)); + + EXPECT_EQ(result, result2); +} + +} // namespace clice::testing diff --git a/unittests/Support/Enum.cpp b/unittests/Support/Enum.cpp index 2d3bceee..1a4f8953 100644 --- a/unittests/Support/Enum.cpp +++ b/unittests/Support/Enum.cpp @@ -108,6 +108,11 @@ TEST(Reflection, MaskEnum) { EXPECT_TRUE(bool(mask6 & Mask::A)); EXPECT_TRUE(bool(mask6 & Mask::B)); EXPECT_TRUE(bool(mask6 & Mask::C)); + + constexpr Mask mask7 = Mask(Mask::A, Mask::B, Mask::C); + static_assert(mask7 & Mask::A); + static_assert(mask7 & Mask::B); + static_assert(mask7 & Mask::C); } struct StringEnum : refl::Enum {