diff --git a/include/Feature/DocumentLink.h b/include/Feature/DocumentLink.h index 731c7e17..d5fe6733 100644 --- a/include/Feature/DocumentLink.h +++ b/include/Feature/DocumentLink.h @@ -15,13 +15,13 @@ struct DocumentLink { std::string file; }; -using DocumentLinkResult = std::vector; +using DocumentLinks = std::vector; /// Generate document link for main file. -DocumentLinkResult documentLink(ASTInfo& AST); +DocumentLinks documentLinks(ASTInfo& AST); /// Generate document link for all source file. -index::Shared indexDocumentLink(ASTInfo& AST); +index::Shared indexDocumentLink(ASTInfo& AST); } // namespace clice::feature diff --git a/include/Feature/DocumentSymbol.h b/include/Feature/DocumentSymbol.h index 96cb4341..d0d2f5a3 100644 --- a/include/Feature/DocumentSymbol.h +++ b/include/Feature/DocumentSymbol.h @@ -33,7 +33,7 @@ using DocumentSymbols = std::vector; DocumentSymbols documentSymbols(ASTInfo& AST); /// Generate document symbols for all file in AST. -index::Shared indexDocumentSymbols(ASTInfo& AST); +index::Shared indexDocumentSymbol(ASTInfo& AST); } // namespace clice::feature diff --git a/include/Feature/FoldingRange.h b/include/Feature/FoldingRange.h index d945e23c..49769c0f 100644 --- a/include/Feature/FoldingRange.h +++ b/include/Feature/FoldingRange.h @@ -44,11 +44,13 @@ struct FoldingRange { std::string text; }; +using FoldingRanges = std::vector; + /// Generate folding range for interested file only. -std::vector foldingRange(ASTInfo& AST); +FoldingRanges foldingRanges(ASTInfo& AST); /// Generate folding range for all files. -index::Shared> indexFoldingRange(ASTInfo& AST); +index::Shared indexFoldingRange(ASTInfo& AST); } // namespace clice::feature diff --git a/include/Feature/InlayHint.h b/include/Feature/InlayHint.h index 97bb7f5e..d08f3be6 100644 --- a/include/Feature/InlayHint.h +++ b/include/Feature/InlayHint.h @@ -35,6 +35,6 @@ using InlayHints = std::vector; InlayHints inlayHints(ASTInfo& AST, LocalSourceRange target); -index::Shared indexInlayHints(ASTInfo& AST); +index::Shared indexInlayHint(ASTInfo& AST); } // namespace clice::feature diff --git a/include/Feature/SemanticTokens.h b/include/Feature/SemanticToken.h similarity index 74% rename from include/Feature/SemanticTokens.h rename to include/Feature/SemanticToken.h index 409f2f02..ed4a3972 100644 --- a/include/Feature/SemanticTokens.h +++ b/include/Feature/SemanticToken.h @@ -18,11 +18,13 @@ struct SemanticToken { SymbolModifiers modifiers; }; +using SemanticTokens = std::vector; + /// Generate semantic tokens for the interested file only. -std::vector semanticTokens(ASTInfo& AST); +SemanticTokens semanticTokens(ASTInfo& AST); /// Generate semantic tokens for all files. -index::Shared> indexSemanticTokens(ASTInfo& AST); +index::Shared indexSemanticToken(ASTInfo& AST); } // namespace clice::feature diff --git a/include/Index/FeatureIndex.h b/include/Index/FeatureIndex.h index ed6319c6..f67c793b 100644 --- a/include/Index/FeatureIndex.h +++ b/include/Index/FeatureIndex.h @@ -3,7 +3,7 @@ #include #include "Shared.h" -#include "Feature/SemanticTokens.h" +#include "Feature/SemanticToken.h" #include "Feature/FoldingRange.h" #include "Feature/DocumentLink.h" #include "Feature/DocumentSymbol.h" @@ -23,13 +23,13 @@ public: /// The content of source file. llvm::StringRef content(); - std::vector semanticTokens() const; + feature::SemanticTokens semanticTokens() const; - std::vector foldingRanges() const; + feature::FoldingRanges foldingRanges() const; - std::vector documentLinks() const; + feature::DocumentLinks documentLinks() const; - std::vector documentSymbols() const; + feature::DocumentSymbols documentSymbols() const; static Shared> build(ASTInfo& AST); diff --git a/include/Server/IncludeGraph.h b/include/Server/IncludeGraph.h index 3d1f1879..4cd5e285 100644 --- a/include/Server/IncludeGraph.h +++ b/include/Server/IncludeGraph.h @@ -3,7 +3,6 @@ #include "Config.h" #include "Protocol.h" #include "Async/Async.h" -#include "Server/SourceConverter.h" #include "Compiler/Command.h" #include "Compiler/Compilation.h" #include "Support/JSON.h" @@ -153,7 +152,6 @@ protected: llvm::StringMap tus; std::vector pathPool; llvm::StringMap pathIndices; - SourceConverter SC; }; } // namespace clice diff --git a/include/Server/LSPConverter.h b/include/Server/LSPConverter.h index b384ca26..7c91d7c9 100644 --- a/include/Server/LSPConverter.h +++ b/include/Server/LSPConverter.h @@ -6,7 +6,7 @@ #include "Feature/InlayHint.h" #include "Feature/FoldingRange.h" #include "Feature/DocumentSymbol.h" -#include "Feature/SemanticTokens.h" +#include "Feature/SemanticToken.h" #include "Feature/DocumentLink.h" #include "Server/Protocol.h" @@ -31,6 +31,9 @@ public: llvm::StringRef workspace(); public: + /// Convert a position into an offset relative to the beginning of the file. + uint32_t convert(llvm::StringRef content, proto::Position position); + proto::SemanticTokens transform(llvm::StringRef content, llvm::ArrayRef tokens); @@ -40,14 +43,6 @@ public: std::vector transform(llvm::StringRef content, llvm::ArrayRef links); - Result convert(llvm::StringRef path, llvm::ArrayRef tokens); - - Result convert(llvm::StringRef path, llvm::ArrayRef foldings); - - Result convert(llvm::StringRef path, llvm::ArrayRef links); - - Result convert(const feature::Hover& hover); - private: proto::InitializeParams params; std::string workspacePath; diff --git a/include/Server/SourceConverter.h b/include/Server/SourceConverter.h deleted file mode 100644 index 53f377a6..00000000 --- a/include/Server/SourceConverter.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include "Server/Protocol.h" -#include "AST/SourceCode.h" -#include "clang/Basic/SourceLocation.h" - -namespace clice { - -/// A helper class to convert `Position, Range, Location` between 1-1 encoding based clang and 0-0 -/// encoding based LSP. The conversion of DocumentUri is also supported. -class SourceConverter { -public: - /// [(origin, new)], map origin header directory to another source directory. - using SourceDirMapping = std::vector>; - - /// Construct a `SourceConverter` with the specified encoding kind and empty source map. - explicit SourceConverter(proto::PositionEncodingKind kind = proto::PositionEncodingKind::UTF8) : - kind(kind), sourceMap() {} - - SourceConverter(proto::PositionEncodingKind kind, SourceDirMapping sourceMap) : - kind(kind), sourceMap(std::move(sourceMap)) {} - - SourceConverter(const SourceConverter&) = delete; - - SourceConverter(SourceConverter&&) = default; - - /// Measure the length (character count) of the content with the specified encoding kind. - std::size_t remeasure(llvm::StringRef content) const; - - /// Same as the below, but input is raw offset to the content beginning. - proto::Position toPosition(llvm::StringRef content, std::uint32_t offset) const; - - /// Convert a clang::SourceLocation to a proto::Position according to the - /// specified encoding kind. Note that `SourceLocation` in clang is 1-based and - /// is always encoded in UTF-8. - proto::Position toPosition(clang::SourceLocation location, - const clang::SourceManager& SM) const; - - /// Convert a clang::SourceRange to proto::Range according to the specified encoding kind. - proto::Range toRange(clang::SourceRange range, const clang::SourceManager& SM) const; - - /// Same as the above, but input is a `LocalSourceRange` and the content is provided. - proto::Range toRange(LocalSourceRange range, llvm::StringRef content) const; - - /// Convert a clang::SourceRange to LocalSourceRange. - LocalSourceRange toLocalRange(clang::SourceRange range, const clang::SourceManager& SM) const; - - /// Convert a proto::Position to a file offset in the content with the specified - /// encoding kind. - std::uint32_t toOffset(llvm::StringRef content, proto::Position position) const; - - /// Get the encoding kind of the content in LSP protocol. - proto::PositionEncodingKind encodingKind() const { - return kind; - } - - /// Convert a real path of a file to URI. Crash if failed. - static proto::DocumentUri toURI(llvm::StringRef fspath); - - /// Convert a file URI to real path with `clice::fs::real_path`. Crash if failed. - static std::string toPath(llvm::StringRef uri); - -private: - /// The encoding kind of the content in LSP protocol. - proto::PositionEncodingKind kind; - - /// A user-defined map from header file to its source directory. - SourceDirMapping sourceMap; -}; - -} // namespace clice diff --git a/include/Support/FileSystem.h b/include/Support/FileSystem.h index f42d656a..8d47b5b9 100644 --- a/include/Support/FileSystem.h +++ b/include/Support/FileSystem.h @@ -7,6 +7,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/ADT/StringExtras.h" namespace clice { @@ -78,6 +79,80 @@ inline std::expected read(llvm::StringRef path) { return buffer.get()->getBuffer().str(); } +inline std::string toURI(llvm::StringRef fspath) { + if(!path::is_absolute(fspath)) + std::abort(); + + llvm::SmallString<128> path("file://"); +#if defined(_WIN32) + path.append("/"); +#endif + + for(auto c: fspath) { + if(c == '\\') { + path.push_back('/'); + } else if(std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '/') { + path.push_back(c); + } else { + path.push_back('%'); + path.push_back(llvm::hexdigit(c >> 4)); + path.push_back(llvm::hexdigit(c & 0xF)); + } + } + + return path.str().str(); +} + +inline std::string decodePercent(llvm::StringRef content) { + std::string result; + result.reserve(content.size()); + + for(auto iter = content.begin(), send = content.end(); iter != send; ++iter) { + auto c = *iter; + if(c == '%' && iter + 2 < send) { + auto m = *(iter + 1); + auto n = *(iter + 2); + if(llvm::isHexDigit(m) && llvm::isHexDigit(n)) { + result += llvm::hexFromNibbles(m, n); + iter += 2; + continue; + } + } + result += c; + } + return result; +} + +inline std::string toPath(llvm::StringRef uri) { + llvm::StringRef cloned = uri; + +#if defined(_WIN32) + if(cloned.starts_with("file:///")) { + cloned = cloned.drop_front(8); + } else { + std::abort(); + } +#elif defined(__unix__) + if(cloned.starts_with("file://")) { + cloned = cloned.drop_front(7); + } else { + std::abort(); + } +#else +#error "Unsupported platform" +#endif + + auto decoded = decodePercent(cloned); + + llvm::SmallString<128> result; + if(auto err = fs::real_path(decoded, result)) { + print("Failed to get real path: {}, Input is {}\n", err.message(), decoded); + std::abort(); + } + + return result.str().str(); +} + } // namespace fs namespace vfs = llvm::vfs; diff --git a/include/Test/IndexTester.h b/include/Test/IndexTester.h index d7ab3bd7..1a9ef89b 100644 --- a/include/Test/IndexTester.h +++ b/include/Test/IndexTester.h @@ -1,6 +1,5 @@ #include "Test/CTest.h" #include "Index/SymbolIndex.h" -#include "Server/SourceConverter.h" namespace clice::testing { diff --git a/src/Feature/DocumentLink.cpp b/src/Feature/DocumentLink.cpp index c89179c4..e70cbc2e 100644 --- a/src/Feature/DocumentLink.cpp +++ b/src/Feature/DocumentLink.cpp @@ -7,10 +7,10 @@ namespace clice::feature { namespace {} -DocumentLinkResult documentLink(ASTInfo& AST); +DocumentLinks documentLinks(ASTInfo& AST); -index::Shared indexDocumentLink(ASTInfo& AST) { - index::Shared result; +index::Shared indexDocumentLink(ASTInfo& AST) { + index::Shared result; for(auto& [fid, diretives]: AST.directives()) { for(auto& include: diretives.includes) { diff --git a/src/Feature/DocumentSymbol.cpp b/src/Feature/DocumentSymbol.cpp index dc37bc57..c66b6289 100644 --- a/src/Feature/DocumentSymbol.cpp +++ b/src/Feature/DocumentSymbol.cpp @@ -94,7 +94,7 @@ DocumentSymbols documentSymbols(ASTInfo& AST) { return std::move(frame.symbols); } -index::Shared indexDocumentSymbols(ASTInfo& AST) { +index::Shared indexDocumentSymbol(ASTInfo& AST) { DocumentSymbolCollector collector(AST, true); collector.TraverseDecl(AST.tu()); diff --git a/src/Feature/FoldingRange.cpp b/src/Feature/FoldingRange.cpp index 0370451a..f00575e0 100644 --- a/src/Feature/FoldingRange.cpp +++ b/src/Feature/FoldingRange.cpp @@ -322,17 +322,17 @@ private: private: clang::SourceManager& SM; clang::syntax::TokenBuffer& TB; - std::vector result; - index::Shared> indexResult; + FoldingRanges result; + index::Shared indexResult; }; } // namespace -std::vector foldingRange(ASTInfo& AST) { +FoldingRanges foldingRanges(ASTInfo& AST) { return FoldingRangeCollector(AST, true).buildForFile(AST); } -index::Shared> indexFoldingRange(ASTInfo& AST) { +index::Shared indexFoldingRange(ASTInfo& AST) { return FoldingRangeCollector(AST, false).buildForIndex(AST); } // namespace feature diff --git a/src/Feature/SemanticTokens.cpp b/src/Feature/SemanticToken.cpp similarity index 89% rename from src/Feature/SemanticTokens.cpp rename to src/Feature/SemanticToken.cpp index 31a8d5e8..71df396c 100644 --- a/src/Feature/SemanticTokens.cpp +++ b/src/Feature/SemanticToken.cpp @@ -2,7 +2,7 @@ #include "Index/Shared.h" #include "Support/Ranges.h" #include "Support/Compare.h" -#include "Feature/SemanticTokens.h" +#include "Feature/SemanticToken.h" namespace clice::feature { @@ -10,9 +10,8 @@ namespace { class SemanticTokensCollector : public SemanticVisitor { public: - SemanticTokensCollector(ASTInfo& AST, bool interestedOnly) : - emitForIndex(!interestedOnly), - SemanticVisitor(AST, interestedOnly) {} + using Base = SemanticVisitor; + using Base::Base; void handleDeclOccurrence(const clang::NamedDecl* decl, RelationKind kind, @@ -84,12 +83,12 @@ public: return std::move(sharedResult); } -private: +public: void addToken(clang::FileID fid, const clang::Token& token, SymbolKind kind) { auto offset = token.getLocation().getRawEncoding() - fakeLoc.getRawEncoding(); LocalSourceRange range{offset, offset + token.getLength()}; - auto& tokens = emitForIndex ? sharedResult[fid] : result; + auto& tokens = interestedOnly ? result : sharedResult[fid]; tokens.emplace_back(range, kind, SymbolModifiers()); } @@ -109,7 +108,7 @@ private: } auto [fid, range] = AST.toLocalRange(location); - auto& tokens = emitForIndex ? sharedResult[fid] : result; + auto& tokens = interestedOnly ? result : sharedResult[fid]; tokens.emplace_back(range, kind, modifiers); } @@ -229,11 +228,11 @@ private: } /// Merge tokens with same range and resolve kind conflict. - void merge(std::vector& tokens) { + void merge(SemanticTokens& tokens) { /// Sort tokens by range. std::ranges::sort(tokens, refl::less, [](const auto& token) { return token.range; }); - std::vector merged; + SemanticTokens merged; for(auto& token: tokens) { if(merged.empty()) { @@ -257,20 +256,34 @@ private: tokens = std::move(merged); } -private: - bool emitForIndex; - std::vector result; - index::Shared> sharedResult; +public: + SemanticTokens result; + index::Shared sharedResult; }; } // namespace -std::vector semanticTokens(ASTInfo& AST) { - return SemanticTokensCollector(AST, true).buildForFile(); +SemanticTokens semanticTokens(ASTInfo& AST) { + SemanticTokensCollector collector(AST, true); + collector.highlight(AST.getInterestedFile()); + collector.run(); + collector.merge(collector.result); + return std::move(collector.result); } -index::Shared> indexSemanticTokens(ASTInfo& AST) { - return SemanticTokensCollector(AST, false).buildForIndex(); +index::Shared indexSemanticToken(ASTInfo& AST) { + SemanticTokensCollector collector(AST, false); + for(auto fid: AST.files()) { + collector.highlight(fid); + } + + collector.run(); + + for(auto& [fid, tokens]: collector.sharedResult) { + collector.merge(tokens); + } + + return std::move(collector.sharedResult); } } // namespace clice::feature diff --git a/src/Index/FeatureIndex.cpp b/src/Index/FeatureIndex.cpp index c81bb484..c2be9c36 100644 --- a/src/Index/FeatureIndex.cpp +++ b/src/Index/FeatureIndex.cpp @@ -14,13 +14,13 @@ struct FeatureIndex { std::string content; /// The index of semantic tokens. - std::vector tokens; + feature::SemanticTokens tokens; /// The index of folding ranges. - std::vector foldings; + feature::FoldingRanges foldings; /// The index of document links. - std::vector links; + feature::DocumentLinks links; /// The index of document symbols. feature::DocumentSymbols symbols; @@ -38,22 +38,22 @@ llvm::StringRef FeatureIndex::content() { return index.get<"content">().as_string(); } -std::vector FeatureIndex::semanticTokens() const { +feature::SemanticTokens FeatureIndex::semanticTokens() const { binary::Proxy index{base, base}; return binary::deserialize(index.get<"tokens">()); } -std::vector FeatureIndex::foldingRanges() const { +feature::FoldingRanges FeatureIndex::foldingRanges() const { binary::Proxy index{base, base}; return binary::deserialize(index.get<"foldings">()); } -std::vector FeatureIndex::documentLinks() const { +feature::DocumentLinks FeatureIndex::documentLinks() const { binary::Proxy index{base, base}; return binary::deserialize(index.get<"links">()); } -std::vector FeatureIndex::documentSymbols() const { +feature::DocumentSymbols FeatureIndex::documentSymbols() const { binary::Proxy index{base, base}; return binary::deserialize(index.get<"symbols">()); } @@ -61,7 +61,7 @@ std::vector FeatureIndex::documentSymbols() const { Shared> FeatureIndex::build(ASTInfo& AST) { Shared indices; - for(auto&& [fid, result]: feature::indexSemanticTokens(AST)) { + for(auto&& [fid, result]: feature::indexSemanticToken(AST)) { indices[fid].tokens = std::move(result); } @@ -73,7 +73,7 @@ Shared> FeatureIndex::build(ASTInfo& AST) { indices[fid].links = std::move(result); } - for(auto&& [fid, result]: feature::indexDocumentSymbols(AST)) { + for(auto&& [fid, result]: feature::indexDocumentSymbol(AST)) { indices[fid].symbols = std::move(result); } diff --git a/src/Server/LSPConverter.cpp b/src/Server/LSPConverter.cpp index c5dc4666..41f40a35 100644 --- a/src/Server/LSPConverter.cpp +++ b/src/Server/LSPConverter.cpp @@ -1,5 +1,5 @@ #include "Server/LSPConverter.h" -#include "Server/SourceConverter.h" +#include "Support/FileSystem.h" namespace clice { @@ -238,7 +238,7 @@ proto::InitializeResult LSPConverter::initialize(json::Value value) { llvm::StringRef LSPConverter::workspace() { if(workspacePath.empty()) { - workspacePath = SourceConverter::toPath(params.workspaceFolders[0].uri); + workspacePath = fs::toPath(params.workspaceFolders[0].uri); } return workspacePath; } @@ -361,45 +361,10 @@ std::vector converter.toPosition(link.range.begin), converter.toPosition(link.range.end), }; - result.emplace_back(range, SourceConverter::toURI(link.file)); + result.emplace_back(range, fs::toURI(link.file)); } return result; } -LSPConverter::Result LSPConverter::convert(llvm::StringRef path, - llvm::ArrayRef tokens) { - auto file = co_await async::fs::read(path.str()); - if(!file) { - co_return json::Value(nullptr); - } - llvm::StringRef content = *file; - co_return json::serialize(transform(content, tokens)); -} - -LSPConverter::Result LSPConverter::convert(llvm::StringRef path, - llvm::ArrayRef foldings) { - auto file = co_await async::fs::read(path.str()); - if(!file) { - co_return json::Value(nullptr); - } - llvm::StringRef content = *file; - co_return json::serialize(transform(content, foldings)); -} - -LSPConverter::Result LSPConverter::convert(llvm::StringRef path, - llvm::ArrayRef links) { - auto file = co_await async::fs::read(path.str()); - if(!file) { - co_return json::Value(nullptr); - } - llvm::StringRef content = *file; - co_return json::serialize(transform(content, links)); -} - -LSPConverter::Result LSPConverter::convert(const feature::Hover& hover) { - /// FIXME: Implement hover information render here. - co_return json::Value(""); -} - } // namespace clice diff --git a/src/Server/Lifecycle.cpp b/src/Server/Lifecycle.cpp index f40b1316..72ff54a1 100644 --- a/src/Server/Lifecycle.cpp +++ b/src/Server/Lifecycle.cpp @@ -1,4 +1,3 @@ -#include "Server/SourceConverter.h" #include "Server/Server.h" #include "Support/FileSystem.h" diff --git a/src/Server/Server.cpp b/src/Server/Server.cpp index a6dc488a..853814c7 100644 --- a/src/Server/Server.cpp +++ b/src/Server/Server.cpp @@ -78,19 +78,39 @@ async::Task Server::onRequest(llvm::StringRef method, json::Value v async::Task Server::onTextDocument(llvm::StringRef method, json::Value value) { if(method == "semanticTokens/full") { auto params2 = json::deserialize(value); - auto path = SourceConverter::toPath(params2.textDocument.uri); - auto tokens = co_await indexer.semanticTokens(path); - co_return co_await converter.convert(path, tokens); + auto path = fs::toPath(params2.textDocument.uri); + + std::string buffer; + if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { + co_return json::serialize( + converter.transform(index->content(), index->semanticTokens())); + } else { + co_return json::Value(nullptr); + } + } else if(method == "foldingRange") { auto params2 = json::deserialize(value); - auto path = SourceConverter::toPath(params2.textDocument.uri); - auto foldings = co_await indexer.foldingRanges(path); - co_return co_await converter.convert(path, foldings); + auto path = fs::toPath(params2.textDocument.uri); + + std::string buffer; + if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { + co_return json::serialize( + converter.transform(index->content(), index->foldingRanges())); + } else { + co_return json::Value(nullptr); + } + } else if(method == "documentLink") { auto params2 = json::deserialize(value); - auto path = SourceConverter::toPath(params2.textDocument.uri); - auto links = co_await indexer.documentLinks(path); - co_return co_await converter.convert(path, links); + auto path = fs::toPath(params2.textDocument.uri); + + std::string buffer; + if(auto index = co_await indexer.getFeatureIndex(buffer, path)) { + co_return json::serialize( + converter.transform(index->content(), index->documentLinks())); + } else { + co_return json::Value(nullptr); + } } co_return json::Value(nullptr); @@ -99,16 +119,16 @@ async::Task Server::onTextDocument(llvm::StringRef method, json::Va async::Task Server::onContext(llvm::StringRef method, json::Value value) { if(method == "current") { auto param2 = json::deserialize(value); - auto path = SourceConverter::toPath(param2.textDocument.uri); + auto path = fs::toPath(param2.textDocument.uri); auto result = indexer.currentContext(path); co_return result ? json::serialize(*result) : json::Value(nullptr); } else if(method == "switch") { auto params = json::deserialize(value); - auto header = SourceConverter::toPath(params.header); + auto header = fs::toPath(params.header); indexer.switchContext(header, params.context); } else if(method == "all") { auto param2 = json::deserialize(value); - auto path = SourceConverter::toPath(param2.textDocument.uri); + auto path = fs::toPath(param2.textDocument.uri); auto result = indexer.allContexts(path); co_return json::serialize(result); } else if(method == "resolve") { diff --git a/src/Server/SourceConverter.cpp b/src/Server/SourceConverter.cpp deleted file mode 100644 index 9ab8862b..00000000 --- a/src/Server/SourceConverter.cpp +++ /dev/null @@ -1,290 +0,0 @@ -#include "Server/Protocol.h" -#include "Server/SourceConverter.h" -#include "Support/FileSystem.h" -#include "clang/Basic/SourceManager.h" -#include "llvm/ADT/StringExtras.h" - -namespace clice { - -/// @brief Iterates over Unicode codepoints in a UTF-8 encoded string and invokes a callback for -/// each codepoint. -/// -/// Processes the input UTF-8 string, calculating the length of each Unicode codepoint in both -/// UTF-8 (bytes) and UTF-16 (code units), and passes these lengths to the callback. -/// Iteration stops early if the callback returns `false`. -/// -/// ASCII characters are treated as 1-byte UTF-8 codepoints with a UTF-16 length of 1. -/// Non-ASCII characters are processed based on their leading byte to determine UTF-8 length: -/// - Valid lengths are 2 to 4 bytes. -/// - Astral codepoints (UTF-8 length of 4) have a UTF-16 length of 2 code units. -/// Invalid UTF-8 sequences are treated as single-byte ASCII characters. -/// -/// Returns `false` if the callback stops the iteration. -template -static bool iterateCodepoints(llvm::StringRef content, const Callback& callback) { - // Iterate over the input string, processing each codepoint. - for(size_t index = 0; index < content.size();) { - unsigned char c = static_cast(content[index]); - - // Handle ASCII characters (1-byte UTF-8, 1-code-unit UTF-16). - if(!(c & 0x80)) [[likely]] { - if(!callback(1, 1)) { - return true; - } - - ++index; - continue; - } - - // Determine the length of the codepoint in UTF-8 by counting the leading 1s. - size_t length = llvm::countl_one(c); - - // Validate UTF-8 encoding: length must be between 2 and 4. - if(length < 2 || length > 4) [[unlikely]] { - assert(false && "Invalid UTF-8 sequence"); - - // Treat the byte as an ASCII character. - if(!callback(1, 1)) { - return true; - } - - ++index; - continue; - } - - // Advance the index by the length of the current UTF-8 codepoint. - index += length; - - // Calculate the UTF-16 length: astral codepoints (4-byte UTF-8) take 2 code units. - if(!callback(length, length == 4 ? 2 : 1)) { - return true; - } - } - - return false; -} - -std::size_t SourceConverter::remeasure(llvm::StringRef content) const { - if(kind == proto::PositionEncodingKind::UTF8) { - return content.size(); - } - - if(kind == proto::PositionEncodingKind::UTF16) { - std::size_t length = 0; - iterateCodepoints(content, [&](size_t, size_t utf16Length) { - length += utf16Length; - return true; - }); - return length; - } - - if(kind == proto::PositionEncodingKind::UTF32) { - std::size_t length = 0; - iterateCodepoints(content, [&](size_t, size_t) { - length += 1; - return true; - }); - return length; - } - - std::unreachable(); -} - -proto::Position SourceConverter::toPosition(llvm::StringRef content, std::uint32_t offset) const { - assert(offset <= content.size() && "Offset is out of range"); - proto::Position position = {0, 0}; - - std::uint32_t line = 0; - std::uint32_t column = 0; - for(std::uint32_t i = 0; i < offset; i++) { - auto c = content[i]; - if(c == '\n') { - line += 1; - column = 0; - } else { - column += 1; - } - } - - /// Line doesn't need to be adjusted. - position.line = line; - - /// Column needs to be adjusted based on the encoding. - if(column > 0) { - auto word = content.substr(offset - column, column); - position.character = remeasure(word); - } - - return position; -} - -proto::Position SourceConverter::toPosition(clang::SourceLocation location, - const clang::SourceManager& SM) const { - assert(location.isValid() && location.isFileID() && - "SourceLocation must be valid and not a macro location"); - auto [fid, offset] = SM.getDecomposedSpellingLoc(location); - auto content = getFileContent(SM, fid); - return toPosition(content, offset); -} - -proto::Range SourceConverter::toRange(clang::SourceRange range, - const clang::SourceManager& SM) const { - auto [begin, end] = range; - assert(begin.isValid() && end.isValid() && "Invalid SourceRange"); - assert(begin.isFileID() && end.isFileID() && "SourceRange must be FileID"); - - auto [fileID, offset] = SM.getDecomposedSpellingLoc(end); - auto content = getFileContent(SM, fileID); - return { - toPosition(begin, SM), - toPosition(content, offset + getTokenLength(SM, end)), - }; -} - -proto::Range SourceConverter::toRange(LocalSourceRange range, llvm::StringRef content) const { - return { - .start = toPosition(content, range.begin), - .end = toPosition(content, range.end), - }; -} - -LocalSourceRange SourceConverter::toLocalRange(clang::SourceRange range, - const clang::SourceManager& SM) const { - return { - .begin = SM.getDecomposedLoc(range.getBegin()).second, - .end = SM.getDecomposedLoc(range.getEnd()).second, - }; -} - -std::uint32_t SourceConverter::toOffset(llvm::StringRef content, proto::Position position) const { - std::uint32_t offset = 0; - for(auto i = 0; i < position.line; i++) { - auto pos = content.find('\n'); - assert(pos != llvm::StringRef::npos && "Line value is out of range"); - - offset += pos + 1; - content = content.substr(pos + 1); - } - - /// Drop the content after the line. - content = content.take_until([](char c) { return c == '\n'; }); - assert(position.character <= content.size() && "Character value is out of range"); - - if(kind == proto::PositionEncodingKind::UTF8) { - offset += position.character; - return offset; - } - - if(kind == proto::PositionEncodingKind::UTF16) { - iterateCodepoints(content, [&](size_t utf8Length, size_t utf16Length) { - assert(position.character >= utf16Length && "Character value is out of range"); - position.character -= utf16Length; - offset += utf8Length; - return position.character != 0; - }); - return offset; - } - - if(kind == proto::PositionEncodingKind::UTF32) { - iterateCodepoints(content, [&](size_t utf8Length, size_t) { - assert(position.character >= 1 && "Character value is out of range"); - position.character -= 1; - offset += utf8Length; - return position.character != 0; - }); - return offset; - } - - std::unreachable(); -} - -namespace { - -/// decodes a string according to percent-encoding, e.g., "a%20b" -> "a b". -static std::string decodePercent(llvm::StringRef content) { - std::string result; - result.reserve(content.size()); - - for(auto iter = content.begin(), send = content.end(); iter != send; ++iter) { - auto c = *iter; - if(c == '%' && iter + 2 < send) { - auto m = *(iter + 1); - auto n = *(iter + 2); - if(llvm::isHexDigit(m) && llvm::isHexDigit(n)) { - result += llvm::hexFromNibbles(m, n); - iter += 2; - continue; - } - } - result += c; - } - return result; -} - -} // namespace - -proto::DocumentUri SourceConverter::toURI(llvm::StringRef fspath) { - if(!path::is_absolute(fspath)) - std::abort(); - - llvm::SmallString<128> path("file://"); -#if defined(_WIN32) - path.append("/"); -#endif - - for(auto c: fspath) { - if(c == '\\') { - path.push_back('/'); - } else if(std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '/') { - path.push_back(c); - } else { - path.push_back('%'); - path.push_back(llvm::hexdigit(c >> 4)); - path.push_back(llvm::hexdigit(c & 0xF)); - } - } - - /// TODO: - /// use `sourceMap` to replace prefix with mapped path. - // for(const auto& [prefix, newPrefix]: sourceMap) { - // if(fspath.starts_with(prefix)) { - // path.append(newPrefix); // todo: newPrefix.end_with('/') ??? - // path.append(fspath.substr(prefix.size())); - // break; - // } - // } - - return path.str().str(); -}; - -std::string SourceConverter::toPath(llvm::StringRef uri) { - llvm::StringRef cloned = uri; - -#if defined(_WIN32) - if(cloned.starts_with("file:///")) { - cloned = cloned.drop_front(8); - } else { - std::abort(); - } -#elif defined(__unix__) - if(cloned.starts_with("file://")) { - cloned = cloned.drop_front(7); - } else { - std::abort(); - } -#else -#error "Unsupported platform" -#endif - - auto decoded = decodePercent(cloned); - - llvm::SmallString<128> result; - if(auto err = fs::real_path(decoded, result)) { - print("Failed to get real path: {}, Input is {}\n", err.message(), decoded); - std::abort(); - } - - return result.str().str(); -} - -} // namespace clice diff --git a/unittests/AST/Selection.cpp b/unittests/AST/Selection.cpp index c8ef7ab2..8e4d888f 100644 --- a/unittests/AST/Selection.cpp +++ b/unittests/AST/Selection.cpp @@ -1,5 +1,4 @@ #include "src/AST/Selection.cpp" -#include "Server/SourceConverter.h" #include "Test/CTest.h" @@ -50,9 +49,6 @@ void debug(const SelectionTree& tree) { } struct SelectionTester : public Tester { - - const SourceConverter cvtr = SourceConverter(proto::PositionEncodingKind::UTF8); - SelectionTester(llvm::StringRef file, llvm::StringRef content) : Tester(file, content) {} void expectPreorderSequence(const SelectionTree& tree, diff --git a/unittests/Compiler/Directive.cpp b/unittests/Compiler/Directive.cpp index 83724e89..6a1d6309 100644 --- a/unittests/Compiler/Directive.cpp +++ b/unittests/Compiler/Directive.cpp @@ -1,5 +1,4 @@ #include "Test/CTest.h" -#include "Server/SourceConverter.h" namespace clice::testing { diff --git a/unittests/Compiler/Preamble.cpp b/unittests/Compiler/Preamble.cpp index 219b83e2..27cd259b 100644 --- a/unittests/Compiler/Preamble.cpp +++ b/unittests/Compiler/Preamble.cpp @@ -1,5 +1,4 @@ #include "Test/Test.h" -#include "Server/SourceConverter.h" #include "Compiler/Preamble.h" #include "Compiler/Compilation.h" diff --git a/unittests/Feature/DocumentLink.cpp b/unittests/Feature/DocumentLink.cpp index c816bf00..702bbe9d 100644 --- a/unittests/Feature/DocumentLink.cpp +++ b/unittests/Feature/DocumentLink.cpp @@ -6,7 +6,7 @@ namespace clice::testing { namespace { struct DocumentLink : TestFixture { - index::Shared result; + index::Shared result; void run(llvm::StringRef code) { addMain("main.cpp", code); diff --git a/unittests/Feature/DocumentSymbol.cpp b/unittests/Feature/DocumentSymbol.cpp index dfcfa567..312e3de8 100644 --- a/unittests/Feature/DocumentSymbol.cpp +++ b/unittests/Feature/DocumentSymbol.cpp @@ -1,5 +1,4 @@ #include "Test/CTest.h" -#include "Server/SourceConverter.h" #include "Feature/DocumentSymbol.h" namespace clice::testing { @@ -224,7 +223,7 @@ int y = 2; auto& info = tx.AST; EXPECT_TRUE(info.has_value()); - auto maps = feature::indexDocumentSymbols(*info); + auto maps = feature::indexDocumentSymbol(*info); for(auto& [fileID, result]: maps) { if(fileID == info->srcMgr().getMainFileID()) { EXPECT_EQ(total_size(result), 2); diff --git a/unittests/Feature/FoldingRange.cpp b/unittests/Feature/FoldingRange.cpp index a4466430..d4c8f434 100644 --- a/unittests/Feature/FoldingRange.cpp +++ b/unittests/Feature/FoldingRange.cpp @@ -11,7 +11,7 @@ struct FoldingRange : TestFixture { void run(llvm::StringRef source) { addMain("main.cpp", source); TestFixture::compile(); - result = feature::foldingRange(*AST); + result = feature::foldingRanges(*AST); } void EXPECT_RANGE(std::size_t index, diff --git a/unittests/Feature/InlayHint.cpp b/unittests/Feature/InlayHint.cpp index 8fb20d01..93a96ebd 100644 --- a/unittests/Feature/InlayHint.cpp +++ b/unittests/Feature/InlayHint.cpp @@ -1,6 +1,5 @@ #include "Test/CTest.h" #include "Feature/InlayHint.h" -#include "Server/SourceConverter.h" namespace clice::testing { diff --git a/unittests/Feature/SemanticTokens.cpp b/unittests/Feature/SemanticToken.cpp similarity index 90% rename from unittests/Feature/SemanticTokens.cpp rename to unittests/Feature/SemanticToken.cpp index 89c39dec..c26f0f35 100644 --- a/unittests/Feature/SemanticTokens.cpp +++ b/unittests/Feature/SemanticToken.cpp @@ -1,17 +1,17 @@ #include "Test/CTest.h" -#include "Feature/SemanticTokens.h" +#include "Feature/SemanticToken.h" namespace clice::testing { namespace { -struct SemanticTokens : TestFixture { - index::Shared> result; +struct SemanticToken : TestFixture { + index::Shared result; void run(llvm::StringRef code) { addMain("main.cpp", code); Tester::compile(); - result = feature::indexSemanticTokens(*AST); + result = feature::indexSemanticToken(*AST); } void EXPECT_TOKEN(llvm::StringRef pos, @@ -67,7 +67,7 @@ struct SemanticTokens : TestFixture { using enum SymbolKind::Kind; using enum SymbolModifiers::Kind; -TEST_F(SemanticTokens, Include) { +TEST_F(SemanticToken, Include) { run(R"cpp( $(0)#include $(1) $(2)#include $(3)"stddef.h" @@ -85,7 +85,7 @@ $(4)# $(5)include $(6)"stddef.h" EXPECT_TOKEN("6", Header, 10); } -TEST_F(SemanticTokens, Comment) { +TEST_F(SemanticToken, Comment) { run(R"cpp( $(line)/// line comment int x = 1; @@ -94,7 +94,7 @@ int x = 1; EXPECT_TOKEN("line", Comment, 16); } -TEST_F(SemanticTokens, Keyword) { +TEST_F(SemanticToken, Keyword) { run(R"cpp( $(int)int main() { $(return)return 0; @@ -105,7 +105,7 @@ $(int)int main() { EXPECT_TOKEN("return", Keyword, 6); } -TEST_F(SemanticTokens, Macro) { +TEST_F(SemanticToken, Macro) { run(R"cpp( $(0)#define $(macro)FOO )cpp"); @@ -114,7 +114,7 @@ $(0)#define $(macro)FOO EXPECT_TOKEN("macro", Macro, 3); } -TEST_F(SemanticTokens, FinalAndOverride) { +TEST_F(SemanticToken, FinalAndOverride) { run(R"cpp( struct A $(0)final {}; @@ -136,7 +136,7 @@ struct D : C { EXPECT_TOKEN("2", Keyword, 5); } -TEST_F(SemanticTokens, VarDecl) { +TEST_F(SemanticToken, VarDecl) { run(R"cpp( extern int $(0)x; @@ -172,7 +172,7 @@ int main() { EXPECT_TOKEN("7", Variable, SymbolModifiers(), 1); } -TEST_F(SemanticTokens, FunctionDecl) { +TEST_F(SemanticToken, FunctionDecl) { run(R"cpp( extern int $(0)foo(); @@ -195,7 +195,7 @@ int $(3)bar() { EXPECT_TOKEN("3", Function, SymbolModifiers(Definition, Templated), 3); } -TEST_F(SemanticTokens, RecordDecl) { +TEST_F(SemanticToken, RecordDecl) { run(R"cpp( class $(0)A; diff --git a/unittests/Server/Indexer.cpp b/unittests/Server/Indexer.cpp index 6ac2f14d..5f62ff50 100644 --- a/unittests/Server/Indexer.cpp +++ b/unittests/Server/Indexer.cpp @@ -25,7 +25,7 @@ TEST(Indexer, Basic) { auto kind = RelationKind(RelationKind::Reference, RelationKind::Definition, RelationKind::Declaration); proto::ReferenceParams params; - params.textDocument = {.uri = SourceConverter::toURI(foo)}; + params.textDocument = {.uri = fs::toURI(foo)}; params.position = {2, 5}; // auto lookup = indexer.lookup(params, kind); diff --git a/unittests/Server/SourceConverter.cpp b/unittests/Server/SourceConverter.cpp deleted file mode 100644 index 270019d9..00000000 --- a/unittests/Server/SourceConverter.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include "Test/CTest.h" -#include "Server/SourceConverter.h" - -namespace clice::testing { - -namespace { - -TEST(SourceConverter, Remeasure) { - using SC = SourceConverter; - - SourceConverter utf8{proto::PositionEncodingKind::UTF8}; - - EXPECT_EQ(utf8.remeasure(""), 0); - EXPECT_EQ(utf8.remeasure("ascii"), 5); - - SourceConverter utf16{proto::PositionEncodingKind::UTF16}; - - EXPECT_EQ(utf16.remeasure("↓"), 1); - EXPECT_EQ(utf16.remeasure("¥"), 1); - - SourceConverter utf32{proto::PositionEncodingKind::UTF32}; - EXPECT_EQ(utf8.remeasure("😂"), 4); - EXPECT_EQ(utf16.remeasure("😂"), 2); - EXPECT_EQ(utf32.remeasure("😂"), 1); -} - -TEST(SourceConverter, Position) { - const char* main = "int a /*😂*/ = 1;$(eof)"; - - Tester txs("main.cpp", main); - txs.compile("-std=c++11"); - - auto& src = txs.AST->srcMgr(); - auto& tks = txs.AST->tokBuf(); - - auto mainid = src.getMainFileID(); - auto tokens = - tks.expandedTokens({src.getLocForStartOfFile(mainid), src.getLocForEndOfFile(mainid)}); - - auto eof = tokens.back().endLocation(); - - { - SourceConverter cvtr{proto::PositionEncodingKind::UTF8}; - auto pos = cvtr.toPosition(eof, src); - EXPECT_EQ(pos.line, 0); - EXPECT_EQ(pos.character, 19); - } - - { - SourceConverter cvtr{proto::PositionEncodingKind::UTF16}; - auto pos = cvtr.toPosition(eof, src); - EXPECT_EQ(pos.line, 0); - EXPECT_EQ(pos.character, 17); - } - - { - SourceConverter cvtr{proto::PositionEncodingKind::UTF32}; - auto pos = cvtr.toPosition(eof, src); - EXPECT_EQ(pos.line, 0); - EXPECT_EQ(pos.character, 16); - } -} - -TEST(SourceConverter, UriAndFsPath) { - using SC = SourceConverter; - - // It must be a existed file. -#if defined(__unix__) - llvm::StringRef paths[] = {"/dev/null"}; - llvm::StringRef uris[] = {"file:///dev/null"}; -#elif defined(_WIN32) - llvm::StringRef paths[] = {"C:\\Windows\\System32\\notepad.exe"}; - llvm::StringRef uris[] = {"file:///C%3A/Windows/System32/notepad.exe"}; -#else -#error "Unsupported platform" -#endif - - EXPECT_EQ(std::size(paths), std::size(uris)); - - for(int i = 0; i < std::size(paths); ++i) { - llvm::outs() << ": " << SC::toURI(paths[i]) << "\n"; - llvm::outs() << ": " << SC::toPath(uris[i]) << "\n"; - EXPECT_EQ(SC::toURI(paths[i]), uris[i]); - EXPECT_EQ(paths[i], SC::toPath(uris[i])); - } -} - -} // namespace - -} // namespace clice::testing