Code completion (from Sema) (#144)

This commit is contained in:
ykiko
2025-07-07 11:32:16 +08:00
committed by GitHub
parent bfd6bbadbb
commit cce576dfdd
12 changed files with 1216 additions and 271 deletions

View File

@@ -168,6 +168,10 @@ void CompilationDatabase::add_command(this Self& self,
auto path_ = self.save_string(path);
auto new_args = self.save_args(args);
/// FIXME: Use a better way to handle resource dir.
/// new_args.push_back(self.save_string(std::format("-resource-dir={}",
/// fs::resource_dir)).data());
auto it = self.commands.find(path_.data());
if(it == self.commands.end()) {
self.commands.try_emplace(path_.data(),

View File

@@ -141,6 +141,11 @@ std::expected<CompilationUnit, std::string> clang_compile(CompilationParams& par
return std::unexpected(std::format("Failed to execute action, because {} ", error));
}
if(instance->getDiagnostics().hasFatalErrorOccurred()) {
action->EndSourceFile();
return report_diagnostics("Fetal error occured!!!", *diagnostics);
}
std::optional<clang::syntax::TokenBuffer> tokBuf;
if(tokCollector) {
tokBuf = std::move(*tokCollector).consume();

View File

@@ -114,7 +114,7 @@ void dumpArg(clang::DiagnosticsEngine::ArgumentKind kind, std::uint64_t value) {
bool locationInRange(clang::SourceLocation L,
clang::CharSourceRange R,
const clang::SourceManager& M) {
assert(R.isCharRange());
/// assert(R.isCharRange());
if(!R.isValid() || M.getFileID(R.getBegin()) != M.getFileID(R.getEnd()) ||
M.getFileID(R.getBegin()) != M.getFileID(L))
return false;

View File

@@ -2,6 +2,7 @@
#include "AST/SymbolKind.h"
#include "Compiler/Compilation.h"
#include "Feature/CodeCompletion.h"
#include "Support/FuzzyMatcher.h"
#include "clang/Sema/Sema.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/CodeCompleteConsumer.h"
@@ -11,120 +12,366 @@ namespace clice::feature {
namespace {
struct CompletionPrefix {
// The unqualified partial name.
// If there is none, begin() == end() == completion position.
llvm::StringRef name;
/// The range that will replaced.
LocalSourceRange range;
// The spelled scope qualifier, such as Foo::.
// If there is none, begin() == end() == name.begin().
llvm::StringRef qualifier;
/// The unqualified partial name.
/// If there is none, begin() == end() == completion position.
llvm::StringRef spelling;
/// The spelled scope qualifier, such as Foo::.
/// If there is none, begin() == end() == name.begin().
/// FIXME: llvm::StringRef qualifier;
static CompletionPrefix from(llvm::StringRef content, std::uint32_t offset) {
assert(offset <= content.size());
CompletionPrefix prefix;
llvm::StringRef rest = content.take_front(offset);
assert(offset <= content.size());
// Consume the unqualified name. We only handle ASCII characters.
// isAsciiIdentifierContinue will let us match "0invalid", but we don't mind.
while(!rest.empty() && clang::isAsciiIdentifierContinue(rest.back())) {
rest = rest.drop_back();
auto start = offset;
while(start > 0 && clang::isAsciiIdentifierStart(content[start - 1])) {
--start;
}
prefix.name = content.slice(rest.size(), offset);
auto end = offset;
while(end < content.size() && clang::isAsciiIdentifierStart(content[end])) {
++end;
}
// Consume qualifiers.
while(rest.consume_back("::") && !rest.ends_with(":")) {
// reject ::::
while(!rest.empty() && clang::isAsciiIdentifierContinue(rest.back())) {
rest = rest.drop_back();
return CompletionPrefix{
LocalSourceRange{start, end},
content.substr(start, offset - start),
};
}
};
/// Get the code completion kind of named declaration.
CompletionItemKind completion_kind(const clang::NamedDecl* decl) {
switch(decl->getKind()) {
case clang::Decl::Namespace:
case clang::Decl::NamespaceAlias: {
/// TODO: Extend `namespace` kind.
return CompletionItemKind::Module;
}
case clang::Decl::Function:
case clang::Decl::FunctionTemplate: {
return CompletionItemKind::Function;
}
case clang::Decl::CXXMethod:
case clang::Decl::CXXConversion:
case clang::Decl::CXXDestructor:
case clang::Decl::CXXDeductionGuide: {
return CompletionItemKind::Method;
}
case clang::Decl::CXXConstructor: {
return CompletionItemKind::Constructor;
}
case clang::Decl::Var:
case clang::Decl::ParmVar:
case clang::Decl::ImplicitParam:
case clang::Decl::Binding:
case clang::Decl::Decomposition:
case clang::Decl::VarTemplate:
case clang::Decl::VarTemplateSpecialization:
case clang::Decl::VarTemplatePartialSpecialization:
case clang::Decl::NonTypeTemplateParm: {
return CompletionItemKind::Variable;
}
case clang::Decl::Label: {
return CompletionItemKind::Variable;
}
case clang::Decl::Enum: {
return CompletionItemKind::Enum;
}
case clang::Decl::EnumConstant: {
return CompletionItemKind::EnumMember;
}
case clang::Decl::Field:
case clang::Decl::IndirectField: {
return CompletionItemKind::Field;
}
case clang::Decl::Record:
case clang::Decl::CXXRecord:
case clang::Decl::ClassTemplate:
case clang::Decl::ClassTemplateSpecialization:
case clang::Decl::ClassTemplatePartialSpecialization: {
return CompletionItemKind::Class;
}
case clang::Decl::Typedef:
case clang::Decl::TypeAlias:
case clang::Decl::TemplateTypeParm:
case clang::Decl::TemplateTemplateParm:
case clang::Decl::TypeAliasTemplate:
case clang::Decl::Concept:
case clang::Decl::BuiltinTemplate: {
return CompletionItemKind::TypeParameter;
}
case clang::Decl::UnresolvedUsingValue:
case clang::Decl::UnnamedGlobalConstant:
case clang::Decl::TemplateParamObject: {
return CompletionItemKind::Constant;
}
case clang::Decl::MSGuid:
case clang::Decl::MSProperty:
case clang::Decl::Using:
case clang::Decl::UsingPack:
case clang::Decl::UsingEnum:
case clang::Decl::UsingShadow:
case clang::Decl::UsingDirective:
case clang::Decl::UnresolvedUsingIfExists:
case clang::Decl::UnresolvedUsingTypename:
case clang::Decl::ConstructorUsingShadow: {
/// FIXME: Is it possible that above kinds occur in
/// code completion result?
llvm_unreachable("Unexpected declaration");
}
/// The following kinds are not `NamedDecl`, we just ignore them.
case clang::Decl::TranslationUnit:
case clang::Decl::PragmaComment:
case clang::Decl::PragmaDetectMismatch:
case clang::Decl::ExternCContext:
case clang::Decl::ImplicitConceptSpecialization:
case clang::Decl::LinkageSpec:
case clang::Decl::Export:
case clang::Decl::FileScopeAsm:
case clang::Decl::TopLevelStmt:
case clang::Decl::AccessSpec:
case clang::Decl::Friend:
case clang::Decl::FriendTemplate:
case clang::Decl::StaticAssert:
case clang::Decl::Block:
/// case clang::Decl::OutlinedFunction:
case clang::Decl::Captured:
case clang::Decl::Import:
case clang::Decl::Empty:
case clang::Decl::RequiresExprBody:
case clang::Decl::LifetimeExtendedTemporary:
case clang::Decl::OMPAllocate:
case clang::Decl::OMPRequires:
case clang::Decl::OMPDeclareMapper:
case clang::Decl::OMPCapturedExpr:
case clang::Decl::OMPThreadPrivate:
case clang::Decl::OMPDeclareReduction:
case clang::Decl::ObjCIvar:
case clang::Decl::ObjCMethod:
case clang::Decl::ObjCProtocol:
case clang::Decl::ObjCProperty:
case clang::Decl::ObjCCategory:
case clang::Decl::ObjCInterface:
case clang::Decl::ObjCTypeParam:
case clang::Decl::ObjCAtDefsField:
case clang::Decl::ObjCPropertyImpl:
case clang::Decl::ObjCCategoryImpl:
case clang::Decl::ObjCImplementation:
case clang::Decl::ObjCCompatibleAlias:
case clang::Decl::HLSLBuffer: {
llvm_unreachable("Invalid code completion item, NamedDecl is expected");
}
}
}
/// Calculate the final code completion result.
class CompletionRender {
public:
CompletionRender(clang::Sema& sema,
std::uint32_t offset,
llvm::StringRef content,
std::vector<CompletionItem>& items,
clang::CodeCompletionContext& cc_context,
const config::CodeCompletionOption& option) :
sema(sema), context(sema.getASTContext()), content(content),
prefix(CompletionPrefix::from(content, offset)), items(items), macther(prefix.spelling),
cc_context(cc_context), option(option) {}
struct OverloadSet {
/// The first declaration of this overload set.
const clang::NamedDecl* first;
/// The score of the overload name.
float score;
/// The count of functions in this overload set.
std::uint32_t count;
};
/// Render edit text for declaration.
std::string render(const clang::NamedDecl* decl) {
return getDeclName(decl);
}
void process_candidate(clang::CodeCompletionResult& candidate) {
feature::CompletionItem item;
item.edit.range = prefix.range;
/// Check whether the name matchs, if so, set the item.
auto check_name = [&](llvm::StringRef name) {
auto score = macther.match(name);
if(!score) {
return false;
}
item.label = name;
item.score = *score;
item.edit.text = item.label;
return true;
};
switch(candidate.Kind) {
case clang::CodeCompletionResult::RK_Keyword: {
if(!check_name(candidate.Keyword)) {
return;
}
item.edit.text = item.label;
item.kind = feature::CompletionItemKind::Keyword;
item.edit.text = item.label;
break;
}
case clang::CodeCompletionResult::RK_Pattern: {
auto text = candidate.Pattern->getAllTypedText();
if(!check_name(text)) {
return;
}
item.kind = CompletionItemKind::Snippet;
item.edit.text = item.label;
break;
}
case clang::CodeCompletionResult::RK_Macro: {
if(!check_name(candidate.Macro->getName())) {
return;
}
item.kind = feature::CompletionItemKind::Unit;
item.edit.text = item.label;
break;
}
case clang::CodeCompletionResult::RK_Declaration: {
auto declaration = candidate.Declaration;
/// if(declaration)
/// auto name = getDeclName();
if(!check_name(getDeclName(declaration))) {
return;
}
/// Otherwise just add this declaration.
item.kind = completion_kind(declaration);
/// If bundle_overloads is enabled and this is a function, just
/// increase the overload count.
if(option.bundle_overloads && item.kind == CompletionItemKind::Function) {
llvm::SmallString<256> qualfied_name;
llvm::raw_svector_ostream os(qualfied_name);
declaration->printQualifiedName(os);
auto hash = llvm::xxh3_64bits(qualfied_name);
OverloadSet set{declaration, item.score, 1};
auto [it, success] = overloads.try_emplace(hash, set);
if(!success) {
it->second.count += 1;
}
return;
}
item.edit.text = render(declaration);
}
}
prefix.qualifier = content.slice(rest.size(), prefix.name.begin() - content.begin());
return prefix;
items.emplace_back(std::move(item));
}
/// TODO: Handle dependent name with `TemplateResolver`.
void handle_dependent() {
/// For qualified name
/// auto specifier = context.getCXXScopeSpecifier();
/// auto NNS = specifier.value()->getScopeRep();
/// For member access like a.c
/// auto base = context.getBaseType();
/// Preferred type.
/// auto type = context.getPreferredType();
}
~CompletionRender() {
/// Add overload set.
for(auto& [_, overload_set]: overloads) {
CompletionItem item;
item.label = getDeclName(overload_set.first);
item.kind = CompletionItemKind::Function;
item.score = overload_set.score;
item.edit.range = prefix.range;
if(overload_set.count == 1) {
item.edit.text = render(overload_set.first);
/// TODO: Render function signature.
item.description = "";
} else {
item.edit.text = item.label;
item.description = "(...)";
}
items.emplace_back(std::move(item));
}
}
private:
/// clang `Sema` ref.
clang::Sema& sema;
/// clang `Sema` ref.
clang::ASTContext& context;
/// The content of current completion file.
llvm::StringRef content;
/// The completion prefix.
CompletionPrefix prefix;
/// The fuzzy matcher to score results.
FuzzyMatcher macther;
/// The code completion results.
std::vector<CompletionItem>& items;
/// The code completion context.
clang::CodeCompletionContext& cc_context;
/// A map between overload identifier hash and overloads count.
llvm::DenseMap<std::uint64_t, OverloadSet> overloads;
/// The code completion option.
const config::CodeCompletionOption& option;
};
class CodeCompletionCollector final : public clang::CodeCompleteConsumer {
public:
CodeCompletionCollector(std::uint32_t offset) :
CodeCompletionCollector(std::uint32_t offset,
std::vector<CompletionItem>& items,
const config::CodeCompletionOption& option) :
clang::CodeCompleteConsumer({}), offset(offset),
info(std::make_shared<clang::GlobalCodeCompletionAllocator>()) {}
CompletionItem processCandidate(clang::CodeCompletionResult& candidate) {
CompletionItem item;
switch(candidate.Kind) {
case clang::CodeCompletionResult::RK_Keyword: {
item.label = candidate.Keyword;
item.kind = CompletionItemKind::Keyword;
break;
}
case clang::CodeCompletionResult::RK_Pattern: {
item.label = candidate.Pattern->getAsString();
item.kind = CompletionItemKind::Snippet;
break;
}
case clang::CodeCompletionResult::RK_Macro: {
item.label = candidate.Macro->getName();
item.kind = CompletionItemKind::Unit;
break;
}
case clang::CodeCompletionResult::RK_Declaration: {
auto decl = candidate.Declaration;
item.label = getDeclName(decl);
item.kind = CompletionItemKind::Function;
break;
}
}
item.deprecated = false;
item.edit.text = item.label;
item.edit.range = editRange;
return item;
}
void initCompletionInfo(clang::Sema& sema) {
if(init) {
return;
}
auto& PP = sema.getPreprocessor();
auto& SM = sema.getSourceManager();
auto loc = PP.getCodeCompletionLoc();
auto content = SM.getBufferData(SM.getFileID(loc));
editRange = {offset, offset};
/// FIXME: consume the prefix of completion prefix, because we may complete
/// full qualified name.
assert(editRange.begin > 0);
while(clang::isAsciiIdentifierContinue(content[editRange.begin - 1])) {
editRange.begin -= 1;
}
if(editRange.end < content.size()) {
while(clang::isAsciiIdentifierContinue(content[editRange.end])) {
editRange.end += 1;
}
}
init = true;
}
void ProcessCodeCompleteResults(clang::Sema& sema,
clang::CodeCompletionContext context,
clang::CodeCompletionResult* candidates,
unsigned count) final {
initCompletionInfo(sema);
for(auto& candidate: llvm::make_range(candidates, candidates + count)) {
completions.emplace_back(processCandidate(candidate));
}
}
info(std::make_shared<clang::GlobalCodeCompletionAllocator>()), items(items),
option(option) {}
clang::CodeCompletionAllocator& getAllocator() final {
return info.getAllocator();
@@ -134,43 +381,57 @@ public:
return info;
}
auto dump() {
return std::move(completions);
void ProcessCodeCompleteResults(clang::Sema& sema,
clang::CodeCompletionContext cc_context,
clang::CodeCompletionResult* candidates,
unsigned candidates_count) final {
// Results from recovery mode are generally useless, and the callback after
// recovery (if any) is usually more interesting. To make sure we handle the
// future callback from sema, we just ignore all callbacks in recovery mode,
// as taking only results from recovery mode results in poor completion
// results.
// FIXME: in case there is no future sema completion callback after the
// recovery mode, we might still want to provide some results (e.g. trivial
// identifier-based completion).
if(cc_context.getKind() == clang::CodeCompletionContext::CCC_Recovery) {
return;
}
if(candidates_count == 0) {
return;
}
/// FIXME: check Sema may run multiple times.
auto& src_mgr = sema.getSourceManager();
auto content = src_mgr.getBufferData(src_mgr.getMainFileID());
CompletionRender render(sema, offset, content, items, cc_context, option);
for(auto& candidate: llvm::make_range(candidates, candidates + candidates_count)) {
render.process_candidate(candidate);
}
}
private:
clang::ASTContext* Ctx;
bool init = false;
/// TODO:
/// 1. 计算 token 边界,思考该怎么计算比较合适
/// 比如 std::vec^ 选择 vector => std::vector
/// 比如 vec 选择 std::vector => std::vector
/// 不仅要考虑前缀,也要考虑 token 后缀的替换
/// 之后记得试一下 clion 里面对 prefix 和 suffix 的处理
///
/// 2. 如果发现用户的光标正在补全头文件,则可以把该行头文件之
/// 前的代码全 substr 掉,然后再在结果上加几行或者 offset这样
/// 可以大大优化补全头文件的速度,毕竟头文件补全只和编译命令有关
/// 由于 #include 后面可能跟着宏,所以确保出现 <> 或者 "" 再进行这种
/// 优化
std::uint32_t offset;
LocalSourceRange editRange;
clang::CodeCompletionTUInfo info;
std::vector<CompletionItem> completions;
std::vector<CompletionItem>& items;
const config::CodeCompletionOption& option;
};
} // namespace
std::vector<CompletionItem> codeCompletion(CompilationParams& params,
const config::CodeCompletionOption& option) {
std::vector<CompletionItem> code_complete(CompilationParams& params,
const config::CodeCompletionOption& option) {
std::vector<CompletionItem> items;
auto& [file, offset] = params.completion;
auto consumer = new CodeCompletionCollector(offset);
auto consumer = new CodeCompletionCollector(offset, items, option);
if(auto info = complete(params, consumer)) {
return consumer->dump();
/// TODO: Handle error here.
} else {
return {};
}
return items;
}
} // namespace clice::feature

View File

@@ -408,13 +408,14 @@ json::Value LSPConverter::convert(llvm::StringRef content,
json::Array result;
for(auto& item: items) {
json::Object object{
{"label", item.label },
{"kind", static_cast<int>(item.kind)},
{"label", item.label},
{"kind", static_cast<int>(item.kind)},
{"textEdit",
json::Object{
{"newText", item.edit.text},
{"range", json::serialize(converter.lookup(item.edit.range))},
} },
}},
{"sortText", std::format("{}", item.score)},
};
result.emplace_back(std::move(object));
}

View File

@@ -36,6 +36,9 @@ async::Task<json::Value> Scheduler::semanticToken(std::string path) {
openFile = &openFiles[path];
auto content = openFile->content;
auto AST = openFile->AST;
if(!AST) {
co_return json::Value(nullptr);
}
auto tokens = co_await async::submit([&] { return feature::semanticTokens(*AST); });
@@ -59,7 +62,7 @@ async::Task<json::Value> Scheduler::completion(std::string path, std::uint32_t o
params.pch = {PCH->path, PCH->preamble.size()};
params.completion = {path, offset};
auto result = co_await async::submit([&] { return feature::codeCompletion(params, {}); });
auto result = co_await async::submit([&] { return feature::code_complete(params, {}); });
openFile = &openFiles[path];
co_return converter.convert(openFile->content, result);
@@ -106,14 +109,26 @@ async::Task<> Scheduler::buildPCH(std::string path, std::string content) {
std::string content) -> async::Task<> {
CompilationParams params;
params.arguments = scheduler.database.get_command(path);
params.outPath = path::join(config::index.dir, path::filename(path) + ".pch");
params.outPath = path::join(config::cache.dir, path::filename(path) + ".pch");
params.add_remapped_file(path, content, bound);
PCHInfo info;
auto result = co_await async::submit([&] { return compile(params, info); });
if(!result) {
/// FIXME: Fails needs cancel waiting tasks.
log::warn("Building PCH fails for {}", path);
/// PCH file is written until destructing, Add a single block
/// for it.
bool cond = co_await async::submit([&] {
auto result = compile(params, info);
if(!result) {
log::warn("Building PCH fails for {}, Because: {}", path, result.error());
return false;
}
/// TODO: index PCH.
return true;
});
if(!cond) {
co_return;
}
@@ -167,19 +182,19 @@ async::Task<> Scheduler::buildAST(std::string path, std::string content) {
params.pch = {PCH->path, PCH->preamble.size()};
/// Check result
auto info = co_await async::submit([&] { return compile(params); });
if(!info) {
auto AST = co_await async::submit([&] { return compile(params); });
if(!AST) {
/// FIXME: Fails needs cancel waiting tasks.
log::warn("Building AST fails for {}", path);
log::warn("Building AST fails for {}, Beacuse: {}", path, AST.error());
co_return;
}
/// Index the source file.
co_await indexer.index(*info);
co_await indexer.index(*AST);
file = &openFiles[path];
/// Update built AST info.
file->AST = std::make_shared<CompilationUnit>(std::move(*info));
file->AST = std::make_shared<CompilationUnit>(std::move(*AST));
/// Dispose the task so that it will destroyed when task complete.
file->ASTBuild.dispose();

View File

@@ -0,0 +1,445 @@
// To check for a match between a Pattern ('u_p') and a Word ('unique_ptr'),
// we consider the possible partial match states:
//
// u n i q u e _ p t r
// +---------------------
// |A . . . . . . . . . .
// u|
// |. . . . . . . . . . .
// _|
// |. . . . . . . O . . .
// p|
// |. . . . . . . . . . B
//
// Each dot represents some prefix of the pattern being matched against some
// prefix of the word.
// - A is the initial state: '' matched against ''
// - O is an intermediate state: 'u_' matched against 'unique_'
// - B is the target state: 'u_p' matched against 'unique_ptr'
//
// We aim to find the best path from A->B.
// - Moving right (consuming a word character)
// Always legal: not all word characters must match.
// - Moving diagonally (consuming both a word and pattern character)
// Legal if the characters match.
// - Moving down (consuming a pattern character) is never legal.
// Never legal: all pattern characters must match something.
// Characters are matched case-insensitively.
// The first pattern character may only match the start of a word segment.
//
// The scoring is based on heuristics:
// - when matching a character, apply a bonus or penalty depending on the
// match quality (does case match, do word segments align, etc)
// - when skipping a character, apply a penalty if it hurts the match
// (it starts a word segment, or splits the matched region, etc)
//
// These heuristics require the ability to "look backward" one character, to
// see whether it was matched or not. Therefore the dynamic-programming matrix
// has an extra dimension (last character matched).
// Each entry also has an additional flag indicating whether the last-but-one
// character matched, which is needed to trace back through the scoring table
// and reconstruct the match.
//
// We treat strings as byte-sequences, so only ASCII has first-class support.
//
// This algorithm was inspired by VS code's client-side filtering, and aims
// to be mostly-compatible.
//
//===----------------------------------------------------------------------===//
#include "Support/FuzzyMatcher.h"
#include "llvm/Support/Format.h"
namespace clice {
static char lower(char C) {
return C >= 'A' && C <= 'Z' ? C + ('a' - 'A') : C;
}
// A "negative infinity" score that won't overflow.
// We use this to mark unreachable states and forbidden solutions.
// Score field is 15 bits wide, min value is -2^14, we use half of that.
constexpr static int AwfulScore = -(1 << 13);
static bool is_awful(int S) {
return S < AwfulScore / 2;
}
constexpr static int PerfectBonus = 4; // Perfect per-pattern-char score.
FuzzyMatcher::FuzzyMatcher(llvm::StringRef pattern) :
pat_n(std::min<int>(MaxPat, pattern.size())),
score_scale(pat_n ? float{1} / (PerfectBonus * pat_n) : 0), word_n(0) {
std::copy(pattern.begin(), pattern.begin() + pat_n, Pat);
for(int I = 0; I < pat_n; ++I)
low_pat[I] = lower(Pat[I]);
scores[0][0][Miss] = {0, Miss};
scores[0][0][Match] = {AwfulScore, Miss};
for(int P = 0; P <= pat_n; ++P) {
for(int W = 0; W < P; ++W) {
for(Action A: {Miss, Match}) {
scores[P][W][A] = {AwfulScore, Miss};
}
}
}
pat_type_set =
calculate_roles(llvm::StringRef(Pat, pat_n), llvm::MutableArrayRef(pat_role, pat_n));
}
std::optional<float> FuzzyMatcher::match(llvm::StringRef word) {
if(!(word_contains_pattern = init(word))) {
return std::nullopt;
}
if(!pat_n) {
return 1;
}
build_graph();
auto best = std::max(scores[pat_n][word_n][Miss].score, scores[pat_n][word_n][Match].score);
if(is_awful(best)) {
return std::nullopt;
}
float score = score_scale * std::min(PerfectBonus * pat_n, std::max<int>(0, best));
// If the pattern is as long as the word, we have an exact string match,
// since every pattern character must match something.
if(word_n == pat_n) {
// May not be perfect 2 if case differs in a significant way.
score *= 2;
}
return score;
}
// We get CharTypes from a lookup table. Each is 2 bits, 4 fit in each byte.
// The top 6 bits of the char select the byte, the bottom 2 select the offset.
// e.g. 'q' = 010100 01 = byte 28 (55), bits 3-2 (01) -> Lower.
constexpr static uint8_t CharTypes[] = {
0x00, 0x00, 0x00, 0x00, // Control characters
0x00, 0x00, 0x00, 0x00, // Control characters
0xff, 0xff, 0xff, 0xff, // Punctuation
0x55, 0x55, 0xf5, 0xff, // Numbers->Lower, more Punctuation.
0xab, 0xaa, 0xaa, 0xaa, // @ and A-O
0xaa, 0xaa, 0xea, 0xff, // P-Z, more Punctuation.
0x57, 0x55, 0x55, 0x55, // ` and a-o
0x55, 0x55, 0xd5, 0x3f, // p-z, Punctuation, DEL.
0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, // Bytes over 127 -> Lower.
0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, // (probably UTF-8).
0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55,
};
// The Role can be determined from the Type of a character and its neighbors:
//
// Example | Chars | Type | Role
// ---------+--------------+-----
// F(o)oBar | Foo | Ull | Tail
// Foo(B)ar | oBa | lUl | Head
// (f)oo | ^fo | Ell | Head
// H(T)TP | HTT | UUU | Tail
//
// Our lookup table maps a 6 bit key (Prev, Curr, Next) to a 2-bit Role.
// A byte packs 4 Roles. (Prev, Curr) selects a byte, Next selects the offset.
// e.g. Lower, Upper, Lower -> 01 10 01 -> byte 6 (aa), bits 3-2 (10) -> Head.
constexpr static uint8_t CharRoles[] = {
// clang-format off
// Curr= Empty Lower Upper Separ
/* Prev=Empty */ 0x00, 0xaa, 0xaa, 0xff, // At start, Lower|Upper->Head
/* Prev=Lower */ 0x00, 0x55, 0xaa, 0xff, // In word, Upper->Head;Lower->Tail
/* Prev=Upper */ 0x00, 0x55, 0x59, 0xff, // Ditto, but U(U)U->Tail
/* Prev=Separ */ 0x00, 0xaa, 0xaa, 0xff, // After separator, like at start
// clang-format on
};
template <typename T>
static T packed_lookup(const uint8_t* Data, int I) {
return static_cast<T>((Data[I >> 2] >> ((I & 3) * 2)) & 3);
}
CharTypeSet calculate_roles(llvm::StringRef text, llvm::MutableArrayRef<CharRole> roles) {
assert(text.size() == roles.size());
if(text.size() == 0) {
return 0;
}
CharType type = packed_lookup<CharType>(CharTypes, text[0]);
CharTypeSet TypeSet = 1 << type;
// Types holds a sliding window of (Prev, Curr, Next) types.
// Initial value is (Empty, Empty, type of Text[0]).
int types = type;
// Rotate slides in the type of the next character.
auto rotate = [&](CharType T) {
types = ((types << 2) | T) & 0x3f;
};
for(unsigned I = 0; I < text.size() - 1; ++I) {
// For each character, rotate in the next, and look up the role.
type = packed_lookup<CharType>(CharTypes, text[I + 1]);
TypeSet |= 1 << type;
rotate(type);
roles[I] = packed_lookup<CharRole>(CharRoles, types);
}
// For the last character, the "next character" is Empty.
rotate(Empty);
roles[text.size() - 1] = packed_lookup<CharRole>(CharRoles, types);
return TypeSet;
}
// Sets up the data structures matching Word.
// Returns false if we can cheaply determine that no match is possible.
bool FuzzyMatcher::init(llvm::StringRef new_word) {
word_n = std::min<int>(MaxWord, new_word.size());
if(pat_n > word_n) {
return false;
}
std::copy(new_word.begin(), new_word.begin() + word_n, word);
if(pat_n == 0) {
return true;
}
for(int I = 0; I < word_n; ++I) {
low_word[I] = lower(word[I]);
}
// Cheap subsequence check.
for(int W = 0, P = 0; P != pat_n; ++W) {
if(W == word_n) {
return false;
}
if(low_word[W] == low_pat[P]) {
++P;
}
}
// FIXME: some words are hard to tokenize algorithmically.
// e.g. vsprintf is V S Print F, and should match [pri] but not [int].
// We could add a tokenization dictionary for common stdlib names.
word_type_set =
calculate_roles(llvm::StringRef(word, word_n), llvm::MutableArrayRef(word_role, word_n));
return true;
}
// The forwards pass finds the mappings of Pattern onto Word.
// Score = best score achieved matching Word[..W] against Pat[..P].
// Unlike other tables, indices range from 0 to N *inclusive*
// Matched = whether we chose to match Word[W] with Pat[P] or not.
//
// Points are mostly assigned to matched characters, with 1 being a good score
// and 3 being a great one. So we treat the score range as [0, 3 * PatN].
// This range is not strict: we can apply larger bonuses/penalties, or penalize
// non-matched characters.
void FuzzyMatcher::build_graph() {
for(int W = 0; W < word_n; ++W) {
scores[0][W + 1][Miss] = {scores[0][W][Miss].score - skip_penalty(W, Miss), Miss};
scores[0][W + 1][Match] = {AwfulScore, Miss};
}
for(int P = 0; P < pat_n; ++P) {
for(int W = P; W < word_n; ++W) {
auto &score = scores[P + 1][W + 1], &PreMiss = scores[P + 1][W];
auto match_miss_score = PreMiss[Match].score;
auto miss_miss_score = PreMiss[Miss].score;
if(P < pat_n - 1) { // Skipping trailing characters is always free.
match_miss_score -= skip_penalty(W, Match);
miss_miss_score -= skip_penalty(W, Miss);
}
score[Miss] = (match_miss_score > miss_miss_score) ? ScoreInfo{match_miss_score, Match}
: ScoreInfo{miss_miss_score, Miss};
auto& pre_match = scores[P][W];
auto match_match_score = allow_match(P, W, Match)
? pre_match[Match].score + match_bonus(P, W, Match)
: AwfulScore;
auto miss_match_score = allow_match(P, W, Miss)
? pre_match[Miss].score + match_bonus(P, W, Miss)
: AwfulScore;
score[Match] = (match_match_score > miss_match_score)
? ScoreInfo{match_match_score, Match}
: ScoreInfo{miss_match_score, Miss};
}
}
}
bool FuzzyMatcher::allow_match(int P, int W, Action last) const {
if(low_pat[P] != low_word[W]) {
return false;
}
// We require a "strong" match:
// - for the first pattern character. [foo] !~ "barefoot"
// - after a gap. [pat] !~ "patnther"
if(last == Miss) {
// We're banning matches outright, so conservatively accept some other cases
// where our segmentation might be wrong:
// - allow matching B in ABCDef (but not in NDEBUG)
// - we'd like to accept print in sprintf, but too many false positives
if(word_role[W] == Tail && (word[W] == low_word[W] || !(word_type_set & 1 << Lower))) {
return false;
}
}
return true;
}
int FuzzyMatcher::skip_penalty(int W, Action Last) const {
// Skipping the first character.
if(W == 0) {
return 3;
}
// Skipping a segment. We want to keep this lower than a consecutive match bonus.
// Instead of penalizing non-consecutive matches, we give a bonus to a
// consecutive match in matchBonus. This produces a better score distribution
// than penalties in case of small patterns, e.g. 'up' for 'unique_ptr'.
if(word_role[W] == Head) {
return 1;
}
return 0;
}
int FuzzyMatcher::match_bonus(int P, int W, Action last) const {
assert(low_pat[P] == low_word[W]);
int S = 1;
bool is_pat_single_case = (pat_type_set == 1 << Lower) || (pat_type_set == 1 << Upper);
// Bonus: case matches, or a Head in the pattern aligns with one in the word.
// Single-case patterns lack segmentation signals and we assume any character
// can be a head of a segment.
if(Pat[P] == word[W] || (word_role[W] == Head && (is_pat_single_case || pat_role[P] == Head))) {
++S;
}
// Bonus: a consecutive match. First character match also gets a bonus to
// ensure prefix final match score normalizes to 1.0.
if(W == 0 || last == Match) {
S += 2;
}
// Penalty: matching inside a segment (and previous char wasn't matched).
if(word_role[W] == Tail && P && last == Miss) {
S -= 3;
}
// Penalty: a Head in the pattern matches in the middle of a word segment.
if(pat_role[P] == Head && word_role[W] == Tail) {
--S;
}
// Penalty: matching the first pattern character in the middle of a segment.
if(P == 0 && word_role[W] == Tail) {
S -= 4;
}
assert(S <= PerfectBonus);
return S;
}
llvm::SmallString<256> FuzzyMatcher::dumpLast(llvm::raw_ostream& OS) const {
llvm::SmallString<256> result;
OS << "=== Match \"" << llvm::StringRef(word, word_n) << "\" against ["
<< llvm::StringRef(Pat, pat_n) << "] ===\n";
if(pat_n == 0) {
OS << "Pattern is empty: perfect match.\n";
return result = llvm::StringRef(word, word_n);
}
if(word_n == 0) {
OS << "Word is empty: no match.\n";
return result;
}
if(!word_contains_pattern) {
OS << "Substring check failed.\n";
return result;
}
if(is_awful(std::max(scores[pat_n][word_n][Match].score, scores[pat_n][word_n][Miss].score))) {
OS << "Substring check passed, but all matches are forbidden\n";
}
if(!(pat_type_set & 1 << Upper)) {
OS << "Lowercase query, so scoring ignores case\n";
}
// Traverse Matched table backwards to reconstruct the Pattern/Word mapping.
// The Score table has cumulative scores, subtracting along this path gives
// us the per-letter scores.
Action last =
(scores[pat_n][word_n][Match].score > scores[pat_n][word_n][Miss].score) ? Match : Miss;
int S[MaxWord];
Action A[MaxWord];
for(int W = word_n - 1, P = pat_n - 1; W >= 0; --W) {
A[W] = last;
const auto& Cell = scores[P + 1][W + 1][last];
if(last == Match)
--P;
const auto& Prev = scores[P + 1][W][Cell.Prev];
S[W] = Cell.score - Prev.score;
last = Cell.Prev;
}
for(int I = 0; I < word_n; ++I) {
if(A[I] == Match && (I == 0 || A[I - 1] == Miss)) {
result.push_back('[');
}
if(A[I] == Miss && I > 0 && A[I - 1] == Match) {
result.push_back(']');
}
result.push_back(word[I]);
}
if(A[word_n - 1] == Match) {
result.push_back(']');
}
for(char C: llvm::StringRef(word, word_n))
OS << " " << C << " ";
OS << "\n";
for(int I = 0, J = 0; I < word_n; I++)
OS << " " << (A[I] == Match ? Pat[J++] : ' ') << " ";
OS << "\n";
for(int I = 0; I < word_n; I++)
OS << llvm::format("%2d ", S[I]);
OS << "\n";
OS << "\nSegmentation:";
OS << "\n'" << llvm::StringRef(word, word_n) << "'\n ";
for(int I = 0; I < word_n; ++I)
OS << "?-+ "[static_cast<int>(word_role[I])];
OS << "\n[" << llvm::StringRef(Pat, pat_n) << "]\n ";
for(int I = 0; I < pat_n; ++I)
OS << "?-+ "[static_cast<int>(pat_role[I])];
OS << "\n";
OS << "\nScoring table (last-Miss, last-Match):\n";
OS << " | ";
for(char C: llvm::StringRef(word, word_n))
OS << " " << C << " ";
OS << "\n";
OS << "-+----" << std::string(word_n * 4, '-') << "\n";
for(int I = 0; I <= pat_n; ++I) {
for(Action A: {Miss, Match}) {
OS << ((I && A == Miss) ? Pat[I - 1] : ' ') << "|";
for(int J = 0; J <= word_n; ++J) {
if(!is_awful(scores[I][J][A].score))
OS << llvm::format("%3d%c",
scores[I][J][A].score,
scores[I][J][A].Prev == Match ? '*' : ' ');
else
OS << " ";
}
OS << "\n";
}
}
return result;
}
} // namespace clice