diff --git a/include/Compiler/Compiler.h b/include/Compiler/Compiler.h index 59f4f610..ff9ba63b 100644 --- a/include/Compiler/Compiler.h +++ b/include/Compiler/Compiler.h @@ -80,6 +80,10 @@ public: return llvm::StringRef(srcMgr().getCharacterData(loc), getTokenLength(loc)); } + auto getLocation(clang::SourceLocation loc) { + return srcMgr().getPresumedLoc(loc); + } + private: std::unique_ptr action; std::unique_ptr instance; @@ -131,6 +135,8 @@ struct CompliationParams { /// Information about reuse PCM(name, path). llvm::SmallVector> pcms; + llvm::IntrusiveRefCntPtr vfs = llvm::vfs::getRealFileSystem(); + uint32_t line = 0, column = 0; void addPCH(const PCHInfo& info) { diff --git a/include/Compiler/Directive.h b/include/Compiler/Directive.h index c03424b7..b0494842 100644 --- a/include/Compiler/Directive.h +++ b/include/Compiler/Directive.h @@ -4,7 +4,16 @@ namespace clice { -struct Include {}; +struct Include { + /// The path of the included file. + llvm::StringRef path; + + /// Location of the directive identifier. + clang::SourceLocation loc; + + /// Range of the filename. + clang::SourceRange range; +}; struct Condition { enum BranchKind : uint8_t { diff --git a/include/Support/Compare.h b/include/Support/Compare.h index 2f038a53..9e1917cf 100644 --- a/include/Support/Compare.h +++ b/include/Support/Compare.h @@ -5,16 +5,16 @@ namespace clice::support { -template +template struct Equal { - static bool equal(const T& lhs, const T& rhs) { + static bool equal(const LHS& lhs, const RHS& rhs) { return lhs == rhs; } }; -template -bool equal(const T& lhs, const T& rhs) { - return Equal::equal(lhs, rhs); +template +bool equal(const LHS& lhs, const RHS& rhs) { + return Equal::equal(lhs, rhs); } template diff --git a/include/Support/JSON.h b/include/Support/JSON.h index 9b491dd7..bfddbff8 100644 --- a/include/Support/JSON.h +++ b/include/Support/JSON.h @@ -173,6 +173,13 @@ struct Serde { } }; +template +struct Serde { + static json::Value serialize(const char (&v)[N]) { + return json::Value(llvm::StringRef(v, N)); + } +}; + template <> struct Serde { using V = llvm::StringRef; diff --git a/src/Compiler/Compiler.cpp b/src/Compiler/Compiler.cpp index c4938baf..9676af63 100644 --- a/src/Compiler/Compiler.cpp +++ b/src/Compiler/Compiler.cpp @@ -28,19 +28,22 @@ void adjustInvocation(clang::CompilerInvocation& invocation) { // FIXME: add more. } -auto createInstance(llvm::ArrayRef args) { +auto createInstance(CompliationParams& params) { auto instance = std::make_unique(); /// TODO: Figure out `CreateInvocationOptions`. clang::CreateInvocationOptions options = {}; - instance->setInvocation(clang::createInvocation(args, options)); + options.VFS = params.vfs; + instance->setInvocation(clang::createInvocation(params.args, options)); /// TODO: use a thread safe filesystem and our customized `DiagnosticConsumer`. instance->createDiagnostics( - *llvm::vfs::getRealFileSystem(), + *params.vfs, new clang::TextDiagnosticPrinter(llvm::outs(), new clang::DiagnosticOptions()), true); + instance->createFileManager(params.vfs); + adjustInvocation(instance->getInvocation()); return instance; @@ -123,7 +126,7 @@ llvm::Expected ExecuteAction(std::unique_ptr i } // namespace llvm::Expected compile(CompliationParams& params) { - auto instance = createInstance(params.args); + auto instance = createInstance(params); auto buffer = llvm::MemoryBuffer::getMemBufferCopy(params.content); instance->getPreprocessorOpts().addRemappedFile(params.path, buffer.release()); @@ -134,7 +137,7 @@ llvm::Expected compile(CompliationParams& params) { } llvm::Expected compile(CompliationParams& params, PCHInfo& out) { - auto instance = createInstance(params.args); + auto instance = createInstance(params); clang::PreambleBounds bounds = {0, false}; if(params.mainpath.empty() || params.mainpath == params.path) { @@ -172,7 +175,7 @@ llvm::Expected compile(CompliationParams& params, PCHInfo& out) { } llvm::Expected compile(CompliationParams& params, PCMInfo& out) { - auto instance = createInstance(params.args); + auto instance = createInstance(params); /// Set options to generate PCM. instance->getFrontendOpts().OutputFile = params.outpath.str(); @@ -194,7 +197,7 @@ llvm::Expected compile(CompliationParams& params, PCMInfo& out) { } llvm::Expected compile(CompliationParams& params, clang::CodeCompleteConsumer* consumer) { - auto instance = createInstance(params.args); + auto instance = createInstance(params); /// Set options to run code completion. instance->getFrontendOpts().CodeCompletionAt.FileName = params.path.str(); diff --git a/src/Compiler/Directive.cpp b/src/Compiler/Directive.cpp index 5abbc6d4..33a907f2 100644 --- a/src/Compiler/Directive.cpp +++ b/src/Compiler/Directive.cpp @@ -7,14 +7,17 @@ namespace clice { namespace { -struct PPCallback : public clang::PPCallbacks { - clang::Preprocessor& pp; - llvm::DenseMap& directives; - llvm::DenseMap macroCache; - +class PPCallback : public clang::PPCallbacks { +public: PPCallback(clang::Preprocessor& pp, llvm::DenseMap& directives) : pp(pp), directives(directives) {} +private: + void addInclude(llvm::StringRef path, clang::SourceLocation loc, clang::SourceRange range) { + auto& directive = directives[pp.getSourceManager().getFileID(loc)]; + directive.includes.emplace_back(Include{path, loc, range}); + } + void addCondition(clang::SourceLocation loc, Condition::BranchKind kind, Condition::ConditionValue value, @@ -58,27 +61,34 @@ struct PPCallback : public clang::PPCallbacks { } void addMacro(const clang::MacroInfo* def, MacroRef::Kind kind, clang::SourceLocation loc) { + if(pp.getSourceManager().isWrittenInBuiltinFile(loc) || + pp.getSourceManager().isWrittenInCommandLineFile(loc)) { + return; + } + auto& directive = directives[pp.getSourceManager().getFileID(loc)]; directive.macros.emplace_back(MacroRef{kind, loc, def}); } -public: +private: void FileChanged(clang::SourceLocation loc, clang::PPCallbacks::FileChangeReason reason, clang::SrcMgr::CharacteristicKind fileType, clang::FileID) override {} - void InclusionDirective(clang::SourceLocation HashLoc, - const clang::Token& IncludeTok, - llvm::StringRef FileName, - bool IsAngled, - clang::CharSourceRange FilenameRange, - clang::OptionalFileEntryRef File, - clang::StringRef SearchPath, - llvm::StringRef RelativePath, - const clang::Module* SuggestedModule, - bool ModuleImported, - clang::SrcMgr::CharacteristicKind FileType) override {} + void InclusionDirective(clang::SourceLocation hashLoc, + const clang::Token& includeTok, + llvm::StringRef filename, + bool isAngled, + clang::CharSourceRange filenameRange, + clang::OptionalFileEntryRef file, + llvm::StringRef searchPath, + llvm::StringRef relativePath, + const clang::Module* suggestedModule, + bool moduleImported, + clang::SrcMgr::CharacteristicKind fileType) override { + addInclude(filename, includeTok.getLocation(), filenameRange.getAsRange()); + } void PragmaDirective(clang::SourceLocation Loc, clang::PragmaIntroducerKind Introducer) override {} @@ -168,6 +178,11 @@ public: addMacro(def, MacroRef::Undef, name.getLocation()); } } + +private: + clang::Preprocessor& pp; + llvm::DenseMap& directives; + llvm::DenseMap macroCache; }; } // namespace diff --git a/tests/Compiler/test.cpp b/tests/Compiler/test.cpp index d7743cab..0a1f09af 100644 --- a/tests/Compiler/test.cpp +++ b/tests/Compiler/test.cpp @@ -2,20 +2,19 @@ #if name -#elif name + + +int x = 1; + +#elif [[$s]]name + +int x = 1; #else +int x = 1; + #endif #undef name -#define name 1 - -#if name - -#elif name - -#else - -#endif \ No newline at end of file diff --git a/tests/Index/test.cpp b/tests/Index/test.cpp index f6c7334b..7d7fe2f3 100644 --- a/tests/Index/test.cpp +++ b/tests/Index/test.cpp @@ -1,3 +1,15 @@ -#ifdef name +#define expr(v) v +#ifdef expr +int x = expr(1); #endif + +#undef expr + +#define expr(v) v + +#ifdef expr +int y = expr(expr(1)); +#endif + +#undef expr diff --git a/unittests/Compiler/Directive.cpp b/unittests/Compiler/Directive.cpp index 39d8f240..51b503f5 100644 --- a/unittests/Compiler/Directive.cpp +++ b/unittests/Compiler/Directive.cpp @@ -3,52 +3,128 @@ namespace clice { -TEST(Compiler, Directive) { +namespace { - const char* code = R"cpp( -#include +TEST(Directive, Include) { + const char* test = ""; -int main(){ - printf("Hello world"); - return 0; -} + const char* test2 = R"cpp( +#include "test.h" )cpp"; - foreachFile("Compiler", [](std::string name, llvm::StringRef content) { - llvm::SmallVector compileArgs = { - "clang++", - "-std=c++20", - name.c_str(), - "-resource-dir", - "/home/ykiko/C++/clice2/build/lib/clang/20", - }; + const char* main = R"cpp( +#$(0)include "test.h" +#$(1)include "test2.h" +#$(2)include "test3.h" +)cpp"; - CompliationParams params; - params.path = name.c_str(); - params.content = content; - params.args = compileArgs; + Tester tester("main.cpp", main); + tester.addFile("test.h", test); + tester.addFile("test2.h", test2); + tester.addFile("test3.h", ""); + tester.run(); - auto info = compile(params); - ASSERT_TRUE(bool(info)); + auto& info = tester.info; + auto& includes = info.directive(info.srcMgr().getMainFileID()).includes; - for(auto& [id, file]: info->directives()) { - for(auto& condition: file.conditions) { - print("kind: {}, value: {}, loc: {}, range: {}\n", - support::enum_name(condition.kind), - support::enum_name(condition.value), - condition.loc.printToString(info->srcMgr()), - condition.conditionRange.printToString(info->srcMgr())); - } - - for(auto& macro: file.macros) { - print("name: {}, kind: {}, loc: {}, def: {}\n", - info->getTokenSpelling(macro.loc), - support::enum_name(macro.kind), - macro.loc.printToString(info->srcMgr()), - static_cast(macro.macro)); - } - } - }); + tester.equal(includes.size(), 3) + .expect("0", includes[0].loc) + .equal("test.h", includes[0].path) + .expect("1", includes[1].loc) + .equal("test2.h", includes[1].path) + .expect("2", includes[2].loc) + .equal("test3.h", includes[2].path); } +TEST(Directive, Condition) { + const char* code = R"cpp( +#$(0)if 0 + +#$(1)elif 1 + +#$(2)else + +#$(3)endif + +#$(4)ifdef name + +#$(5)elifdef name + +#$(6)else + +#$(7)endif +)cpp"; + + Tester tester("main.cpp", code); + tester.run("-std=c++23"); + auto& info = tester.info; + auto& conditions = info.directive(info.srcMgr().getMainFileID()).conditions; + + tester.equal(conditions.size(), 8) + .equal(conditions[0].kind, Condition::BranchKind::If) + .expect("0", conditions[0].loc) + .equal(conditions[1].kind, Condition::BranchKind::Elif) + .expect("1", conditions[1].loc) + .equal(conditions[2].kind, Condition::BranchKind::Else) + .expect("2", conditions[2].loc) + .equal(conditions[3].kind, Condition::BranchKind::EndIf) + .expect("3", conditions[3].loc) + .equal(conditions[4].kind, Condition::BranchKind::Ifdef) + .expect("4", conditions[4].loc) + .equal(conditions[5].kind, Condition::BranchKind::Elifdef) + .expect("5", conditions[5].loc) + .equal(conditions[6].kind, Condition::BranchKind::Else) + .expect("6", conditions[6].loc) + .equal(conditions[7].kind, Condition::BranchKind::EndIf) + .expect("7", conditions[7].loc); +} + +TEST(Directive, Macro) { + const char* code = R"cpp( +#define $(0)expr(v) v + +#ifdef $(1)expr +int x = $(2)expr(1); +#endif + +#undef $(3)expr + +#define $(4)expr(v) v + +#ifdef $(5)expr +int y = $(6)expr($(7)expr(1)); +#endif + +#undef $(8)expr + +)cpp"; + + Tester tester("main.cpp", code); + tester.run(); + auto& info = tester.info; + auto& macros = info.directive(info.srcMgr().getMainFileID()).macros; + + tester.equal(macros.size(), 9) + .equal(macros[0].kind, MacroRef::Kind::Def) + .expect("0", macros[0].loc) + .equal(macros[1].kind, MacroRef::Kind::Ref) + .expect("1", macros[1].loc) + .equal(macros[2].kind, MacroRef::Kind::Ref) + .expect("2", macros[2].loc) + .equal(macros[3].kind, MacroRef::Kind::Undef) + .expect("3", macros[3].loc) + .equal(macros[4].kind, MacroRef::Kind::Def) + .expect("4", macros[4].loc) + .equal(macros[5].kind, MacroRef::Kind::Ref) + .expect("5", macros[5].loc) + .equal(macros[6].kind, MacroRef::Kind::Ref) + .expect("6", macros[6].loc) + .equal(macros[7].kind, MacroRef::Kind::Ref) + .expect("7", macros[7].loc) + .equal(macros[8].kind, MacroRef::Kind::Undef) + .expect("8", macros[8].loc); +} + +} // namespace + } // namespace clice diff --git a/unittests/Index/Annotation.h b/unittests/Index/Annotation.h deleted file mode 100644 index 4501dde7..00000000 --- a/unittests/Index/Annotation.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include -#include - -namespace clice { - - -class Annotation { -public: - Annotation(llvm::StringRef source) : m_source() { - m_source.reserve(source.size()); - - uint32_t line = 0; - uint32_t column = 0; - - for(uint32_t i = 0; i < source.size();) { - auto c = source[i]; - - if(c == '@') { - i += 1; - auto key = source.substr(i).take_until([](char c) { return c == ' '; }); - assert(!locations.contains(key) && "duplicate key"); - locations.try_emplace(key, line, column); - continue; - } - - if(c == '$') { - assert(i + 1 < source.size() && source[i + 1] == '(' && "expect $(name)"); - i += 2; - auto key = source.substr(i).take_until([](char c) { return c == ')'; }); - i += key.size() + 1; - assert(!locations.contains(key) && "duplicate key"); - locations.try_emplace(key, line, column); - continue; - } - - if(c == '\n') { - line += 1; - column = 0; - } else { - column += 1; - } - - i += 1; - m_source.push_back(c); - } - } - - llvm::StringRef source() const { - return m_source; - } - - proto::Position position(llvm::StringRef key) const { - return locations.lookup(key); - } - -private: - std::string m_source; - llvm::StringMap locations; -}; - -} // namespace clice diff --git a/unittests/Index/Tester.h b/unittests/Index/Tester.h index 65374557..90bcc6d0 100644 --- a/unittests/Index/Tester.h +++ b/unittests/Index/Tester.h @@ -1,7 +1,6 @@ #pragma once #include "../Test.h" -#include "Annotation.h" #include "Compiler/Compiler.h" #include "Support/Support.h" diff --git a/unittests/Test.h b/unittests/Test.h index b15ac3a8..9fe6abd2 100644 --- a/unittests/Test.h +++ b/unittests/Test.h @@ -1,18 +1,78 @@ #pragma once #include - -#include -#include -#include +#include +#include +#include namespace clice { -std::string test_dir(); + +namespace test { + +llvm::StringRef source_dir(); + +llvm::StringRef resource_dir(); + +} // namespace test + +class Annotation { +public: + Annotation(llvm::StringRef source) : m_source() { + m_source.reserve(source.size()); + + uint32_t line = 0; + uint32_t column = 0; + + for(uint32_t i = 0; i < source.size();) { + auto c = source[i]; + + if(c == '@') { + i += 1; + auto key = source.substr(i).take_until([](char c) { return c == ' '; }); + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '$') { + assert(i + 1 < source.size() && source[i + 1] == '(' && "expect $(name)"); + i += 2; + auto key = source.substr(i).take_until([](char c) { return c == ')'; }); + i += key.size() + 1; + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '\n') { + line += 1; + column = 0; + } else { + column += 1; + } + + i += 1; + m_source.push_back(c); + } + } + + llvm::StringRef source() const { + return m_source; + } + + proto::Position position(llvm::StringRef key) const { + return locations.lookup(key); + } + +private: + std::string m_source; + llvm::StringMap locations; +}; template inline void foreachFile(std::string name, const Callback& callback) { llvm::SmallString<128> path; - path += test_dir(); + path += test::source_dir(); path::append(path, name); std::error_code error; fs::directory_iterator iter(path, error); @@ -30,5 +90,119 @@ inline void foreachFile(std::string name, const Callback& callback) { } } +class Tester { +public: + CompliationParams params; + std::unique_ptr vfs; + ASTInfo info; + + /// Annoated locations. + std::vector sources; + llvm::StringMap locations; + +public: + Tester(llvm::StringRef file, llvm::StringRef content) { + params.path = file; + params.content = annoate(content); + vfs = std::make_unique(); + } + + void addFile(llvm::StringRef name, llvm::StringRef content) { + vfs->addFile(name, 0, llvm::MemoryBuffer::getMemBufferCopy(annoate(content))); + } + + llvm::StringRef annoate(llvm::StringRef content) { + auto& source = sources.emplace_back(); + source.reserve(content.size()); + + uint32_t line = 0; + uint32_t column = 0; + for(uint32_t i = 0; i < content.size();) { + auto c = content[i]; + + if(c == '@') { + i += 1; + auto key = content.substr(i).take_until([](char c) { return c == ' '; }); + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '$') { + assert(i + 1 < content.size() && content[i + 1] == '(' && "expect $(name)"); + i += 2; + auto key = content.substr(i).take_until([](char c) { return c == ')'; }); + i += key.size() + 1; + assert(!locations.contains(key) && "duplicate key"); + locations.try_emplace(key, line, column); + continue; + } + + if(c == '\n') { + line += 1; + column = 0; + } else { + column += 1; + } + + i += 1; + source.push_back(c); + } + + return source; + } + + Tester& run(const char* standard = "-std=c++20") { + params.vfs = std::move(vfs); + + llvm::SmallVector args = { + "clang++", + standard, + params.path.c_str(), + "-resource-dir", + test::resource_dir().data(), + }; + params.args = args; + + auto info = compile(params); + if(!info) { + llvm::errs() << "Failed to build AST\n"; + std::terminate(); + } + + this->info = std::move(*info); + return *this; + } + + Tester& fail(const auto& lhs, const auto& rhs, std::source_location loc) { + auto msg = + std::format("expect: {}, actual: {}\n", json::serialize(lhs), json::serialize(rhs)); + ::testing::internal::AssertHelper(::testing ::TestPartResult ::kNonFatalFailure, + loc.file_name(), + loc.line(), + msg.c_str()) = ::testing ::Message(); + return *this; + } + + Tester& equal(const auto& lhs, + const auto& rhs, + std::source_location loc = std::source_location::current()) { + if(!support::equal(lhs, rhs)) { + return fail(lhs, rhs, loc); + } + return *this; + } + + Tester& expect(llvm::StringRef name, + clang::SourceLocation loc, + std::source_location current = std::source_location::current()) { + auto pos = locations.lookup(name); + auto presumed = info.srcMgr().getPresumedLoc(loc); + /// FIXME: + equal(pos, proto::Position{presumed.getLine() - 1, presumed.getColumn() - 1}, current); + return *this; + } +}; + } // namespace clice diff --git a/unittests/main.cpp b/unittests/main.cpp index 7500346f..6f5b2d60 100644 --- a/unittests/main.cpp +++ b/unittests/main.cpp @@ -1,22 +1,42 @@ -#include +#include "Test.h" #include namespace clice { -llvm::cl::opt test_dir_path("test-dir", - llvm::cl::desc("specify the test source directory path"), - llvm::cl::value_desc("path"), - llvm::cl::Required); +namespace cl { -std::string test_dir() { - return test_dir_path; +llvm::cl::opt test_dir("test-dir", + llvm::cl::desc("specify the test source directory path"), + llvm::cl::value_desc("path"), + llvm::cl::Required); + +llvm::SmallString<128> resource_dir; + +} // namespace cl + +namespace test { + +llvm::StringRef source_dir() { + return cl::test_dir.getValue(); } +llvm::StringRef resource_dir() { + return cl::resource_dir.c_str(); +} + +} // namespace test + } // namespace clice int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); llvm::cl::ParseCommandLineOptions(argc, argv, "clice test\n"); + + if(auto error = clice::fs::real_path(llvm::Twine(argv[0]) + "/../../lib/clang/20)", + clice::cl::resource_dir)) { + llvm::errs() << "Failed to get resource directory, because " << error.message() << '\n'; + } + return RUN_ALL_TESTS(); }