Implement Indexer (#48)

include/Basic/SourceCode.h

@@ -1,9 +1,19 @@
 #pragma once
 
+#include "llvm/ADT/FunctionExtras.h"
+#include "clang/Lex/Token.h"
 #include "clang/Basic/SourceLocation.h"
 
 namespace clice {
 
+struct LocalSourceRange {
+    /// The begin position offset to the source file.
+    uint32_t begin;
+
+    /// The end position offset to the source file.
+    uint32_t end;
+};
+
 /// Get the content of the file with the given file ID.
 llvm::StringRef getFileContent(const clang::SourceManager& SM, clang::FileID fid);
 
@@ -16,4 +26,13 @@ std::uint32_t getTokenLength(const clang::SourceManager& SM, clang::SourceLocati
 /// Get the spelling of the token at the given location.
 llvm::StringRef getTokenSpelling(const clang::SourceManager& SM, clang::SourceLocation location);
 
+/// @brief Run `clang::Lexer` in raw mode and tokenize the content.
+/// @param content The content to tokenize.
+/// @param callback The callback to call for each token. Return false to break.
+/// @param langOpts The language options to use. If not provided, lastest C++ standard is used.
+void tokenize(llvm::StringRef content,
+              llvm::unique_function<bool(const clang::Token&)> callback,
+              bool ignoreComments = true,
+              const clang::LangOptions* langOpts = nullptr);
+
 }  // namespace clice