diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..1744547 --- /dev/null +++ b/.clang-format @@ -0,0 +1,8 @@ +--- +BasedOnStyle: Google +AccessModifierOffset: -4 +ColumnLimit: 120 +IndentWidth: 4 +TabWidth: 4 +UseTab: Always + diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 1b00cee..23df486 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,9 +1,15 @@ add_executable(1brc main.cxx + io.cxx + map.cxx + tricks.cxx ) target_sources(1brc PUBLIC FILE_SET 1brc_modules TYPE CXX_MODULES FILES + io.ixx + map.ixx + tricks.ixx # util/logger.ixx ) diff --git a/source/io.cxx b/source/io.cxx new file mode 100644 index 0000000..8fbe262 --- /dev/null +++ b/source/io.cxx @@ -0,0 +1,69 @@ +module; + +#include +#include +#include +#include +#include +#include + +module io; + +import std; + +namespace brc { + +std::string read_file(const std::filesystem::path& path) { + auto file = std::ifstream(path); + if (!file.is_open()) { + throw std::runtime_error(std::format("could not open file: {}", path.string())); + } + auto stream = std::stringstream(); + stream << file.rdbuf(); + return stream.str(); +} + +mapped_file::mapped_file(const std::filesystem::path& path) { + const auto file_size = std::filesystem::file_size(path); + + fd = open(path.generic_string().data(), O_RDONLY); + data = mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + length = file_size; + + if (data == MAP_FAILED) { + close(fd); + throw std::runtime_error("failed to map file"); + } +} + +mapped_file::mapped_file(mapped_file&& other) noexcept + : fd(other.fd), data(std::exchange(other.data, nullptr)), length(other.length) {} + +mapped_file& mapped_file::operator=(mapped_file&& other) noexcept { + if (&other == this) { + return *this; + } + + auto moved = std::move(other); + + std::swap(moved.fd, this->fd); + std::swap(moved.data, this->data); + std::swap(moved.length, this->length); + + return *this; +} + +mapped_file::~mapped_file() { + if (data != nullptr) { + munmap(data, length); + close(fd); + } +} + +mapped_file::operator std::string_view() const noexcept { + return std::string_view(reinterpret_cast(data), length); +} + +std::size_t mapped_file::size() const noexcept { return length; } + +} // namespace brc diff --git a/source/io.ixx b/source/io.ixx new file mode 100644 index 0000000..d917e88 --- /dev/null +++ b/source/io.ixx @@ -0,0 +1,31 @@ +export module io; + +import std; + +namespace brc { + +export [[nodiscard]] std::string read_file(const std::filesystem::path& path); + +export class mapped_file { +private: + int fd; + void* data = nullptr; + std::size_t length = 0; + +public: + explicit mapped_file(const std::filesystem::path& path); + + mapped_file(const mapped_file&) = delete; + mapped_file& operator=(const mapped_file&) = delete; + + mapped_file(mapped_file&& rhs) noexcept; + mapped_file& operator=(mapped_file&& rhs) noexcept; + + ~mapped_file(); + + operator std::string_view() const noexcept; + + [[nodiscard]] std::size_t size() const noexcept; +}; + +} // namespace brc diff --git a/source/liburing.ixx b/source/liburing.ixx new file mode 100644 index 0000000..c1bfa41 --- /dev/null +++ b/source/liburing.ixx @@ -0,0 +1,11 @@ +module; + +#include + +export module liburing; + +namespace uring { + +using ::io_uring; + +} diff --git a/source/main.cxx b/source/main.cxx index 1593f68..7cbeed6 100644 --- a/source/main.cxx +++ b/source/main.cxx @@ -1,3 +1,116 @@ import std; +import io; +import map; +import tricks; -auto main() -> int { std::println("hello world"); } +auto calculate_boundaries(std::string_view file, std::size_t thread_count) -> std::inplace_vector { + auto bounds = std::inplace_vector(); + + const auto step_estimate = file.size() / thread_count; + while (bounds.size() < thread_count) { + auto at_guess = file.data() + (step_estimate * (1 + bounds.size())); + while (*at_guess++ != '\n') [[likely]] { + } + bounds.push_back(at_guess - file.data()); + } + + return bounds; +} + +[[nodiscard]] auto parse_temp(std::string_view sv) { + constexpr auto dot_bit_mask = std::uint64_t(0x10101000ULL); + + auto word = std::uint64_t(); + std::memcpy(&word, sv.data(), sizeof(word)); + + const auto decimal_sep_pos = std::countr_zero(~word & dot_bit_mask); + const auto dot_byte = decimal_sep_pos / 8; + const auto shift = 28 - decimal_sep_pos; + + // I don't trust auto here + std::int64_t is_neg = (static_cast(~word) << 59) >> 63; + std::uint64_t sign_mask = ~(static_cast(is_neg) & 0xFF); + std::uint64_t digits = ((word & sign_mask) << shift) & 0xFFFFFFFFFFULL; + + std::int64_t abs_value = ((digits * 0x640a0001ULL) >> 32) & 0x3FF; + std::int64_t value = (abs_value ^ is_neg) - is_neg; + + return std::make_pair(static_cast(value), dot_byte + 2); +} + +template +[[nodiscard]] std::inplace_vector, BatchSize> parse_temp( + std::span batch) { + auto results = std::inplace_vector, BatchSize>(); + + for (std::size_t i = 0; i < batch.size(); ++i) { + results.push_back(parse_temp(batch[i])); + } + + return results; +} + +void process_chunk(std::string_view chunk, brc::map& results) { + auto it = chunk.begin(); + while (it != chunk.end()) { + const auto semicolon = brc::find_delim({it, chunk.end()}, ';'); + const auto city = std::string_view(it, it + semicolon); + + const auto [value, len] = parse_temp({it + semicolon + 1, chunk.end()}); + + results.insert(city, value); + + it = it + semicolon + 1 + len + 1; + } +} + +auto main(int argc, char** argv) -> int { + if (argc != 2) { + std::println("usage: {} ", argv[0]); + return 1; + } + + const auto filename = std::filesystem::path(argv[1]); + const auto file = brc::mapped_file(filename); + const auto view = std::string_view(file); + const auto thread_count = std::thread::hardware_concurrency() / 1; + + const auto boundaries = calculate_boundaries(file, thread_count); + + auto threads = std::vector(); + auto results = std::vector(); + for (std::size_t i = 0; i < thread_count; i++) { + results.emplace_back(1ull << 10ull); + } + + for (std::size_t i = 0; i < thread_count; i++) { + const auto begin = i == 0 ? 0 : boundaries[i - 1]; + const auto end = boundaries[i]; + const auto sv = std::string_view(view.begin() + begin, view.begin() + end); + threads.emplace_back([i, sv, &results]() { process_chunk(sv, results[i]); }); + } + + for (std::size_t i = 0; i < thread_count; i++) { + threads[i].join(); + } + + for (std::size_t i = 1; i < results.size(); i++) { + results[0].merge_against(results[i]); + } + + const auto& map = results[0]; + const auto entries = map.collapse(); + std::print("{{"); + for (auto it = entries.begin(); it != entries.end(); it++) { + const auto entry = *it; + const auto name = std::string_view(entry.key_ptr, entry.key_len); + const auto min = entry.min / 10.0f; + const auto mean = (entry.total / entry.count) / 10.0f; + const auto max = entry.max / 10.0f; + std::print("{}={:.1f}/{:.1f}/{:.1f}", name, min, mean, max); + if (it + 1 != entries.end()) { + std::print(","); + } + } + std::print("}}"); +} diff --git a/source/map.cxx b/source/map.cxx new file mode 100644 index 0000000..445f1e1 --- /dev/null +++ b/source/map.cxx @@ -0,0 +1,84 @@ +module map; + +import std; + +namespace brc { + +map::map(std::size_t entry_count) : entries(std::make_unique(entry_count)), entry_count(entry_count) {} + +std::uint64_t map::get_key(std::string_view city) noexcept { + std::uint64_t raw = 0; + const auto data_count = std::min(static_cast(city.size()), 8); + std::memcpy(&raw, city.data(), data_count); + + raw *= 0x9E3779B97F4A7C15ULL; + return raw ^ (raw >> 32); +} + +std::uint64_t map::get_index(std::string_view city) const noexcept { + const auto key = get_key(city); + auto index = key & (entry_count - 1); + + while (true) { + auto& e = entries[index]; + if (e.key_ptr == nullptr) { + e.key_hash = key; + e.key_ptr = city.data(); + e.key_len = static_cast(city.size()); + break; + } else if (key == e.key_hash && city == std::string_view(e.key_ptr, e.key_len)) { + break; + } + index = (index + 1 == entry_count) ? 0 : index + 1; + } + + return index; +} + +void map::insert(std::string_view city, std::int32_t measurement) { + const auto index = get_index(city); + + auto& entry = entries[index]; + entry.count += 1; + entry.total += measurement; + entry.min = std::min(entry.min, measurement); + entry.max = std::max(entry.max, measurement); +} + +void map::merge_against(const map& other) { + for (std::size_t i = 0; i < other.entry_count; i++) { + if (other.entries[i].key_ptr == nullptr) { + continue; + } + + auto& other_entry = other.entries[i]; + auto& our_entry = entries[get_index(std::string_view(other_entry.key_ptr, other_entry.key_len))]; + our_entry.min = std::min(our_entry.min, other_entry.min); + our_entry.max = std::max(our_entry.max, other_entry.max); + our_entry.count += other_entry.count; + our_entry.total += other_entry.total; + } +} + +std::vector map::collapse() const noexcept { + auto data = std::vector(); + data.reserve(entry_count); + + for (std::size_t i = 0; i < entry_count; i++) { + if (entries[i].key_ptr == nullptr) { + continue; + } + + data.push_back(entries[i]); + } + + std::sort(data.begin(), data.end(), [](const entry& a, const entry& b) { + const auto a_sv = std::string_view(a.key_ptr, a.key_len); + const auto b_sv = std::string_view(b.key_ptr, b.key_len); + return a_sv < b_sv; + }); + + return data; +} + +} // namespace brc diff --git a/source/map.ixx b/source/map.ixx new file mode 100644 index 0000000..38e9f77 --- /dev/null +++ b/source/map.ixx @@ -0,0 +1,38 @@ +export module map; + +import std; + +namespace brc { + +export struct entry { + std::uint64_t key_hash = 0; + const char* key_ptr = nullptr; // 8, 8 + std::int64_t total = 0; // 8, 26 + std::int32_t min = std::numeric_limits::max(); // 4, 12 + std::int32_t max = std::numeric_limits::min(); // 4, 16 + std::uint32_t count = 0; // 4, 30 + std::uint16_t key_len = 0; // 2, 32 +}; + +export class map { +private: + std::size_t entry_count; + std::unique_ptr entries; + + [[nodiscard]] static std::uint64_t get_key(std::string_view city) noexcept; + + [[nodiscard]] std::uint64_t get_index(std::string_view city) const noexcept; + +public: + explicit map(std::size_t entry_count); + + void insert(std::string_view city, std::int32_t measurement); + + [[nodiscard]] entry& operator[](std::string_view city); + + void merge_against(const map& other); + + [[nodiscard]] std::vector collapse() const noexcept; +}; + +} // namespace brc diff --git a/source/tricks.cxx b/source/tricks.cxx new file mode 100644 index 0000000..14d0b2a --- /dev/null +++ b/source/tricks.cxx @@ -0,0 +1,64 @@ +module; + +#include + +module tricks; + +import std; + +namespace brc { + +std::size_t find_delim(std::string_view string, char target) { + constexpr auto width = std::simd::vec::size(); + const auto needle = std::simd::vec(static_cast(target)); + + auto i = std::size_t(0); + for (; i + width <= string.size(); i += width) { + const auto chunk = std::simd::vec( + std::span(reinterpret_cast(string.data() + i), width)); + + const auto mask = chunk == needle; + if (const auto bits = mask.to_bitset(); bits.any()) { + return i + std::countr_zero(bits.to_ullong()); + } + } + + for (; i < string.size(); ++i) { + if (string[i] == target) return i; + } + return string.size(); +} + +std::inplace_vector find_delim_batch(std::string_view string, char target) { + constexpr auto BatchSize = 4; + auto sizes = std::inplace_vector(); + + constexpr auto width = std::simd::vec::size(); + const auto needle = std::simd::vec(static_cast(target)); + + std::size_t i = 0; + + for (; i + width <= string.size() && sizes.size() < BatchSize; i += width) { + const auto chunk = std::simd::vec( + std::span(reinterpret_cast(string.data() + i), width)); + + const auto mask = chunk == needle; + auto bits = mask.to_bitset().to_ullong(); + + while (bits != 0 && sizes.size() < BatchSize) { + const auto bit_pos = std::countr_zero(bits); + sizes.push_back(i + bit_pos); + bits &= bits - 1; + } + } + + for (; i < string.size() && sizes.size() < BatchSize; ++i) { + if (string[i] == target) { + sizes.push_back(i); + } + } + + return sizes; +} + +} // namespace brc diff --git a/source/tricks.ixx b/source/tricks.ixx new file mode 100644 index 0000000..106b416 --- /dev/null +++ b/source/tricks.ixx @@ -0,0 +1,11 @@ +export module tricks; + +import std; + +namespace brc { + +export [[nodiscard]] std::size_t find_delim(std::string_view string, char target); + +export [[nodiscard]] std::inplace_vector find_delim_batch(std::string_view string, char target); + +} // namespace brc