caio/1brc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commit 1.2

Browse Source
This commit is contained in:
caio
2026-09-09 00:52:00 -03:00
parent 572b5c7bd5
commit f30f25a91e
10 changed files with 436 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.clang-format
+8
View File
@@ -0,0 +1,8 @@
---
BasedOnStyle: Google
AccessModifierOffset: -4
ColumnLimit: 120
IndentWidth: 4
TabWidth: 4
UseTab: Always
source/CMakeLists.txt
+6
View File
@@ -1,9 +1,15 @@
add_executable(1brc
main.cxx
io.cxx
map.cxx
tricks.cxx
)
target_sources(1brc
PUBLIC
FILE_SET 1brc_modules TYPE CXX_MODULES FILES
io.ixx
map.ixx
tricks.ixx
# util/logger.ixx
)
source/io.cxx
+69
View File
@@ -0,0 +1,69 @@
module;
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
module io;
import std;
namespace brc {
std::string read_file(const std::filesystem::path& path) {
auto file = std::ifstream(path);
if (!file.is_open()) {
throw std::runtime_error(std::format("could not open file: {}", path.string()));
}
auto stream = std::stringstream();
stream << file.rdbuf();
return stream.str();
}
mapped_file::mapped_file(const std::filesystem::path& path) {
const auto file_size = std::filesystem::file_size(path);
fd = open(path.generic_string().data(), O_RDONLY);
data = mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
length = file_size;
if (data == MAP_FAILED) {
close(fd);
throw std::runtime_error("failed to map file");
}
}
mapped_file::mapped_file(mapped_file&& other) noexcept
: fd(other.fd), data(std::exchange(other.data, nullptr)), length(other.length) {}
mapped_file& mapped_file::operator=(mapped_file&& other) noexcept {
if (&other == this) {
return *this;
}
auto moved = std::move(other);
std::swap(moved.fd, this->fd);
std::swap(moved.data, this->data);
std::swap(moved.length, this->length);
return *this;
}
mapped_file::~mapped_file() {
if (data != nullptr) {
munmap(data, length);
close(fd);
}
}
mapped_file::operator std::string_view() const noexcept {
return std::string_view(reinterpret_cast<const char*>(data), length);
}
std::size_t mapped_file::size() const noexcept { return length; }
} // namespace brc
source/io.ixx
+31
View File
@@ -0,0 +1,31 @@
export module io;
import std;
namespace brc {
export [[nodiscard]] std::string read_file(const std::filesystem::path& path);
export class mapped_file {
private:
int fd;
void* data = nullptr;
std::size_t length = 0;
public:
explicit mapped_file(const std::filesystem::path& path);
mapped_file(const mapped_file&) = delete;
mapped_file& operator=(const mapped_file&) = delete;
mapped_file(mapped_file&& rhs) noexcept;
mapped_file& operator=(mapped_file&& rhs) noexcept;
~mapped_file();
operator std::string_view() const noexcept;
[[nodiscard]] std::size_t size() const noexcept;
};
} // namespace brc
source/liburing.ixx
+11
View File
@@ -0,0 +1,11 @@
module;
#include <liburing.h>
export module liburing;
namespace uring {
using ::io_uring;
}
source/main.cxx
+114 -1
View File
@@ -1,3 +1,116 @@
import std;
import io;
import map;
import tricks;
auto main() -> int { std::println("hello world"); }
auto calculate_boundaries(std::string_view file, std::size_t thread_count) -> std::inplace_vector<std::size_t, 64> {
auto bounds = std::inplace_vector<std::size_t, 64>();
const auto step_estimate = file.size() / thread_count;
while (bounds.size() < thread_count) {
auto at_guess = file.data() + (step_estimate * (1 + bounds.size()));
while (*at_guess++ != '\n') [[likely]] {
}
bounds.push_back(at_guess - file.data());
}
return bounds;
}
[[nodiscard]] auto parse_temp(std::string_view sv) {
constexpr auto dot_bit_mask = std::uint64_t(0x10101000ULL);
auto word = std::uint64_t();
std::memcpy(&word, sv.data(), sizeof(word));
const auto decimal_sep_pos = std::countr_zero(~word & dot_bit_mask);
const auto dot_byte = decimal_sep_pos / 8;
const auto shift = 28 - decimal_sep_pos;
// I don't trust auto here
std::int64_t is_neg = (static_cast<std::int64_t>(~word) << 59) >> 63;
std::uint64_t sign_mask = ~(static_cast<std::uint64_t>(is_neg) & 0xFF);
std::uint64_t digits = ((word & sign_mask) << shift) & 0xFFFFFFFFFFULL;
std::int64_t abs_value = ((digits * 0x640a0001ULL) >> 32) & 0x3FF;
std::int64_t value = (abs_value ^ is_neg) - is_neg;
return std::make_pair(static_cast<std::int32_t>(value), dot_byte + 2);
}
template <std::size_t BatchSize>
[[nodiscard]] std::inplace_vector<std::pair<std::int32_t, std::size_t>, BatchSize> parse_temp(
std::span<const std::string_view> batch) {
auto results = std::inplace_vector<std::pair<std::int32_t, std::size_t>, BatchSize>();
for (std::size_t i = 0; i < batch.size(); ++i) {
results.push_back(parse_temp(batch[i]));
}
return results;
}
void process_chunk(std::string_view chunk, brc::map& results) {
auto it = chunk.begin();
while (it != chunk.end()) {
const auto semicolon = brc::find_delim({it, chunk.end()}, ';');
const auto city = std::string_view(it, it + semicolon);
const auto [value, len] = parse_temp({it + semicolon + 1, chunk.end()});
results.insert(city, value);
it = it + semicolon + 1 + len + 1;
}
}
auto main(int argc, char** argv) -> int {
if (argc != 2) {
std::println("usage: {} <file.txt>", argv[0]);
return 1;
}
const auto filename = std::filesystem::path(argv[1]);
const auto file = brc::mapped_file(filename);
const auto view = std::string_view(file);
const auto thread_count = std::thread::hardware_concurrency() / 1;
const auto boundaries = calculate_boundaries(file, thread_count);
auto threads = std::vector<std::thread>();
auto results = std::vector<brc::map>();
for (std::size_t i = 0; i < thread_count; i++) {
results.emplace_back(1ull << 10ull);
}
for (std::size_t i = 0; i < thread_count; i++) {
const auto begin = i == 0 ? 0 : boundaries[i - 1];
const auto end = boundaries[i];
const auto sv = std::string_view(view.begin() + begin, view.begin() + end);
threads.emplace_back([i, sv, &results]() { process_chunk(sv, results[i]); });
}
for (std::size_t i = 0; i < thread_count; i++) {
threads[i].join();
}
for (std::size_t i = 1; i < results.size(); i++) {
results[0].merge_against(results[i]);
}
const auto& map = results[0];
const auto entries = map.collapse();
std::print("{{");
for (auto it = entries.begin(); it != entries.end(); it++) {
const auto entry = *it;
const auto name = std::string_view(entry.key_ptr, entry.key_len);
const auto min = entry.min / 10.0f;
const auto mean = (entry.total / entry.count) / 10.0f;
const auto max = entry.max / 10.0f;
std::print("{}={:.1f}/{:.1f}/{:.1f}", name, min, mean, max);
if (it + 1 != entries.end()) {
std::print(",");
}
}
std::print("}}");
}
source/map.cxx
+84
View File
@@ -0,0 +1,84 @@
module map;
import std;
namespace brc {
map::map(std::size_t entry_count) : entries(std::make_unique<entry[]>(entry_count)), entry_count(entry_count) {}
std::uint64_t map::get_key(std::string_view city) noexcept {
std::uint64_t raw = 0;
const auto data_count = std::min(static_cast<int>(city.size()), 8);
std::memcpy(&raw, city.data(), data_count);
raw *= 0x9E3779B97F4A7C15ULL;
return raw ^ (raw >> 32);
}
std::uint64_t map::get_index(std::string_view city) const noexcept {
const auto key = get_key(city);
auto index = key & (entry_count - 1);
while (true) {
auto& e = entries[index];
if (e.key_ptr == nullptr) {
e.key_hash = key;
e.key_ptr = city.data();
e.key_len = static_cast<std::uint16_t>(city.size());
break;
} else if (key == e.key_hash && city == std::string_view(e.key_ptr, e.key_len)) {
break;
}
index = (index + 1 == entry_count) ? 0 : index + 1;
}
return index;
}
void map::insert(std::string_view city, std::int32_t measurement) {
const auto index = get_index(city);
auto& entry = entries[index];
entry.count += 1;
entry.total += measurement;
entry.min = std::min(entry.min, measurement);
entry.max = std::max(entry.max, measurement);
}
void map::merge_against(const map& other) {
for (std::size_t i = 0; i < other.entry_count; i++) {
if (other.entries[i].key_ptr == nullptr) {
continue;
}
auto& other_entry = other.entries[i];
auto& our_entry = entries[get_index(std::string_view(other_entry.key_ptr, other_entry.key_len))];
our_entry.min = std::min(our_entry.min, other_entry.min);
our_entry.max = std::max(our_entry.max, other_entry.max);
our_entry.count += other_entry.count;
our_entry.total += other_entry.total;
}
}
std::vector<entry> map::collapse() const noexcept {
auto data = std::vector<entry>();
data.reserve(entry_count);
for (std::size_t i = 0; i < entry_count; i++) {
if (entries[i].key_ptr == nullptr) {
continue;
}
data.push_back(entries[i]);
}
std::sort(data.begin(), data.end(), [](const entry& a, const entry& b) {
const auto a_sv = std::string_view(a.key_ptr, a.key_len);
const auto b_sv = std::string_view(b.key_ptr, b.key_len);
return a_sv < b_sv;
});
return data;
}
} // namespace brc
source/map.ixx
+38
View File
@@ -0,0 +1,38 @@
export module map;
import std;
namespace brc {
export struct entry {
std::uint64_t key_hash = 0;
const char* key_ptr = nullptr; // 8, 8
std::int64_t total = 0; // 8, 26
std::int32_t min = std::numeric_limits<std::int32_t>::max(); // 4, 12
std::int32_t max = std::numeric_limits<std::int32_t>::min(); // 4, 16
std::uint32_t count = 0; // 4, 30
std::uint16_t key_len = 0; // 2, 32
};
export class map {
private:
std::size_t entry_count;
std::unique_ptr<entry[]> entries;
[[nodiscard]] static std::uint64_t get_key(std::string_view city) noexcept;
[[nodiscard]] std::uint64_t get_index(std::string_view city) const noexcept;
public:
explicit map(std::size_t entry_count);
void insert(std::string_view city, std::int32_t measurement);
[[nodiscard]] entry& operator[](std::string_view city);
void merge_against(const map& other);
[[nodiscard]] std::vector<entry> collapse() const noexcept;
};
} // namespace brc
source/tricks.cxx
+64
View File
@@ -0,0 +1,64 @@
module;
#include <simd>
module tricks;
import std;
namespace brc {
std::size_t find_delim(std::string_view string, char target) {
constexpr auto width = std::simd::vec<std::uint8_t>::size();
const auto needle = std::simd::vec<std::uint8_t>(static_cast<std::uint8_t>(target));
auto i = std::size_t(0);
for (; i + width <= string.size(); i += width) {
const auto chunk = std::simd::vec<std::uint8_t>(
std::span<const std::uint8_t, width>(reinterpret_cast<const std::uint8_t*>(string.data() + i), width));
const auto mask = chunk == needle;
if (const auto bits = mask.to_bitset(); bits.any()) {
return i + std::countr_zero(bits.to_ullong());
}
}
for (; i < string.size(); ++i) {
if (string[i] == target) return i;
}
return string.size();
}
std::inplace_vector<std::size_t, 4> find_delim_batch(std::string_view string, char target) {
constexpr auto BatchSize = 4;
auto sizes = std::inplace_vector<std::size_t, BatchSize>();
constexpr auto width = std::simd::vec<std::uint8_t>::size();
const auto needle = std::simd::vec<std::uint8_t>(static_cast<std::uint8_t>(target));
std::size_t i = 0;
for (; i + width <= string.size() && sizes.size() < BatchSize; i += width) {
const auto chunk = std::simd::vec<std::uint8_t>(
std::span<const std::uint8_t, width>(reinterpret_cast<const std::uint8_t*>(string.data() + i), width));
const auto mask = chunk == needle;
auto bits = mask.to_bitset().to_ullong();
while (bits != 0 && sizes.size() < BatchSize) {
const auto bit_pos = std::countr_zero(bits);
sizes.push_back(i + bit_pos);
bits &= bits - 1;
}
}
for (; i < string.size() && sizes.size() < BatchSize; ++i) {
if (string[i] == target) {
sizes.push_back(i);
}
}
return sizes;
}
} // namespace brc
source/tricks.ixx
+11
View File
@@ -0,0 +1,11 @@
export module tricks;
import std;
namespace brc {
export [[nodiscard]] std::size_t find_delim(std::string_view string, char target);
export [[nodiscard]] std::inplace_vector<std::size_t, 4> find_delim_batch(std::string_view string, char target);
} // namespace brc