tigerbeetle-node 0.9.0 → 0.10.0

package/src/tigerbeetle/src/ewah_benchmark.zig

@@ -0,0 +1,121 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const ewah = @import("ewah.zig").ewah(usize);
+
+const BitSetConfig = struct {
+    words: usize,
+    run_length_e: usize,
+    literals_length_e: usize,
+};
+
+const samples = 100;
+const repeats: usize = 100_000;
+
+// Explanation of fields:
+// - "n": Number of randomly generate bitsets to test.
+// - "words": The length of the decoded bitset, in u64s.
+// - "run_length_e": The expected length of a run, ignoring truncation due to reaching the end of
+//   the bitset.
+// - "literals_length_e": Expected length of a sequence of literals.
+const configs = [_]BitSetConfig{
+    // primarily runs
+    .{ .words = 640, .run_length_e = 10, .literals_length_e = 10 },
+    .{ .words = 640, .run_length_e = 100, .literals_length_e = 10 },
+    .{ .words = 640, .run_length_e = 200, .literals_length_e = 10 },
+    // primarily literals
+    .{ .words = 640, .run_length_e = 1, .literals_length_e = 100 },
+};
+
+var prng = std.rand.DefaultPrng.init(42);
+
+pub fn main() !void {
+    const stdout = std.io.getStdOut().writer();
+
+    for (configs) |config| {
+        var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+        defer arena.deinit();
+
+        var i: usize = 0;
+        var bitsets: [samples][]usize = undefined;
+        var bitsets_encoded: [samples][]align(@alignOf(usize)) u8 = undefined;
+        var bitsets_decoded: [samples][]usize = undefined;
+        var bitset_lengths: [samples]usize = undefined;
+        while (i < samples) : (i += 1) {
+            bitsets[i] = try make_bitset(&arena.allocator, config);
+            bitsets_encoded[i] = try arena.allocator.alignedAlloc(u8, @alignOf(usize), ewah.encode_size_max(bitsets[0].len));
+            bitsets_decoded[i] = try arena.allocator.alloc(usize, config.words);
+        }
+
+        // Benchmark encoding.
+        const encode_timer = try std.time.Timer.start();
+        i = 0;
+        while (i < samples) : (i += 1) {
+            var j: usize = 0;
+            var size: usize = undefined;
+            while (j < repeats) : (j += 1) {
+                size = ewah.encode(bitsets[i], bitsets_encoded[i]);
+            }
+            bitset_lengths[i] = size;
+        }
+        const encode_time = encode_timer.read() / samples / repeats;
+
+        const decode_timer = try std.time.Timer.start();
+        // Benchmark decoding.
+        i = 0;
+        while (i < samples) : (i += 1) {
+            const bitset_encoded = bitsets_encoded[i][0..bitset_lengths[i]];
+            var j: usize = 0;
+            while (j < repeats) : (j += 1) {
+                _ = ewah.decode(bitset_encoded, bitsets_decoded[i]);
+            }
+        }
+        const decode_time = decode_timer.read() / samples / repeats;
+
+        i = 0;
+        while (i < samples) : (i += 1) {
+            assert(std.mem.eql(usize, bitsets[i], bitsets_decoded[i]));
+        }
+
+        // Compute compression ratio.
+        var total_uncompressed: f64 = 0.0;
+        var total_compressed: f64 = 0.0;
+        i = 0;
+        while (i < samples) : (i += 1) {
+            total_uncompressed += @intToFloat(f64, bitsets[i].len * @sizeOf(usize));
+            total_compressed += @intToFloat(f64, bitset_lengths[i]);
+        }
+
+        try stdout.print("Words={:_>3} E(Run)={:_>3} E(Literal)={:_>3} EncTime={:_>6}ns DecTime={:_>6}ns Ratio={d:_>6.2}\n", .{
+            config.words,
+            config.run_length_e,
+            config.literals_length_e,
+            encode_time,
+            decode_time,
+            total_uncompressed / total_compressed,
+        });
+    }
+}
+
+fn make_bitset(allocator: *std.mem.Allocator, config: BitSetConfig) ![]usize {
+    var words = try allocator.alloc(usize, config.words);
+    var w: usize = 0;
+    var run: bool = true;
+    var literal: usize = 1;
+    while (w < words.len) : (w += 1) {
+        const start = w;
+        const run_length = prng.random.uintLessThan(usize, 2 * config.run_length_e);
+        const literals_length = prng.random.uintLessThan(usize, 2 * config.literals_length_e);
+        const run_bit = prng.random.boolean();
+
+        const run_end = std.math.min(w + run_length, words.len);
+        while (w < run_end) : (w += 1) {
+            words[w] = if (run_bit) std.math.maxInt(usize) else 0;
+        }
+        const literals_end = std.math.min(w + literals_length, words.len);
+        while (w < literals_end) : (w += 1) {
+            words[w] = literal;
+            literal += 1;
+        }
+    }
+    return words;
+}

package/src/tigerbeetle/src/eytzinger_benchmark.zig

@@ -0,0 +1,317 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const math = std.math;
+
+const binary_search = @import("./binary_search.zig").binary_search;
+const eytzinger = @import("./eytzinger.zig").eytzinger;
+const perf = @import("./benchmarks/perf.zig");
+
+const GiB = 1 << 30;
+const searches = 500_000;
+
+const kv_types = .{
+    .{ .key_size = 8, .value_size = 128 },
+    .{ .key_size = 8, .value_size = 64 },
+    .{ .key_size = 16, .value_size = 16 },
+    .{ .key_size = 32, .value_size = 32 },
+};
+
+// keys_per_summary = values_per_page / summary_fraction
+const summary_fractions = .{ 4, 8, 16, 32 };
+const values_per_page = .{ 128, 256, 512, 1024, 2048, 4096, 8192 };
+const body_fmt = "{:_>2}B/{:_>3}B {:_>4}/{:_>4} {s}{s}: WT={:_>6}ns UT={:_>6}ns" ++
+    " CY={:_>6} IN={:_>6} CR={:_>5} CM={:_>5} BM={}\n";
+
+const summary_sizes = blk: {
+    var sizes: [values_per_page.len][summary_fractions.len]usize = undefined;
+    for (values_per_page) |values_count, v| {
+        for (summary_fractions) |fraction, k| {
+            // Set in reverse order so that the summary sizes ascend.
+            sizes[v][summary_fractions.len - k - 1] = values_count / fraction;
+        }
+    }
+    break :blk sizes;
+};
+
+pub fn main() !void {
+    std.log.info("Samples: {}", .{searches});
+    std.log.info("WT: Wall time/search", .{});
+    std.log.info("UT: utime time/search", .{});
+    std.log.info("CY: CPU cycles/search", .{});
+    std.log.info("IN: instructions/search", .{});
+    std.log.info("CR: cache references/search", .{});
+    std.log.info("CM: cache misses/search", .{});
+    std.log.info("BM: branch misses/search", .{});
+
+    var seed: u64 = undefined;
+    try std.os.getrandom(std.mem.asBytes(&seed));
+    var prng = std.rand.DefaultPrng.init(seed);
+
+    // Allocate on the heap just once.
+    // All page allocations reuse this buffer to speed up the run time.
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+
+    const blob_size = GiB;
+    var blob = try arena.allocator.alloc(u8, blob_size);
+
+    inline for (kv_types) |kv| {
+        inline for (values_per_page) |values_count, v| {
+            inline for (summary_sizes[v]) |keys_count| {
+                try run_benchmark(.{
+                    .blob_size = blob_size,
+                    .key_size = kv.key_size,
+                    .value_size = kv.value_size,
+                    .keys_count = keys_count,
+                    .values_count = values_count,
+                    .searches = searches,
+                }, blob, &prng.random);
+            }
+        }
+    }
+}
+
+fn run_benchmark(comptime layout: Layout, blob: []u8, random: *std.rand.Random) !void {
+    assert(blob.len == layout.blob_size);
+    const Eytzinger = eytzinger(layout.keys_count - 1, layout.values_count);
+    const V = Value(layout);
+    const K = V.Key;
+    const Page = struct {
+        keys: [layout.keys_count]K,
+        values: [layout.values_count]V,
+    };
+    const page_count = layout.blob_size / @sizeOf(Page);
+
+    // Search pages and keys in random order.
+    var page_picker = shuffled_index(page_count, random);
+    var value_picker = shuffled_index(layout.values_count, random);
+
+    // Generate 1GiB worth of 24KiB pages.
+    var blob_alloc = std.heap.FixedBufferAllocator.init(blob);
+    var pages = try blob_alloc.allocator.alloc(Page, page_count);
+    random.bytes(std.mem.sliceAsBytes(pages));
+    for (pages) |*page| {
+        for (page.values) |*value, i| value.key = i;
+        Eytzinger.layout_from_keys_or_values(K, V, V.key_from_value, V.max_key, &page.values, &page.keys);
+    }
+
+    const stdout = std.io.getStdOut().writer();
+    {
+        var benchmark = try Benchmark.begin();
+        var i: usize = 0;
+        var v: usize = 0;
+        while (i < layout.searches) : (i += 1) {
+            const page_index = page_picker[i % page_picker.len];
+            const target = value_picker[v % value_picker.len];
+            const page = &pages[page_index];
+            const bounds = Eytzinger.search_values(K, V, V.key_compare, &page.keys, &page.values, target);
+            const hit = bounds[binary_search(K, V, V.key_from_value, V.key_compare, bounds, target)];
+
+            assert(hit.key == target);
+            if (i % pages.len == 0) v += 1;
+        }
+
+        const result = try benchmark.end(layout.searches);
+        try stdout.print(body_fmt, .{
+            layout.key_size,
+            layout.value_size,
+            layout.keys_count,
+            layout.values_count,
+            "E",
+            "B",
+            result.wall_time,
+            result.utime,
+            result.cpu_cycles,
+            result.instructions,
+            result.cache_references,
+            result.cache_misses,
+            result.branch_misses,
+        });
+    }
+
+    {
+        var benchmark = try Benchmark.begin();
+        var i: usize = 0;
+        var v: usize = 0;
+        while (i < layout.searches) : (i += 1) {
+            const target = value_picker[v % value_picker.len];
+            const page = &pages[page_picker[i % page_picker.len]];
+            const hit = page.values[binary_search(K, V, V.key_from_value, V.key_compare, page.values[0..], target)];
+
+            assert(hit.key == target);
+            if (i % pages.len == 0) v += 1;
+        }
+        const result = try benchmark.end(layout.searches);
+        try stdout.print(body_fmt, .{
+            layout.key_size,
+            layout.value_size,
+            layout.keys_count,
+            layout.values_count,
+            "_",
+            "B",
+            result.wall_time,
+            result.utime,
+            result.cpu_cycles,
+            result.instructions,
+            result.cache_references,
+            result.cache_misses,
+            result.branch_misses,
+        });
+    }
+}
+
+const Layout = struct {
+    blob_size: usize, // bytes allocated for all pages
+    key_size: usize, // bytes per key
+    value_size: usize, // bytes per value
+    keys_count: usize, // keys per page (in the summary)
+    values_count: usize, // values per page
+    searches: usize,
+};
+
+fn Value(comptime layout: Layout) type {
+    return struct {
+        pub const max_key = 1 << (8 * layout.key_size) - 1;
+        pub const Key = math.IntFittingRange(0, max_key);
+        const Self = @This();
+        key: Key,
+        body: [layout.value_size - layout.key_size]u8,
+
+        comptime {
+            assert(@sizeOf(Key) == layout.key_size);
+            assert(@sizeOf(Self) == layout.value_size);
+        }
+
+        inline fn key_from_value(self: Self) Key {
+            return self.key;
+        }
+
+        inline fn key_from_key(x: Key) Key {
+            return x;
+        }
+
+        inline fn key_compare(a: Key, b: Key) math.Order {
+            return math.order(a, b);
+        }
+    };
+}
+
+const BenchmarkResult = struct {
+    wall_time: u64, // nanoseconds
+    utime: u64, // nanoseconds
+    cpu_cycles: usize,
+    instructions: usize,
+    cache_references: usize,
+    cache_misses: usize,
+    branch_misses: usize,
+};
+
+const PERF = perf.PERF;
+const perf_event_attr = perf.perf_event_attr;
+const perf_event_open = perf.perf_event_open;
+const perf_counters = [_]PERF.COUNT.HW{
+    PERF.COUNT.HW.CPU_CYCLES,
+    PERF.COUNT.HW.INSTRUCTIONS,
+    PERF.COUNT.HW.CACHE_REFERENCES,
+    PERF.COUNT.HW.CACHE_MISSES,
+    PERF.COUNT.HW.BRANCH_MISSES,
+};
+
+const Benchmark = struct {
+    timer: std.time.Timer,
+    rusage: std.os.rusage,
+    perf_fds: [perf_counters.len]std.os.fd_t,
+
+    fn begin() !Benchmark {
+        const flags = PERF.FLAG.FD_NO_GROUP;
+        var perf_fds = [1]std.os.fd_t{-1} ** perf_counters.len;
+        for (perf_counters) |counter, i| {
+            var attr: perf_event_attr = .{
+                .type = PERF.TYPE.HARDWARE,
+                .config = @enumToInt(counter),
+                .flags = .{
+                    .disabled = true,
+                    .exclude_kernel = true,
+                    .exclude_hv = true,
+                },
+            };
+            perf_fds[i] = try perf_event_open(&attr, 0, -1, perf_fds[0], PERF.FLAG.FD_CLOEXEC);
+        }
+        const err = std.os.linux.ioctl(perf_fds[0], PERF.EVENT_IOC.ENABLE, PERF.IOC_FLAG_GROUP);
+        if (err == -1) return error.Unexpected;
+
+        // Start the wall clock after perf, since setup is slow.
+        const timer = try std.time.Timer.start();
+        return Benchmark{
+            .timer = timer,
+            // TODO pass std.os.linux.rusage.SELF once Zig is upgraded
+            .rusage = std.os.getrusage(0),
+            .perf_fds = perf_fds,
+        };
+    }
+
+    fn end(self: *Benchmark, samples: usize) !BenchmarkResult {
+        defer {
+            for (perf_counters) |_, i| {
+                std.os.close(self.perf_fds[i]);
+                self.perf_fds[i] = -1;
+            }
+        }
+
+        const rusage = std.os.getrusage(0);
+        const err = std.os.linux.ioctl(self.perf_fds[0], PERF.EVENT_IOC.DISABLE, PERF.IOC_FLAG_GROUP);
+        if (err == -1) return error.Unexpected;
+        return BenchmarkResult{
+            .wall_time = self.timer.read() / samples,
+            .utime = (timeval_to_ns(rusage.utime) - timeval_to_ns(self.rusage.utime)) / samples,
+            .cpu_cycles = (try readPerfFd(self.perf_fds[0])) / samples,
+            .instructions = (try readPerfFd(self.perf_fds[1])) / samples,
+            .cache_references = (try readPerfFd(self.perf_fds[2])) / samples,
+            .cache_misses = (try readPerfFd(self.perf_fds[3])) / samples,
+            .branch_misses = (try readPerfFd(self.perf_fds[4])) / samples,
+        };
+    }
+};
+
+// shuffle([0,1,…,n-1])
+fn shuffled_index(comptime n: usize, rand: *std.rand.Random) [n]usize {
+    var indices: [n]usize = undefined;
+    for (indices) |*i, j| i.* = j;
+    rand.shuffle(usize, indices[0..]);
+    return indices;
+}
+
+fn timeval_to_ns(tv: std.os.timeval) u64 {
+    const ns_per_us = std.time.ns_per_s / std.time.us_per_s;
+    return @bitCast(u64, tv.tv_sec) * std.time.ns_per_s +
+        @bitCast(u64, tv.tv_usec) * ns_per_us;
+}
+
+fn readPerfFd(fd: std.os.fd_t) !usize {
+    var result: usize = 0;
+    const n = try std.os.read(fd, std.mem.asBytes(&result));
+    assert(n == @sizeOf(usize));
+
+    return result;
+}
+
+fn binary_search_keys(
+    comptime layout: Layout,
+    comptime Key: type,
+    comptime V: type,
+    comptime compare_keys: fn (Key, Key) math.Order,
+    keys: []const Key,
+    values: []const V,
+    key: Key,
+) []const V {
+    assert(keys.len == layout.keys_count);
+    assert(values.len == layout.values_count);
+
+    const key_index = binary_search(Key, Key, V.key_from_key, compare_keys, keys, key);
+    const key_stride = layout.values_count / layout.keys_count;
+    const high = key_index * key_stride;
+    if (key_index < keys.len and keys[key_index] == key) {
+        return if (high == 0) values[0..1] else values[high - 1 .. high];
+    }
+    return values[high - key_stride .. high];
+}

package/src/tigerbeetle/src/fifo.zig

@@ -34,6 +34,10 @@ pub fn FIFO(comptime T: type) type {
             return self.out;
         }
 
+        pub fn empty(self: Self) bool {
+            return self.peek() == null;
+        }
+
         /// Remove an element from the FIFO. Asserts that the element is
         /// in the FIFO. This operation is O(N), if this is done often you
         /// probably want a different data structure.
@@ -55,7 +59,7 @@ pub fn FIFO(comptime T: type) type {
     };
 }
 
-test "push/pop/peek/remove" {
+test "push/pop/peek/remove/empty" {
     const testing = @import("std").testing;
 
     const Foo = struct { next: ?*@This() = null };
@@ -65,34 +69,45 @@ test "push/pop/peek/remove" {
     var three: Foo = .{};
 
     var fifo: FIFO(Foo) = .{};
+    try testing.expect(fifo.empty());
 
     fifo.push(&one);
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &one), fifo.peek());
 
     fifo.push(&two);
     fifo.push(&three);
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &one), fifo.peek());
 
     fifo.remove(&one);
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &two), fifo.pop());
     try testing.expectEqual(@as(?*Foo, &three), fifo.pop());
     try testing.expectEqual(@as(?*Foo, null), fifo.pop());
+    try testing.expect(fifo.empty());
 
     fifo.push(&one);
     fifo.push(&two);
     fifo.push(&three);
     fifo.remove(&two);
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &one), fifo.pop());
     try testing.expectEqual(@as(?*Foo, &three), fifo.pop());
     try testing.expectEqual(@as(?*Foo, null), fifo.pop());
+    try testing.expect(fifo.empty());
 
     fifo.push(&one);
     fifo.push(&two);
     fifo.push(&three);
     fifo.remove(&three);
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &one), fifo.pop());
+    try testing.expect(!fifo.empty());
     try testing.expectEqual(@as(?*Foo, &two), fifo.pop());
+    try testing.expect(fifo.empty());
     try testing.expectEqual(@as(?*Foo, null), fifo.pop());
+    try testing.expect(fifo.empty());
 
     fifo.push(&one);
     fifo.push(&two);
@@ -101,4 +116,5 @@ test "push/pop/peek/remove" {
     try testing.expectEqual(@as(?*Foo, &one), fifo.pop());
     try testing.expectEqual(@as(?*Foo, &three), fifo.pop());
     try testing.expectEqual(@as(?*Foo, null), fifo.pop());
+    try testing.expect(fifo.empty());
 }

package/src/tigerbeetle/src/io/darwin.zig

@@ -83,7 +83,7 @@ pub const IO = struct {
         const change_events = self.flush_io(&events, &io_pending);
 
         // Only call kevent() if we need to submit io events or if we need to wait for completions.
-        if (change_events > 0 or self.completed.peek() == null) {
+        if (change_events > 0 or self.completed.empty()) {
             // Zero timeouts for kevent() implies a non-blocking poll
             var ts = std.mem.zeroes(os.timespec);
 
@@ -91,7 +91,7 @@ pub const IO = struct {
             // We should never wait indefinitely (timeout_ptr = null for kevent) given:
             // - tick() is non-blocking (wait_for_completions = false)
             // - run_for_ns() always submits a timeout
-            if (change_events == 0 and self.completed.peek() == null) {
+            if (change_events == 0 and self.completed.empty()) {
                 if (wait_for_completions) {
                     const timeout_ns = next_timeout orelse @panic("kevent() blocking forever");
                     ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
@@ -430,6 +430,7 @@ pub const IO = struct {
         IsDir,
         SystemResources,
         Unseekable,
+        ConnectionTimedOut,
     } || os.UnexpectedError;
 
     pub fn read(
@@ -481,6 +482,7 @@ pub const IO = struct {
                             .NXIO => error.Unseekable,
                             .OVERFLOW => error.Unseekable,
                             .SPIPE => error.Unseekable,
+                            .TIMEDOUT => error.ConnectionTimedOut,
                             else => |err| os.unexpectedErrno(err),
                         };
                     }
@@ -635,10 +637,12 @@ pub const IO = struct {
     }
 
     /// Opens a directory with read only access.
-    pub fn open_dir(dir_path: [:0]const u8) !os.fd_t {
-        return os.openZ(dir_path, os.O.CLOEXEC | os.O.RDONLY, 0);
+    pub fn open_dir(dir_path: []const u8) !os.fd_t {
+        return os.open(dir_path, os.O.CLOEXEC | os.O.RDONLY, 0);
     }
 
+    pub const INVALID_FILE: os.fd_t = -1;
+
     /// Opens or creates a journal file:
     /// - For reading and writing.
     /// - For Direct I/O (required on darwin).
@@ -648,14 +652,11 @@ pub const IO = struct {
     ///   The caller is responsible for ensuring that the parent directory inode is durable.
     /// - Verifies that the file size matches the expected file size before returning.
     pub fn open_file(
-        self: *IO,
         dir_fd: os.fd_t,
-        relative_path: [:0]const u8,
+        relative_path: []const u8,
         size: u64,
         must_create: bool,
     ) !os.fd_t {
-        _ = self;
-
         assert(relative_path.len > 0);
         assert(size >= config.sector_size);
         assert(size % config.sector_size == 0);
@@ -685,7 +686,7 @@ pub const IO = struct {
 
         // Be careful with openat(2): "If pathname is absolute, then dirfd is ignored." (man page)
         assert(!std.fs.path.isAbsolute(relative_path));
-        const fd = try os.openatZ(dir_fd, relative_path, flags, mode);
+        const fd = try os.openat(dir_fd, relative_path, flags, mode);
         // TODO Return a proper error message when the path exists or does not exist (init/start).
         errdefer os.close(fd);
 
@@ -720,8 +721,9 @@ pub const IO = struct {
         // We always do this when opening because we don't know if this was done before crashing.
         try fs_sync(dir_fd);
 
+        // TODO Document that `size` is now `data_file_size_min` from `main.zig`.
         const stat = try os.fstat(fd);
-        if (stat.size != size) @panic("data file inode size was truncated or corrupted");
+        if (stat.size < size) @panic("data file inode size was truncated or corrupted");
 
         return fd;
     }