tigerbeetle-node 0.11.5 → 0.11.7

package/src/tigerbeetle/src/test/cluster.zig

@@ -11,7 +11,7 @@ const Message = MessagePool.Message;
 const Network = @import("network.zig").Network;
 const NetworkOptions = @import("network.zig").NetworkOptions;
 
-pub const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
+pub const StateMachine = constants.StateMachineType(Storage, .{
     .message_body_size_max = constants.message_body_size_max,
 });
 const MessageBus = @import("message_bus.zig").MessageBus;
@@ -29,7 +29,7 @@ pub const ClusterOptions = struct {
     cluster: u32,
     replica_count: u8,
     client_count: u8,
-    grid_size_max: usize,
+    storage_size_limit: u64,
 
     seed: u64,
     on_change_state: fn (replica: *const Replica) void,
@@ -76,8 +76,8 @@ pub const Cluster = struct {
     pub fn create(allocator: mem.Allocator, prng: std.rand.Random, options: ClusterOptions) !*Cluster {
         assert(options.replica_count > 0);
         assert(options.client_count > 0);
-        assert(options.grid_size_max > 0);
-        assert(options.grid_size_max % constants.sector_size == 0);
+        assert(options.storage_size_limit % constants.sector_size == 0);
+        assert(options.storage_size_limit <= constants.storage_size_max);
         assert(options.health_options.crash_probability < 1.0);
         assert(options.health_options.crash_probability >= 0.0);
         assert(options.health_options.restart_probability < 1.0);
@@ -136,17 +136,19 @@ pub const Cluster = struct {
             var storage_options = options.storage_options;
             storage_options.replica_index = @intCast(u8, replica_index);
             storage_options.faulty_wal_areas = faulty_wal_areas[replica_index];
-            storage.* = try Storage.init(
-                allocator,
-                superblock_zone_size + constants.journal_size_max + options.grid_size_max,
-                storage_options,
-            );
+            storage.* = try Storage.init(allocator, options.storage_size_limit, storage_options);
+            // Disable most faults at startup, so that the replicas don't get stuck in recovery mode.
+            storage.faulty = replica_index >= vsr.quorums(options.replica_count).view_change;
         }
         errdefer for (cluster.storages) |*storage| storage.deinit(allocator);
 
         // Format each replica's storage (equivalent to "tigerbeetle format ...").
         for (cluster.storages) |*storage, replica_index| {
-            var superblock = try SuperBlock.init(allocator, storage, &cluster.pools[replica_index]);
+            var superblock = try SuperBlock.init(allocator, .{
+                .storage = storage,
+                .message_pool = &cluster.pools[replica_index],
+                .storage_size_limit = options.storage_size_limit,
+            });
             defer superblock.deinit(allocator);
 
             try vsr.format(
@@ -330,6 +332,8 @@ pub const Cluster = struct {
             .{
                 .replica_count = @intCast(u8, cluster.replicas.len),
                 .storage = &cluster.storages[replica_index],
+                // TODO Test restarting with a higher storage limit.
+                .storage_size_limit = cluster.options.storage_size_limit,
                 .message_pool = &cluster.pools[replica_index],
                 .time = time,
                 .state_machine_options = cluster.options.state_machine_options,
@@ -339,7 +343,6 @@ pub const Cluster = struct {
         assert(replica.cluster == cluster.options.cluster);
         assert(replica.replica == replica_index);
         assert(replica.replica_count == cluster.replicas.len);
-        assert(replica.status == .recovering);
 
         replica.on_change_state = cluster.options.on_change_state;
         replica.on_compact = cluster.options.on_compact;

package/src/tigerbeetle/src/test/conductor.zig

@@ -36,7 +36,6 @@ pub fn ConductorType(
     comptime Client: type,
     comptime MessageBus: type,
     comptime StateMachine: type,
-    comptime Workload: type,
 ) type {
     return struct {
         const Self = @This();
@@ -51,7 +50,7 @@ pub fn ConductorType(
             constants.clients_max * constants.client_request_queue_max * 2;
 
         random: std.rand.Random,
-        workload: *Workload,
+        workload: *StateMachine.Workload,
         options: Options,
         client_id_permutation: IdPermutation,
 
@@ -90,7 +89,7 @@ pub fn ConductorType(
         pub fn init(
             allocator: std.mem.Allocator,
             random: std.rand.Random,
-            workload: *Workload,
+            workload: *StateMachine.Workload,
             options: Options,
         ) !Self {
             assert(options.replica_count >= 1);

package/src/tigerbeetle/src/test/id.zig

@@ -62,6 +62,16 @@ pub const IdPermutation = union(enum) {
             .random => @truncate(usize, id >> 32),
         };
     }
+
+    pub fn generate(random: std.rand.Random) IdPermutation {
+        return switch (random.uintLessThan(usize, 4)) {
+            0 => .{ .identity = {} },
+            1 => .{ .inversion = {} },
+            2 => .{ .zigzag = {} },
+            3 => .{ .random = random.int(u64) },
+            else => unreachable,
+        };
+    }
 };
 
 test "IdPermutation" {

package/src/tigerbeetle/src/test/state_checker.zig

@@ -70,7 +70,7 @@ pub const StateChecker = struct {
     }
 
     pub fn check_state(state_checker: *StateChecker, replica_index: u8) !void {
-        const replica = state_checker.replicas[replica_index];
+        const replica = &state_checker.replicas[replica_index];
         const commit_header = header: {
             if (replica.journal.status == .recovered) {
                 const commit_header = replica.journal.header_with_op(replica.commit_min);

package/src/tigerbeetle/src/test/state_machine.zig

@@ -1,68 +1,65 @@
 const std = @import("std");
 const assert = std.debug.assert;
 
+const constants = @import("../constants.zig");
+const vsr = @import("../vsr.zig");
 const log = std.log.scoped(.state_machine);
 
-pub fn StateMachineType(comptime Storage: type) type {
+pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
+    message_body_size_max: usize,
+}) type {
     _ = Storage;
+    _ = constants_;
+
     return struct {
         const StateMachine = @This();
         const Grid = @import("../lsm/grid.zig").GridType(Storage);
 
+        pub const Workload = WorkloadType(StateMachine);
+
         pub const Operation = enum(u8) {
             /// Operations reserved by VR protocol (for all state machines):
             reserved,
             root,
             register,
 
-            random,
+            echo,
         };
 
-        /// Minimum/mean number of ticks to perform the specified operation.
-        /// Each mean must be greater-or-equal-to their respective minimum.
-        pub const Options = struct {
-            seed: u64,
-            prefetch_mean: u64,
-            compact_mean: u64,
-            checkpoint_mean: u64,
-        };
+        pub const Options = struct {};
 
         options: Options,
-        prng: std.rand.DefaultPrng,
+        grid: *Grid,
+        grid_block: Grid.BlockPtr,
+        grid_write: Grid.Write = undefined,
         prepare_timestamp: u64 = 0,
         commit_timestamp: u64 = 0,
 
         callback: ?fn (state_machine: *StateMachine) void = null,
-        callback_ticks: usize = 0,
 
-        pub fn init(_: std.mem.Allocator, _: *Grid, options: Options) !StateMachine {
+        pub fn init(allocator: std.mem.Allocator, grid: *Grid, options: Options) !StateMachine {
+            const grid_block = try allocator.alignedAlloc(
+                u8,
+                constants.sector_size,
+                constants.block_size,
+            );
+            errdefer allocator.free(grid_block);
+            std.mem.set(u8, grid_block, 0);
+
             return StateMachine{
                 .options = options,
-                .prng = std.rand.DefaultPrng.init(options.seed),
+                .grid = grid,
+                .grid_block = grid_block[0..constants.block_size],
             };
         }
 
-        pub fn deinit(_: *StateMachine, _: std.mem.Allocator) void {}
-
-        // TODO This is dead code — tick() has been removed from the StateMachine
-        // interface. If we start using the test StateMachine again, tick will need
-        // to be called explicitly from the simulator to ensure async operations can
-        // finish.
-        pub fn tick(state_machine: *StateMachine) void {
-            if (state_machine.callback) |callback| {
-                if (state_machine.callback_ticks == 0) {
-                    state_machine.callback = null;
-                    callback(state_machine);
-                } else {
-                    state_machine.callback_ticks -= 1;
-                }
-            }
+        pub fn deinit(state_machine: *StateMachine, allocator: std.mem.Allocator) void {
+            allocator.free(state_machine.grid_block);
         }
 
-        /// Don't add latency to `StateMachine.open`: the simulator calls it synchronously during
-        /// replica setup.
-        pub fn open(self: *StateMachine, callback: fn (*StateMachine) void) void {
-            callback(self);
+        // TODO Grid.next_tick
+        pub fn open(state_machine: *StateMachine, callback: fn (*StateMachine) void) void {
+            callback(state_machine);
         }
 
         pub fn prepare(
@@ -86,10 +83,8 @@ pub fn StateMachineType(comptime Storage: type) type {
             _ = op;
             _ = operation;
             _ = input;
-            assert(state_machine.callback == null);
-            assert(state_machine.callback_ticks == 0);
-            state_machine.callback = callback;
-            state_machine.callback_ticks = state_machine.latency(state_machine.options.prefetch_mean);
+
+            state_machine.next_tick(callback);
         }
 
         pub fn commit(
@@ -109,34 +104,144 @@ pub fn StateMachineType(comptime Storage: type) type {
             switch (operation) {
                 .reserved, .root => unreachable,
                 .register => return 0,
-                .random => return 0,
+                .echo => {
+                    std.mem.copy(u8, output, input);
+                    return input.len;
+                },
             }
         }
 
+        // TODO(Grid Recovery): Actually write blocks so that this state machine can be used
+        // to test grid recovery.
         pub fn compact(
             state_machine: *StateMachine,
             callback: fn (*StateMachine) void,
             op: u64,
         ) void {
             _ = op;
-            assert(state_machine.callback == null);
-            assert(state_machine.callback_ticks == 0);
-            state_machine.callback = callback;
-            state_machine.callback_ticks = state_machine.latency(state_machine.options.compact_mean);
+            state_machine.next_tick(callback);
         }
 
         pub fn checkpoint(
             state_machine: *StateMachine,
             callback: fn (*StateMachine) void,
         ) void {
+            state_machine.next_tick(callback);
+        }
+
+        // TODO Replace with Grid.next_tick()
+        fn next_tick(state_machine: *StateMachine, callback: fn (*StateMachine) void) void {
+            // TODO This is a hack to defer till the next tick; use Grid.next_tick instead.
+            var free_set = state_machine.grid.superblock.free_set;
+            const reservation = free_set.reserve(1).?;
+            defer free_set.forfeit(reservation);
+
+            const address = free_set.acquire(reservation).?;
+            const header = std.mem.bytesAsValue(
+                vsr.Header,
+                state_machine.grid_block[0..@sizeOf(vsr.Header)],
+            );
+            header.op = address;
+
             assert(state_machine.callback == null);
-            assert(state_machine.callback_ticks == 0);
             state_machine.callback = callback;
-            state_machine.callback_ticks = state_machine.latency(state_machine.options.checkpoint_mean);
+            state_machine.grid.write_block(
+                next_tick_callback,
+                &state_machine.grid_write,
+                state_machine.grid_block,
+                address,
+            );
         }
 
-        fn latency(state_machine: *StateMachine, mean: u64) u64 {
-            return state_machine.prng.random().uintLessThan(u64, 2 * mean);
+        fn next_tick_callback(write: *Grid.Write) void {
+            const state_machine = @fieldParentPtr(StateMachine, "grid_write", write);
+            const callback = state_machine.callback.?;
+            state_machine.callback = null;
+            callback(state_machine);
         }
     };
 }
+
+fn WorkloadType(comptime StateMachine: type) type {
+    return struct {
+        const Workload = @This();
+
+        random: std.rand.Random,
+        requests_sent: usize = 0,
+        requests_delivered: usize = 0,
+
+        pub fn init(
+            allocator: std.mem.Allocator,
+            random: std.rand.Random,
+            options: Options,
+        ) !Workload {
+            _ = allocator;
+            _ = options;
+
+            return Workload{
+                .random = random,
+            };
+        }
+
+        pub fn deinit(workload: *Workload, allocator: std.mem.Allocator) void {
+            _ = workload;
+            _ = allocator;
+        }
+
+        pub fn done(workload: *const Workload) bool {
+            return workload.requests_sent == workload.requests_delivered;
+        }
+
+        pub fn build_request(
+            workload: *Workload,
+            client_index: usize,
+            body: []align(@alignOf(vsr.Header)) u8,
+        ) struct {
+            operation: StateMachine.Operation,
+            size: usize,
+        } {
+            _ = client_index;
+
+            workload.requests_sent += 1;
+
+            // +1 for inclusive limit.
+            const size = workload.random.uintLessThan(usize, constants.message_body_size_max + 1);
+            workload.random.bytes(body[0..size]);
+
+            return .{
+                .operation = .echo,
+                .size = size,
+            };
+        }
+
+        pub fn on_reply(
+            workload: *Workload,
+            client_index: usize,
+            operation: vsr.Operation,
+            timestamp: u64,
+            request_body: []align(@alignOf(vsr.Header)) const u8,
+            reply_body: []align(@alignOf(vsr.Header)) const u8,
+        ) void {
+            _ = workload;
+            _ = client_index;
+            _ = timestamp;
+
+            workload.requests_delivered += 1;
+            assert(workload.requests_delivered <= workload.requests_sent);
+
+            assert(operation.cast(StateMachine) == .echo);
+            assert(std.mem.eql(u8, request_body, reply_body));
+        }
+
+        pub const Options = struct {
+            pub fn generate(random: std.rand.Random, options: struct {
+                client_count: usize,
+                in_flight_max: usize,
+            }) Options {
+                _ = random;
+                _ = options;
+                return .{};
+            }
+        };
+    };
+}

package/src/tigerbeetle/src/test/storage.zig

@@ -25,6 +25,7 @@ const assert = std.debug.assert;
 const math = std.math;
 const mem = std.mem;
 
+const FIFO = @import("../fifo.zig").FIFO;
 const constants = @import("../constants.zig");
 const vsr = @import("../vsr.zig");
 const superblock = @import("../vsr/superblock.zig");
@@ -119,6 +120,11 @@ pub const Storage = struct {
         }
     };
 
+    pub const NextTick = struct {
+        next: ?*NextTick = null,
+        callback: fn (next_tick: *NextTick) void,
+    };
+
     /// Faulty areas are always sized to message_size_max
     /// If the faulty areas of all replicas are superimposed, the padding between them is always message_size_max.
     /// For a single replica, the padding between faulty areas depends on the number of other replicas.
@@ -148,6 +154,7 @@ pub const Storage = struct {
     writes: std.PriorityQueue(*Storage.Write, void, Storage.Write.less_than),
 
     ticks: u64 = 0,
+    next_tick_queue: FIFO(NextTick) = .{},
 
     pub fn init(allocator: mem.Allocator, size: u64, options: Storage.Options) !Storage {
         assert(options.write_latency_mean >= options.write_latency_min);
@@ -222,6 +229,7 @@ pub const Storage = struct {
     }
 
     pub fn deinit(storage: *Storage, allocator: mem.Allocator) void {
+        assert(storage.next_tick_queue.empty());
         allocator.free(storage.memory);
         storage.memory_written.deinit(allocator);
         storage.faults.deinit(allocator);
@@ -280,6 +288,19 @@ pub const Storage = struct {
             _ = storage.writes.remove();
             storage.write_sectors_finish(write);
         }
+
+        var queue = storage.next_tick_queue;
+        storage.next_tick_queue = .{};
+        while (queue.pop()) |next_tick| next_tick.callback(next_tick);
+    }
+
+    pub fn on_next_tick(
+        storage: *Storage,
+        callback: fn (next_tick: *Storage.NextTick) void,
+        next_tick: *Storage.NextTick,
+    ) void {
+        next_tick.* = .{ .callback = callback };
+        storage.next_tick_queue.push(next_tick);
     }
 
     /// * Verifies that the read fits within the target sector.
@@ -787,7 +808,7 @@ pub const Storage = struct {
     pub fn grid_block(
         storage: *const Storage,
         address: u64,
-    ) *align(constants.sector_size) const [constants.block_size]u8 {
+    ) *align(constants.sector_size) [constants.block_size]u8 {
         assert(address > 0);
 
         const block_offset = vsr.Zone.grid.offset((address - 1) * constants.block_size);

package/src/tigerbeetle/src/tigerbeetle.zig

@@ -134,7 +134,6 @@ pub const CreateTransferResult = enum(u32) {
     accounts_must_be_different,
 
     pending_id_must_be_zero,
-    pending_transfer_must_timeout,
 
     ledger_must_not_be_zero,
     code_must_not_be_zero,

package/src/tigerbeetle/src/tracer.zig

@@ -73,15 +73,11 @@ pub const Event = union(enum) {
     state_machine_prefetch,
     state_machine_commit,
     state_machine_compact,
-    tree_compaction_beat: struct {
-        tree_name: []const u8,
-    },
+    tree_compaction_beat,
     tree_compaction_tick: struct {
-        tree_name: []const u8,
         level_b: u8,
     },
     tree_compaction_merge: struct {
-        tree_name: []const u8,
         level_b: u8,
     },
 
@@ -95,24 +91,50 @@ pub const Event = union(enum) {
         _ = options;
 
         switch (event) {
-            .tracer_flush => try writer.print("tracer_flush", .{}),
+            .tracer_flush,
+            .checkpoint,
+            .state_machine_prefetch,
+            .state_machine_commit,
+            .state_machine_compact,
+            .tree_compaction_beat,
+            => try writer.writeAll(@tagName(event)),
             .commit => |commit| try writer.print("commit({})", .{commit.op}),
-            .checkpoint => try writer.print("checkpoint", .{}),
-            .state_machine_prefetch => try writer.print("state_machine_prefetch", .{}),
-            .state_machine_commit => try writer.print("state_machine_commit", .{}),
-            .state_machine_compact => try writer.print("state_machine_compact", .{}),
-            .tree_compaction_beat => |tree_compaction_tick| try writer.print(
-                "tree_compaction_beat({s})",
-                .{tree_compaction_tick.tree_name},
-            ),
-            .tree_compaction_tick => |tree_compaction_tick| try writer.print(
-                "tree_compaction_tick({s}, {})",
-                .{ tree_compaction_tick.tree_name, tree_compaction_tick.level_b },
-            ),
-            .tree_compaction_merge => |tree_compaction_merge| try writer.print(
-                "tree_compaction_merge({s}, {})",
-                .{ tree_compaction_merge.tree_name, tree_compaction_merge.level_b },
-            ),
+            .tree_compaction_tick => |args| {
+                if (args.level_b == 0)
+                    try writer.print(
+                        "tree_compaction_tick({s}->{})",
+                        .{
+                            "immutable",
+                            args.level_b,
+                        },
+                    )
+                else
+                    try writer.print(
+                        "tree_compaction_tick({}->{})",
+                        .{
+                            args.level_b - 1,
+                            args.level_b,
+                        },
+                    );
+            },
+            .tree_compaction_merge => |args| {
+                if (args.level_b == 0)
+                    try writer.print(
+                        "tree_compaction_merge({s}->{})",
+                        .{
+                            "immutable",
+                            args.level_b,
+                        },
+                    )
+                else
+                    try writer.print(
+                        "tree_compaction_merge({}->{})",
+                        .{
+                            args.level_b - 1,
+                            args.level_b,
+                        },
+                    );
+            },
         }
     }
 };
@@ -124,12 +146,16 @@ pub const EventGroup = union(enum) {
     tree: struct {
         tree_name: [:0]const u8,
     },
+    tree_compaction: struct {
+        compaction_name: [:0]const u8,
+    },
 
     fn name(event_group: EventGroup) [:0]const u8 {
         return switch (event_group) {
             .main => "main",
             .tracer => "tracer",
-            .tree => |tree| tree.tree_name,
+            .tree => |args| args.tree_name,
+            .tree_compaction => |args| args.compaction_name,
         };
     }
 };
@@ -317,11 +343,7 @@ pub const TracerPerfetto = struct {
                 },
             };
 
-            const tid_64 = switch (span.group) {
-                .main => 0,
-                .tracer => 1,
-                .tree => |tree| std.hash_map.hashString(tree.tree_name),
-            };
+            const tid_64 = std.hash_map.hashString(span.group.name());
             const tid = @truncate(u32, tid_64) ^ @truncate(u32, tid_64 >> 32);
 
             var buffered_writer = std.io.bufferedWriter(log_file.writer());

package/src/tigerbeetle/src/unit_tests.zig

@@ -11,18 +11,18 @@ test {
     //_ = @import("vsr/clock.zig");
 
     _ = @import("state_machine.zig");
+    _ = @import("state_machine/auditor.zig");
+    _ = @import("state_machine/workload.zig");
 
     _ = @import("fifo.zig");
     _ = @import("ring_buffer.zig");
 
     _ = @import("io.zig");
-
-    _ = @import("cli.zig");
     _ = @import("ewah.zig");
     _ = @import("util.zig");
 
-    _ = @import("c/test.zig");
-    _ = @import("c/tb_client_header_test.zig");
+    _ = @import("clients/c/test.zig");
+    _ = @import("clients/c/tb_client_header_test.zig");
 
     // TODO Add remaining unit tests from lsm namespace.
     _ = @import("lsm/forest.zig");
@@ -30,8 +30,13 @@ test {
     _ = @import("lsm/segmented_array.zig");
 
     _ = @import("test/id.zig");
-    _ = @import("test/accounting/auditor.zig");
-    _ = @import("test/accounting/workload.zig");
     _ = @import("test/storage.zig");
     _ = @import("test/table.zig");
+
+    _ = @import("clients/go/go_bindings_test.zig");
+    _ = @import("clients/dotnet/dotnet_bindings_test.zig");
+    _ = @import("clients/java/java_bindings.zig");
+
+    // This one is a bit sketchy: we rely on tests not actually using the `vsr` package.
+    _ = @import("tigerbeetle/cli.zig");
 }

package/src/tigerbeetle/src/vopr.zig

@@ -138,14 +138,14 @@ fn build_simulator(
     mode: std.builtin.Mode,
 ) void {
     const mode_str = switch (mode) {
-        .Debug => "-ODebug",
-        .ReleaseSafe => "-OReleaseSafe",
+        .Debug => "-Ddebug",
+        .ReleaseSafe => "-Drelease-safe",
         else => unreachable,
     };
 
     const exec_result = std.ChildProcess.exec(.{
         .allocator = allocator,
-        .argv = &.{ "zig/zig", "build-exe", "./src/simulator.zig", mode_str },
+        .argv = &.{ "zig/zig", "build", "simulator", mode_str },
     }) catch |err| {
         fatal("unable to build the simulator binary. Error: {}", .{err});
     };
@@ -184,7 +184,7 @@ fn run_simulator(
     // simulator's exit code.
     const exit_code = run_child_process(
         allocator,
-        &.{ "./simulator", seed_str.items },
+        &.{ "./zig-out/bin/simulator", seed_str.items },
     );
 
     const result = switch (exit_code) {