tigerbeetle-node 0.11.5 → 0.11.6

package/src/tigerbeetle/src/main.zig

@@ -45,18 +45,7 @@ pub fn main() !void {
 
     switch (parse_args) {
         .format => |*args| try Command.format(allocator, args.cluster, args.replica, args.path),
-        .start => |*args| try Command.start(
-            &arena,
-            args.addresses,
-            .{
-                // TODO Tune lsm_forest_node_count better.
-                .lsm_forest_node_count = 4096,
-                .cache_entries_accounts = args.cache_accounts,
-                .cache_entries_transfers = args.cache_transfers,
-                .cache_entries_posted = args.cache_transfers_posted,
-            },
-            args.path,
-        ),
+        .start => |*args| try Command.start(&arena, args),
         .version => |*args| try Command.version(allocator, args.verbose),
     }
 }
@@ -113,43 +102,51 @@ const Command = struct {
 
         var superblock = try SuperBlock.init(
             allocator,
-            &command.storage,
-            &command.message_pool,
+            .{
+                .storage = &command.storage,
+                .storage_size_limit = data_file_size_min,
+                .message_pool = &command.message_pool,
+            },
         );
         defer superblock.deinit(allocator);
 
         try vsr.format(Storage, allocator, cluster, replica, &command.storage, &superblock);
     }
 
-    pub fn start(
-        arena: *std.heap.ArenaAllocator,
-        addresses: []std.net.Address,
-        options: StateMachine.Options,
-        path: [:0]const u8,
-    ) !void {
+    pub fn start(arena: *std.heap.ArenaAllocator, args: *const cli.Command.Start) !void {
         var tracer_allocator = if (constants.tracer_backend == .tracy)
             tracer.TracerAllocator.init(arena.allocator())
         else
             arena;
 
+        // TODO Panic if the data file's size is larger that args.storage_size_limit.
+        // (Here or in Replica.open()?).
+
         const allocator = tracer_allocator.allocator();
 
         try tracer.init(allocator);
         defer tracer.deinit(allocator);
 
         var command: Command = undefined;
-        try command.init(allocator, path, false);
+        try command.init(allocator, args.path, false);
         defer command.deinit(allocator);
 
         var replica: Replica = undefined;
         replica.open(allocator, .{
-            .replica_count = @intCast(u8, addresses.len),
+            .replica_count = @intCast(u8, args.addresses.len),
+            .storage_size_limit = args.storage_size_limit,
             .storage = &command.storage,
             .message_pool = &command.message_pool,
             .time = .{},
-            .state_machine_options = options,
+            .state_machine_options = .{
+                // TODO Tune lsm_forest_node_count better.
+                .lsm_forest_node_count = 4096,
+                .cache_entries_accounts = args.cache_accounts,
+                .cache_entries_transfers = args.cache_transfers,
+                .cache_entries_posted = args.cache_transfers_posted,
+            },
             .message_bus_options = .{
-                .configuration = addresses,
+                .configuration = args.addresses,
                 .io = &command.io,
             },
         }) catch |err| switch (err) {
@@ -178,7 +175,7 @@ const Command = struct {
         log_main.info("{}: cluster={}: listening on {}", .{
             replica.replica,
             replica.cluster,
-            addresses[replica.replica],
+            args.addresses[replica.replica],
         });
 
         while (true) {

package/src/tigerbeetle/src/message_pool.zig

@@ -22,7 +22,8 @@ const message_size_max_padded = constants.message_size_max + constants.sector_si
 pub const messages_max_replica = messages_max: {
     var sum: usize = 0;
 
-    sum += constants.journal_iops_read_max + constants.journal_iops_write_max; // Journal I/O
+    sum += constants.journal_iops_read_max; // Journal reads
+    sum += constants.journal_iops_write_max; // Journal writes
     sum += constants.clients_max; // SuperBlock.client_table
     sum += 1; // Replica.loopback_queue
     sum += constants.pipeline_max; // Replica.pipeline

package/src/tigerbeetle/src/simulator.zig

@@ -17,9 +17,7 @@ const StateChecker = @import("test/state_checker.zig").StateChecker;
 const StorageChecker = @import("test/storage_checker.zig").StorageChecker;
 const PartitionMode = @import("test/packet_simulator.zig").PartitionMode;
 const MessageBus = @import("test/message_bus.zig").MessageBus;
-const auditor = @import("test/accounting/auditor.zig");
-const Workload = @import("test/accounting/workload.zig").WorkloadType(StateMachine);
-const Conductor = @import("test/conductor.zig").ConductorType(Client, MessageBus, StateMachine, Workload);
+const Conductor = @import("test/conductor.zig").ConductorType(Client, MessageBus, StateMachine);
 const IdPermutation = @import("test/id.zig").IdPermutation;
 
 /// The `log` namespace in this root file is required to implement our custom `log` function.
@@ -88,16 +86,17 @@ pub fn main() !void {
     const idle_on_probability = random.uintLessThan(u8, 20);
     const idle_off_probability = 10 + random.uintLessThan(u8, 10);
 
-    // TODO: When block recovery and state transfer are implemented, remove this flag to allow
-    // crashes to coexist with WAL wraps.
+    // The maximum number of transitions from calling `client.request()`, not including
+    // `register` messages.
     const requests_committed_max: usize = constants.journal_slot_count * 3;
 
     const cluster_options: ClusterOptions = .{
         .cluster = cluster_id,
         .replica_count = replica_count,
         .client_count = client_count,
-        // TODO Compute an upper-bound for this based on requests_committed_max.
-        .grid_size_max = 1024 * 1024 * 256,
+        .storage_size_limit = vsr.sector_floor(
+            constants.storage_size_max - random.uintLessThan(u64, constants.storage_size_max / 10),
+        ),
         .seed = random.int(u64),
         .on_change_state = on_replica_change_state,
         .on_compact = on_replica_compact,
@@ -132,11 +131,7 @@ pub fn main() !void {
             .write_latency_mean = 3 + random.uintLessThan(u16, 100),
             .read_fault_probability = random.uintLessThan(u8, 10),
             .write_fault_probability = random.uintLessThan(u8, 10),
-            // TODO Allow WAL faults on crash when replica_count=1 when redundant-header-repair
-            // is implemented after recovering with decision=fix. Otherwise we can end up with
-            // multiple crashes faulting first a redundant headers, then a prepare, upgrading
-            // a decision=fix to decision=vsr.
-            .crash_fault_probability = if (replica_count == 1) 0 else 80 + random.uintLessThan(u8, 21),
+            .crash_fault_probability = 80 + random.uintLessThan(u8, 21),
             .faulty_superblock = true,
         },
         .health_options = .{
@@ -145,62 +140,18 @@ pub fn main() !void {
             .restart_probability = 0.0001,
             .restart_stability = random.uintLessThan(u32, 1_000),
         },
-        .state_machine_options = .{
-            // TODO What should these fields be set to? Can they be randomized (and with what constraints)?
-            .lsm_forest_node_count = 4096,
-            .cache_entries_accounts = 2048,
-            .cache_entries_transfers = 2048,
-            .cache_entries_posted = 2048,
+        // TODO(dj) SimulatorType(StateMachine).init(state_machine_options)
+        .state_machine_options = switch (constants.state_machine) {
+            .testing => .{},
+            .accounting => .{
+                .lsm_forest_node_count = 4096,
+                .cache_entries_accounts = if (random.boolean()) 0 else 2048,
+                .cache_entries_transfers = 0,
+                .cache_entries_posted = if (random.boolean()) 0 else 2048,
+            },
         },
     };
 
-    const workload_options: Workload.Options = .{
-        .auditor_options = .{
-            .accounts_max = 2 + random.uintLessThan(usize, 128),
-            .account_id_permutation = random_id_permutation(random),
-            .client_count = client_count,
-            .transfers_pending_max = 256,
-            .in_flight_max = Conductor.stalled_queue_capacity,
-        },
-        .transfer_id_permutation = random_id_permutation(random),
-        .operations = .{
-            .create_accounts = 1 + random.uintLessThan(usize, 10),
-            .create_transfers = 1 + random.uintLessThan(usize, 100),
-            .lookup_accounts = 1 + random.uintLessThan(usize, 20),
-            .lookup_transfers = 1 + random.uintLessThan(usize, 20),
-        },
-        .create_account_invalid_probability = 1,
-        .create_transfer_invalid_probability = 1,
-        .create_transfer_limit_probability = random.uintLessThan(u8, 101),
-        .create_transfer_pending_probability = 1 + random.uintLessThan(u8, 100),
-        .create_transfer_post_probability = 1 + random.uintLessThan(u8, 50),
-        .create_transfer_void_probability = 1 + random.uintLessThan(u8, 50),
-        .lookup_account_invalid_probability = 1,
-        .lookup_transfer = .{
-            .delivered = 1 + random.uintLessThan(usize, 10),
-            .sending = 1 + random.uintLessThan(usize, 10),
-        },
-        .lookup_transfer_span_mean = 10 + random.uintLessThan(usize, 1000),
-        .account_limit_probability = random.uintLessThan(u8, 80),
-        .linked_valid_probability = random.uintLessThan(u8, 101),
-        // 100% chance because this only applies to consecutive invalid transfers, which are rare.
-        .linked_invalid_probability = 100,
-        // TODO(Timeouts): When timeouts are implemented in the StateMachine, change this to the
-        // (commented out) value so that timeouts can actually trigger.
-        .pending_timeout_mean = std.math.maxInt(u64) / 2,
-        // .pending_timeout_mean = 1 + random.uintLessThan(usize, 1_000_000_000 / 4),
-        .accounts_batch_size_min = 0,
-        .accounts_batch_size_span = 1 + random.uintLessThan(
-            usize,
-            StateMachine.constants.batch_max.create_accounts,
-        ),
-        .transfers_batch_size_min = 0,
-        .transfers_batch_size_span = 1 + random.uintLessThan(
-            usize,
-            StateMachine.constants.batch_max.create_transfers,
-        ),
-    };
-
     output.info(
         \\
         \\          SEED={}
@@ -266,7 +217,12 @@ pub fn main() !void {
     cluster = try Cluster.create(allocator, random, cluster_options);
     defer cluster.destroy();
 
-    var workload = try Workload.init(allocator, random, workload_options);
+    const workload_options = StateMachine.Workload.Options.generate(random, .{
+        .client_count = client_count,
+        .in_flight_max = Conductor.stalled_queue_capacity,
+    });
+
+    var workload = try StateMachine.Workload.init(allocator, random, workload_options);
     defer workload.deinit(allocator);
 
     var conductor = try Conductor.init(allocator, random, &workload, .{
@@ -318,9 +274,6 @@ pub fn main() !void {
         storage.faulty = replica_normal_min <= i;
     }
 
-    // The maximum number of transitions from calling `client.request()`, not including
-    // `register` messages.
-    // TODO When storage is supported, run more transitions than fit in the journal.
     var tick: u64 = 0;
     while (tick < ticks_max) : (tick += 1) {
         const health_options = &cluster.options.health_options;
@@ -461,16 +414,6 @@ fn random_partition_mode(random: std.rand.Random) PartitionMode {
     return @intToEnum(PartitionMode, enumAsInt);
 }
 
-fn random_id_permutation(random: std.rand.Random) IdPermutation {
-    return switch (random.uintLessThan(usize, 4)) {
-        0 => .{ .identity = {} },
-        1 => .{ .inversion = {} },
-        2 => .{ .zigzag = {} },
-        3 => .{ .random = random.int(u64) },
-        else => unreachable,
-    };
-}
-
 pub fn parse_seed(bytes: []const u8) u64 {
     return std.fmt.parseUnsigned(u64, bytes, 10) catch |err| switch (err) {
         error.Overflow => @panic("seed exceeds a 64-bit unsigned integer"),

package/src/tigerbeetle/src/{test/accounting → state_machine}/auditor.zig

@@ -6,16 +6,16 @@ const std = @import("std");
 const assert = std.debug.assert;
 const log = std.log.scoped(.test_auditor);
 
-const constants = @import("../../constants.zig");
-const tb = @import("../../tigerbeetle.zig");
-const vsr = @import("../../vsr.zig");
-const RingBuffer = @import("../../ring_buffer.zig").RingBuffer;
-const IdPermutation = @import("../id.zig").IdPermutation;
+const constants = @import("../constants.zig");
+const tb = @import("../tigerbeetle.zig");
+const vsr = @import("../vsr.zig");
+const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
+const IdPermutation = @import("../test/id.zig").IdPermutation;
 
 // TODO(zig) This won't be necessary in Zig 0.10.
-const PriorityQueue = @import("../priority_queue.zig").PriorityQueue;
-const Storage = @import("../storage.zig").Storage;
-const StateMachine = @import("../../state_machine.zig").StateMachineType(Storage, .{
+const PriorityQueue = @import("../test/priority_queue.zig").PriorityQueue;
+const Storage = @import("../test/storage.zig").Storage;
+const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
     .message_body_size_max = constants.message_body_size_max,
 });
 

package/src/tigerbeetle/src/{test/accounting → state_machine}/workload.zig

@@ -21,18 +21,16 @@ const std = @import("std");
 const assert = std.debug.assert;
 const log = std.log.scoped(.test_workload);
 
-const constants = @import("../../constants.zig");
-const tb = @import("../../tigerbeetle.zig");
-const vsr = @import("../../vsr.zig");
-const accounting_auditor = @import("./auditor.zig");
+const constants = @import("../constants.zig");
+const tb = @import("../tigerbeetle.zig");
+const vsr = @import("../vsr.zig");
+const accounting_auditor = @import("auditor.zig");
 const Auditor = accounting_auditor.AccountingAuditor;
-const IdPermutation = @import("../id.zig").IdPermutation;
-const fuzz = @import("../fuzz.zig");
+const IdPermutation = @import("../test/id.zig").IdPermutation;
+const fuzz = @import("../test/fuzz.zig");
 
 // TODO(zig) This won't be necessary in Zig 0.10.
-const PriorityQueue = @import("../priority_queue.zig").PriorityQueue;
-
-// TODO Test linked create_accounts.
+const PriorityQueue = @import("../test/priority_queue.zig").PriorityQueue;
 
 const TransferOutcome = enum {
     /// The transfer is guaranteed to commit.
@@ -177,44 +175,7 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
     return struct {
         const Self = @This();
 
-        pub const Options = struct {
-            auditor_options: Auditor.Options,
-            transfer_id_permutation: IdPermutation,
-
-            operations: std.enums.EnumFieldStruct(Action, usize, null),
-
-            create_account_invalid_probability: u8, // ≤ 100
-            create_transfer_invalid_probability: u8, // ≤ 100
-            create_transfer_limit_probability: u8, // ≤ 100
-            create_transfer_pending_probability: u8, // ≤ 100
-            create_transfer_post_probability: u8, // ≤ 100
-            create_transfer_void_probability: u8, // ≤ 100
-            lookup_account_invalid_probability: u8, // ≤ 100
-
-            lookup_transfer: std.enums.EnumFieldStruct(enum {
-                /// Query a transfer that has either been committed or rejected.
-                delivered,
-                /// Query a transfer whose `create_transfers` is in-flight.
-                sending,
-            }, usize, null),
-
-            // Size of timespan for querying, measured in transfers
-            lookup_transfer_span_mean: usize,
-
-            account_limit_probability: u8, // ≤ 100
-
-            /// This probability is only checked for consecutive guaranteed-successful transfers.
-            linked_valid_probability: u8,
-            /// This probability is only checked for consecutive invalid transfers.
-            linked_invalid_probability: u8,
-
-            pending_timeout_mean: u64,
-
-            accounts_batch_size_min: usize,
-            accounts_batch_size_span: usize, // inclusive
-            transfers_batch_size_min: usize,
-            transfers_batch_size_span: usize, // inclusive
-        };
+        pub const Options = OptionsType(AccountingStateMachine, Action);
 
         random: std.rand.Random,
         auditor: Auditor,
@@ -303,7 +264,11 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
         }
 
         /// A client may build multiple requests to queue up while another is in-flight.
-        pub fn build_request(self: *Self, client_index: usize, body: []align(@alignOf(vsr.Header)) u8) struct {
+        pub fn build_request(
+            self: *Self,
+            client_index: usize,
+            body: []align(@alignOf(vsr.Header)) u8,
+        ) struct {
             operation: Operation,
             size: usize,
         } {
@@ -785,6 +750,101 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
     };
 }
 
+fn OptionsType(comptime StateMachine: type, comptime Action: type) type {
+    return struct {
+        const Options = @This();
+
+        auditor_options: Auditor.Options,
+        transfer_id_permutation: IdPermutation,
+
+        operations: std.enums.EnumFieldStruct(Action, usize, null),
+
+        create_account_invalid_probability: u8, // ≤ 100
+        create_transfer_invalid_probability: u8, // ≤ 100
+        create_transfer_limit_probability: u8, // ≤ 100
+        create_transfer_pending_probability: u8, // ≤ 100
+        create_transfer_post_probability: u8, // ≤ 100
+        create_transfer_void_probability: u8, // ≤ 100
+        lookup_account_invalid_probability: u8, // ≤ 100
+
+        lookup_transfer: std.enums.EnumFieldStruct(enum {
+            /// Query a transfer that has either been committed or rejected.
+            delivered,
+            /// Query a transfer whose `create_transfers` is in-flight.
+            sending,
+        }, usize, null),
+
+        // Size of timespan for querying, measured in transfers
+        lookup_transfer_span_mean: usize,
+
+        account_limit_probability: u8, // ≤ 100
+
+        /// This probability is only checked for consecutive guaranteed-successful transfers.
+        linked_valid_probability: u8,
+        /// This probability is only checked for consecutive invalid transfers.
+        linked_invalid_probability: u8,
+
+        pending_timeout_mean: u64,
+
+        accounts_batch_size_min: usize,
+        accounts_batch_size_span: usize, // inclusive
+        transfers_batch_size_min: usize,
+        transfers_batch_size_span: usize, // inclusive
+
+        pub fn generate(random: std.rand.Random, options: struct {
+            client_count: usize,
+            in_flight_max: usize,
+        }) Options {
+            return .{
+                .auditor_options = .{
+                    .accounts_max = 2 + random.uintLessThan(usize, 128),
+                    .account_id_permutation = IdPermutation.generate(random),
+                    .client_count = options.client_count,
+                    .transfers_pending_max = 256,
+                    .in_flight_max = options.in_flight_max,
+                },
+                .transfer_id_permutation = IdPermutation.generate(random),
+                .operations = .{
+                    .create_accounts = 1 + random.uintLessThan(usize, 10),
+                    .create_transfers = 1 + random.uintLessThan(usize, 100),
+                    .lookup_accounts = 1 + random.uintLessThan(usize, 20),
+                    .lookup_transfers = 1 + random.uintLessThan(usize, 20),
+                },
+                .create_account_invalid_probability = 1,
+                .create_transfer_invalid_probability = 1,
+                .create_transfer_limit_probability = random.uintLessThan(u8, 101),
+                .create_transfer_pending_probability = 1 + random.uintLessThan(u8, 100),
+                .create_transfer_post_probability = 1 + random.uintLessThan(u8, 50),
+                .create_transfer_void_probability = 1 + random.uintLessThan(u8, 50),
+                .lookup_account_invalid_probability = 1,
+                .lookup_transfer = .{
+                    .delivered = 1 + random.uintLessThan(usize, 10),
+                    .sending = 1 + random.uintLessThan(usize, 10),
+                },
+                .lookup_transfer_span_mean = 10 + random.uintLessThan(usize, 1000),
+                .account_limit_probability = random.uintLessThan(u8, 80),
+                .linked_valid_probability = random.uintLessThan(u8, 101),
+                // 100% chance because this only applies to consecutive invalid transfers, which are rare.
+                .linked_invalid_probability = 100,
+                // TODO(Timeouts): When timeouts are implemented in the StateMachine, change this to the
+                // (commented out) value so that timeouts can actually trigger.
+                .pending_timeout_mean = std.math.maxInt(u64) / 2,
+                // .pending_timeout_mean = 1 + random.uintLessThan(usize, 1_000_000_000 / 4),
+                .accounts_batch_size_min = 0,
+                .accounts_batch_size_span = 1 + random.uintLessThan(
+                    usize,
+                    StateMachine.constants.batch_max.create_accounts,
+                ),
+                .transfers_batch_size_min = 0,
+                .transfers_batch_size_span = 1 + random.uintLessThan(
+                    usize,
+                    StateMachine.constants.batch_max.create_transfers,
+                ),
+            };
+        }
+    };
+}
+
 /// Sample from a discrete distribution.
 /// Use integers instead of floating-point numbers to avoid nondeterminism on different hardware.
 fn sample_distribution(

package/src/tigerbeetle/src/state_machine.zig

@@ -8,6 +8,7 @@ const tracer = @import("tracer.zig");
 
 const tb = @import("tigerbeetle.zig");
 const snapshot_latest = @import("lsm/tree.zig").snapshot_latest;
+const WorkloadType = @import("state_machine/workload.zig").WorkloadType;
 
 const Account = tb.Account;
 const AccountFlags = tb.AccountFlags;
@@ -26,7 +27,6 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
 }) type {
     return struct {
         const StateMachine = @This();
-
         const Grid = @import("lsm/grid.zig").GridType(Storage);
         const GrooveType = @import("lsm/groove.zig").GrooveType;
         const ForestType = @import("lsm/forest.zig").ForestType;
@@ -49,6 +49,8 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
         );
         const PostedGroove = @import("lsm/posted_groove.zig").PostedGrooveType(Storage);
 
+        pub const Workload = WorkloadType(StateMachine);
+
         pub const Forest = ForestType(Storage, .{
             .accounts = AccountsGroove,
             .transfers = TransfersGroove,
@@ -180,13 +182,13 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
         /// Returns the header's timestamp.
         pub fn prepare(self: *StateMachine, operation: Operation, input: []u8) u64 {
             switch (operation) {
+                .reserved => unreachable,
                 .root => unreachable,
                 .register => {},
                 .create_accounts => self.prepare_timestamps(.create_accounts, input),
                 .create_transfers => self.prepare_timestamps(.create_transfers, input),
                 .lookup_accounts => {},
                 .lookup_transfers => {},
-                else => unreachable,
             }
             return self.prepare_timestamp;
         }
@@ -660,10 +662,7 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
             if (t.credit_account_id == t.debit_account_id) return .accounts_must_be_different;
 
             if (t.pending_id != 0) return .pending_id_must_be_zero;
-            if (t.flags.pending) {
-                // Otherwise, reserved amounts may never be released.
-                if (t.timeout == 0) return .pending_transfer_must_timeout;
-            } else {
+            if (!t.flags.pending) {
                 if (t.timeout != 0) return .timeout_reserved_for_pending_transfer;
             }
 
@@ -822,8 +821,9 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
             }
 
             assert(p.timestamp < t.timestamp);
-            assert(p.timeout > 0);
-            if (p.timestamp + p.timeout <= t.timestamp) return .pending_transfer_expired;
+            if (p.timeout > 0) {
+                if (p.timestamp + p.timeout <= t.timestamp) return .pending_transfer_expired;
+            }
 
             self.forest.grooves.transfers.put_no_clobber(&Transfer{
                 .id = t.id,
@@ -1113,6 +1113,7 @@ test "sum_overflows" {
 const TestContext = struct {
     const Storage = @import("test/storage.zig").Storage;
     const MessagePool = @import("message_pool.zig").MessagePool;
+    const data_file_size_min = @import("vsr/superblock.zig").data_file_size_min;
     const SuperBlock = @import("vsr/superblock.zig").SuperBlockType(Storage);
     const Grid = @import("lsm/grid.zig").GridType(Storage);
     const StateMachine = StateMachineType(Storage, .{
@@ -1147,7 +1148,11 @@ const TestContext = struct {
         };
         errdefer ctx.message_pool.deinit(allocator);
 
-        ctx.superblock = try SuperBlock.init(allocator, &ctx.storage, &ctx.message_pool);
+        ctx.superblock = try SuperBlock.init(allocator, .{
+            .storage = &ctx.storage,
+            .storage_size_limit = data_file_size_min,
+            .message_pool = &ctx.message_pool,
+        });
         errdefer ctx.superblock.deinit(allocator);
 
         // Pretend that the superblock is open so that the Forest can initialize.
@@ -1622,7 +1627,6 @@ test "create_transfers/lookup_transfers" {
         \\ transfer T1 A8 -0  _  _ T1   _ L0 C0 _ PEN _ _  _    0 credit_account_id_must_not_be_int_max
         \\ transfer T1 A8 A8  _  _ T1   _ L0 C0 _ PEN _ _  _    0 accounts_must_be_different
         \\ transfer T1 A8 A9  _  _ T1   _ L0 C0 _ PEN _ _  _    0 pending_id_must_be_zero
-        \\ transfer T1 A8 A9  _  _  _   _ L0 C0 _ PEN _ _  _    0 pending_transfer_must_timeout
         \\ transfer T1 A8 A9  _  _  _  -0 L0 C0 _   _ _ _  _    0 timeout_reserved_for_pending_transfer
         \\ transfer T1 A8 A9  _  _  _  -0 L0 C0 _ PEN _ _  _    0 ledger_must_not_be_zero
         \\ transfer T1 A8 A9  _  _  _  -0 L9 C0 _ PEN _ _  _    0 code_must_not_be_zero
@@ -1680,11 +1684,12 @@ test "create/lookup 2-phase transfers" {
         \\ transfer   T3 A1 A2  _ _   _   50 L1 C1 _ PEN   _   _ _ 15 ok
         \\ transfer   T4 A1 A2  _ _   _    1 L1 C1 _ PEN   _   _ _ 15 ok
         \\ transfer   T5 A1 A2 U9 _   _   50 L1 C1 _ PEN   _   _ _  7 ok
+        \\ transfer   T6 A1 A2  _ _   _    0 L1 C1 _ PEN   _   _ _  1 ok
         \\ commit create_transfers
 
         // Check balances before resolving.
-        \\ lookup_account A1 52 15  0  0
-        \\ lookup_account A2  0  0 52 15
+        \\ lookup_account A1 53 15  0  0
+        \\ lookup_account A2  0  0 53 15
         \\ commit lookup_accounts
 
         // Second phase.
@@ -1715,11 +1720,12 @@ test "create/lookup 2-phase transfers" {
         \\ transfer T102 A1 A2 U1 _  T3    _ L1 C1 _   _ POS   _ _ 13 pending_transfer_already_voided
         \\ transfer T102 A1 A2 U1 _  T4    _ L1 C1 _   _   _ VOI _ 15 pending_transfer_expired
         \\ transfer T105 A0 A0 U0 _  T5    _ L0 C0 _   _ POS   _ _  _ ok
+        \\ transfer T106 A0 A0 U0 _  T6    _ L1 C1 _   _ POS   _ _  0 ok
         \\ commit create_transfers
 
         // Check balances after resolving.
-        \\ lookup_account A1 15 35  0  0
-        \\ lookup_account A2  0  0 15 35
+        \\ lookup_account A1 15 36  0  0
+        \\ lookup_account A2  0  0 15 36
         \\ commit lookup_accounts
     );
 }

package/src/tigerbeetle/src/test/cluster.zig

@@ -11,7 +11,7 @@ const Message = MessagePool.Message;
 const Network = @import("network.zig").Network;
 const NetworkOptions = @import("network.zig").NetworkOptions;
 
-pub const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
+pub const StateMachine = constants.StateMachineType(Storage, .{
     .message_body_size_max = constants.message_body_size_max,
 });
 const MessageBus = @import("message_bus.zig").MessageBus;
@@ -29,7 +29,7 @@ pub const ClusterOptions = struct {
     cluster: u32,
     replica_count: u8,
     client_count: u8,
-    grid_size_max: usize,
+    storage_size_limit: u64,
 
     seed: u64,
     on_change_state: fn (replica: *const Replica) void,
@@ -76,8 +76,8 @@ pub const Cluster = struct {
     pub fn create(allocator: mem.Allocator, prng: std.rand.Random, options: ClusterOptions) !*Cluster {
         assert(options.replica_count > 0);
         assert(options.client_count > 0);
-        assert(options.grid_size_max > 0);
-        assert(options.grid_size_max % constants.sector_size == 0);
+        assert(options.storage_size_limit % constants.sector_size == 0);
+        assert(options.storage_size_limit <= constants.storage_size_max);
         assert(options.health_options.crash_probability < 1.0);
         assert(options.health_options.crash_probability >= 0.0);
         assert(options.health_options.restart_probability < 1.0);
@@ -136,17 +136,17 @@ pub const Cluster = struct {
             var storage_options = options.storage_options;
             storage_options.replica_index = @intCast(u8, replica_index);
             storage_options.faulty_wal_areas = faulty_wal_areas[replica_index];
-            storage.* = try Storage.init(
-                allocator,
-                superblock_zone_size + constants.journal_size_max + options.grid_size_max,
-                storage_options,
-            );
+            storage.* = try Storage.init(allocator, options.storage_size_limit, storage_options);
         }
         errdefer for (cluster.storages) |*storage| storage.deinit(allocator);
 
         // Format each replica's storage (equivalent to "tigerbeetle format ...").
         for (cluster.storages) |*storage, replica_index| {
-            var superblock = try SuperBlock.init(allocator, storage, &cluster.pools[replica_index]);
+            var superblock = try SuperBlock.init(allocator, .{
+                .storage = storage,
+                .message_pool = &cluster.pools[replica_index],
+                .storage_size_limit = options.storage_size_limit,
+            });
             defer superblock.deinit(allocator);
 
             try vsr.format(
@@ -330,6 +330,7 @@ pub const Cluster = struct {
             .{
                 .replica_count = @intCast(u8, cluster.replicas.len),
                 .storage = &cluster.storages[replica_index],
+                .storage_size_limit = cluster.options.storage_size_limit,
                 .message_pool = &cluster.pools[replica_index],
                 .time = time,
                 .state_machine_options = cluster.options.state_machine_options,
@@ -339,7 +340,6 @@ pub const Cluster = struct {
         assert(replica.cluster == cluster.options.cluster);
         assert(replica.replica == replica_index);
         assert(replica.replica_count == cluster.replicas.len);
-        assert(replica.status == .recovering);
 
         replica.on_change_state = cluster.options.on_change_state;
         replica.on_compact = cluster.options.on_compact;