tigerbeetle-node 0.11.5 → 0.11.7

package/src/tigerbeetle/src/message_pool.zig

@@ -22,7 +22,8 @@ const message_size_max_padded = constants.message_size_max + constants.sector_si
 pub const messages_max_replica = messages_max: {
     var sum: usize = 0;
 
-    sum += constants.journal_iops_read_max + constants.journal_iops_write_max; // Journal I/O
+    sum += constants.journal_iops_read_max; // Journal reads
+    sum += constants.journal_iops_write_max; // Journal writes
     sum += constants.clients_max; // SuperBlock.client_table
     sum += 1; // Replica.loopback_queue
     sum += constants.pipeline_max; // Replica.pipeline

package/src/tigerbeetle/src/simulator.zig

@@ -17,9 +17,7 @@ const StateChecker = @import("test/state_checker.zig").StateChecker;
 const StorageChecker = @import("test/storage_checker.zig").StorageChecker;
 const PartitionMode = @import("test/packet_simulator.zig").PartitionMode;
 const MessageBus = @import("test/message_bus.zig").MessageBus;
-const auditor = @import("test/accounting/auditor.zig");
-const Workload = @import("test/accounting/workload.zig").WorkloadType(StateMachine);
-const Conductor = @import("test/conductor.zig").ConductorType(Client, MessageBus, StateMachine, Workload);
+const Conductor = @import("test/conductor.zig").ConductorType(Client, MessageBus, StateMachine);
 const IdPermutation = @import("test/id.zig").IdPermutation;
 
 /// The `log` namespace in this root file is required to implement our custom `log` function.
@@ -88,16 +86,17 @@ pub fn main() !void {
     const idle_on_probability = random.uintLessThan(u8, 20);
     const idle_off_probability = 10 + random.uintLessThan(u8, 10);
 
-    // TODO: When block recovery and state transfer are implemented, remove this flag to allow
-    // crashes to coexist with WAL wraps.
+    // The maximum number of transitions from calling `client.request()`, not including
+    // `register` messages.
     const requests_committed_max: usize = constants.journal_slot_count * 3;
 
     const cluster_options: ClusterOptions = .{
         .cluster = cluster_id,
         .replica_count = replica_count,
         .client_count = client_count,
-        // TODO Compute an upper-bound for this based on requests_committed_max.
-        .grid_size_max = 1024 * 1024 * 256,
+        .storage_size_limit = vsr.sector_floor(
+            constants.storage_size_max - random.uintLessThan(u64, constants.storage_size_max / 10),
+        ),
         .seed = random.int(u64),
         .on_change_state = on_replica_change_state,
         .on_compact = on_replica_compact,
@@ -132,11 +131,7 @@ pub fn main() !void {
             .write_latency_mean = 3 + random.uintLessThan(u16, 100),
             .read_fault_probability = random.uintLessThan(u8, 10),
             .write_fault_probability = random.uintLessThan(u8, 10),
-            // TODO Allow WAL faults on crash when replica_count=1 when redundant-header-repair
-            // is implemented after recovering with decision=fix. Otherwise we can end up with
-            // multiple crashes faulting first a redundant headers, then a prepare, upgrading
-            // a decision=fix to decision=vsr.
-            .crash_fault_probability = if (replica_count == 1) 0 else 80 + random.uintLessThan(u8, 21),
+            .crash_fault_probability = 80 + random.uintLessThan(u8, 21),
             .faulty_superblock = true,
         },
         .health_options = .{
@@ -145,60 +140,16 @@ pub fn main() !void {
             .restart_probability = 0.0001,
             .restart_stability = random.uintLessThan(u32, 1_000),
         },
-        .state_machine_options = .{
-            // TODO What should these fields be set to? Can they be randomized (and with what constraints)?
-            .lsm_forest_node_count = 4096,
-            .cache_entries_accounts = 2048,
-            .cache_entries_transfers = 2048,
-            .cache_entries_posted = 2048,
-        },
-    };
-
-    const workload_options: Workload.Options = .{
-        .auditor_options = .{
-            .accounts_max = 2 + random.uintLessThan(usize, 128),
-            .account_id_permutation = random_id_permutation(random),
-            .client_count = client_count,
-            .transfers_pending_max = 256,
-            .in_flight_max = Conductor.stalled_queue_capacity,
-        },
-        .transfer_id_permutation = random_id_permutation(random),
-        .operations = .{
-            .create_accounts = 1 + random.uintLessThan(usize, 10),
-            .create_transfers = 1 + random.uintLessThan(usize, 100),
-            .lookup_accounts = 1 + random.uintLessThan(usize, 20),
-            .lookup_transfers = 1 + random.uintLessThan(usize, 20),
-        },
-        .create_account_invalid_probability = 1,
-        .create_transfer_invalid_probability = 1,
-        .create_transfer_limit_probability = random.uintLessThan(u8, 101),
-        .create_transfer_pending_probability = 1 + random.uintLessThan(u8, 100),
-        .create_transfer_post_probability = 1 + random.uintLessThan(u8, 50),
-        .create_transfer_void_probability = 1 + random.uintLessThan(u8, 50),
-        .lookup_account_invalid_probability = 1,
-        .lookup_transfer = .{
-            .delivered = 1 + random.uintLessThan(usize, 10),
-            .sending = 1 + random.uintLessThan(usize, 10),
+        // TODO(dj) SimulatorType(StateMachine).init(state_machine_options)
+        .state_machine_options = switch (constants.state_machine) {
+            .testing => .{},
+            .accounting => .{
+                .lsm_forest_node_count = 4096,
+                .cache_entries_accounts = if (random.boolean()) 0 else 2048,
+                .cache_entries_transfers = 0,
+                .cache_entries_posted = if (random.boolean()) 0 else 2048,
+            },
         },
-        .lookup_transfer_span_mean = 10 + random.uintLessThan(usize, 1000),
-        .account_limit_probability = random.uintLessThan(u8, 80),
-        .linked_valid_probability = random.uintLessThan(u8, 101),
-        // 100% chance because this only applies to consecutive invalid transfers, which are rare.
-        .linked_invalid_probability = 100,
-        // TODO(Timeouts): When timeouts are implemented in the StateMachine, change this to the
-        // (commented out) value so that timeouts can actually trigger.
-        .pending_timeout_mean = std.math.maxInt(u64) / 2,
-        // .pending_timeout_mean = 1 + random.uintLessThan(usize, 1_000_000_000 / 4),
-        .accounts_batch_size_min = 0,
-        .accounts_batch_size_span = 1 + random.uintLessThan(
-            usize,
-            StateMachine.constants.batch_max.create_accounts,
-        ),
-        .transfers_batch_size_min = 0,
-        .transfers_batch_size_span = 1 + random.uintLessThan(
-            usize,
-            StateMachine.constants.batch_max.create_transfers,
-        ),
     };
 
     output.info(
@@ -266,7 +217,12 @@ pub fn main() !void {
     cluster = try Cluster.create(allocator, random, cluster_options);
     defer cluster.destroy();
 
-    var workload = try Workload.init(allocator, random, workload_options);
+    const workload_options = StateMachine.Workload.Options.generate(random, .{
+        .client_count = client_count,
+        .in_flight_max = Conductor.stalled_queue_capacity,
+    });
+
+    var workload = try StateMachine.Workload.init(allocator, random, workload_options);
     defer workload.deinit(allocator);
 
     var conductor = try Conductor.init(allocator, random, &workload, .{
@@ -313,14 +269,6 @@ pub fn main() !void {
         }
     };
 
-    // Disable most faults at startup, so that the replicas don't get stuck in recovery mode.
-    for (cluster.storages) |*storage, i| {
-        storage.faulty = replica_normal_min <= i;
-    }
-
-    // The maximum number of transitions from calling `client.request()`, not including
-    // `register` messages.
-    // TODO When storage is supported, run more transitions than fit in the journal.
     var tick: u64 = 0;
     while (tick < ticks_max) : (tick += 1) {
         const health_options = &cluster.options.health_options;
@@ -461,16 +409,6 @@ fn random_partition_mode(random: std.rand.Random) PartitionMode {
     return @intToEnum(PartitionMode, enumAsInt);
 }
 
-fn random_id_permutation(random: std.rand.Random) IdPermutation {
-    return switch (random.uintLessThan(usize, 4)) {
-        0 => .{ .identity = {} },
-        1 => .{ .inversion = {} },
-        2 => .{ .zigzag = {} },
-        3 => .{ .random = random.int(u64) },
-        else => unreachable,
-    };
-}
-
 pub fn parse_seed(bytes: []const u8) u64 {
     return std.fmt.parseUnsigned(u64, bytes, 10) catch |err| switch (err) {
         error.Overflow => @panic("seed exceeds a 64-bit unsigned integer"),

package/src/tigerbeetle/src/{test/accounting → state_machine}/auditor.zig

@@ -6,16 +6,16 @@ const std = @import("std");
 const assert = std.debug.assert;
 const log = std.log.scoped(.test_auditor);
 
-const constants = @import("../../constants.zig");
-const tb = @import("../../tigerbeetle.zig");
-const vsr = @import("../../vsr.zig");
-const RingBuffer = @import("../../ring_buffer.zig").RingBuffer;
-const IdPermutation = @import("../id.zig").IdPermutation;
+const constants = @import("../constants.zig");
+const tb = @import("../tigerbeetle.zig");
+const vsr = @import("../vsr.zig");
+const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
+const IdPermutation = @import("../test/id.zig").IdPermutation;
 
 // TODO(zig) This won't be necessary in Zig 0.10.
-const PriorityQueue = @import("../priority_queue.zig").PriorityQueue;
-const Storage = @import("../storage.zig").Storage;
-const StateMachine = @import("../../state_machine.zig").StateMachineType(Storage, .{
+const PriorityQueue = @import("../test/priority_queue.zig").PriorityQueue;
+const Storage = @import("../test/storage.zig").Storage;
+const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
     .message_body_size_max = constants.message_body_size_max,
 });
 

package/src/tigerbeetle/src/{test/accounting → state_machine}/workload.zig

@@ -21,18 +21,16 @@ const std = @import("std");
 const assert = std.debug.assert;
 const log = std.log.scoped(.test_workload);
 
-const constants = @import("../../constants.zig");
-const tb = @import("../../tigerbeetle.zig");
-const vsr = @import("../../vsr.zig");
-const accounting_auditor = @import("./auditor.zig");
+const constants = @import("../constants.zig");
+const tb = @import("../tigerbeetle.zig");
+const vsr = @import("../vsr.zig");
+const accounting_auditor = @import("auditor.zig");
 const Auditor = accounting_auditor.AccountingAuditor;
-const IdPermutation = @import("../id.zig").IdPermutation;
-const fuzz = @import("../fuzz.zig");
+const IdPermutation = @import("../test/id.zig").IdPermutation;
+const fuzz = @import("../test/fuzz.zig");
 
 // TODO(zig) This won't be necessary in Zig 0.10.
-const PriorityQueue = @import("../priority_queue.zig").PriorityQueue;
-
-// TODO Test linked create_accounts.
+const PriorityQueue = @import("../test/priority_queue.zig").PriorityQueue;
 
 const TransferOutcome = enum {
     /// The transfer is guaranteed to commit.
@@ -177,44 +175,7 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
     return struct {
         const Self = @This();
 
-        pub const Options = struct {
-            auditor_options: Auditor.Options,
-            transfer_id_permutation: IdPermutation,
-
-            operations: std.enums.EnumFieldStruct(Action, usize, null),
-
-            create_account_invalid_probability: u8, // ≤ 100
-            create_transfer_invalid_probability: u8, // ≤ 100
-            create_transfer_limit_probability: u8, // ≤ 100
-            create_transfer_pending_probability: u8, // ≤ 100
-            create_transfer_post_probability: u8, // ≤ 100
-            create_transfer_void_probability: u8, // ≤ 100
-            lookup_account_invalid_probability: u8, // ≤ 100
-
-            lookup_transfer: std.enums.EnumFieldStruct(enum {
-                /// Query a transfer that has either been committed or rejected.
-                delivered,
-                /// Query a transfer whose `create_transfers` is in-flight.
-                sending,
-            }, usize, null),
-
-            // Size of timespan for querying, measured in transfers
-            lookup_transfer_span_mean: usize,
-
-            account_limit_probability: u8, // ≤ 100
-
-            /// This probability is only checked for consecutive guaranteed-successful transfers.
-            linked_valid_probability: u8,
-            /// This probability is only checked for consecutive invalid transfers.
-            linked_invalid_probability: u8,
-
-            pending_timeout_mean: u64,
-
-            accounts_batch_size_min: usize,
-            accounts_batch_size_span: usize, // inclusive
-            transfers_batch_size_min: usize,
-            transfers_batch_size_span: usize, // inclusive
-        };
+        pub const Options = OptionsType(AccountingStateMachine, Action);
 
         random: std.rand.Random,
         auditor: Auditor,
@@ -303,7 +264,11 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
         }
 
         /// A client may build multiple requests to queue up while another is in-flight.
-        pub fn build_request(self: *Self, client_index: usize, body: []align(@alignOf(vsr.Header)) u8) struct {
+        pub fn build_request(
+            self: *Self,
+            client_index: usize,
+            body: []align(@alignOf(vsr.Header)) u8,
+        ) struct {
             operation: Operation,
             size: usize,
         } {
@@ -785,6 +750,101 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
     };
 }
 
+fn OptionsType(comptime StateMachine: type, comptime Action: type) type {
+    return struct {
+        const Options = @This();
+
+        auditor_options: Auditor.Options,
+        transfer_id_permutation: IdPermutation,
+
+        operations: std.enums.EnumFieldStruct(Action, usize, null),
+
+        create_account_invalid_probability: u8, // ≤ 100
+        create_transfer_invalid_probability: u8, // ≤ 100
+        create_transfer_limit_probability: u8, // ≤ 100
+        create_transfer_pending_probability: u8, // ≤ 100
+        create_transfer_post_probability: u8, // ≤ 100
+        create_transfer_void_probability: u8, // ≤ 100
+        lookup_account_invalid_probability: u8, // ≤ 100
+
+        lookup_transfer: std.enums.EnumFieldStruct(enum {
+            /// Query a transfer that has either been committed or rejected.
+            delivered,
+            /// Query a transfer whose `create_transfers` is in-flight.
+            sending,
+        }, usize, null),
+
+        // Size of timespan for querying, measured in transfers
+        lookup_transfer_span_mean: usize,
+
+        account_limit_probability: u8, // ≤ 100
+
+        /// This probability is only checked for consecutive guaranteed-successful transfers.
+        linked_valid_probability: u8,
+        /// This probability is only checked for consecutive invalid transfers.
+        linked_invalid_probability: u8,
+
+        pending_timeout_mean: u64,
+
+        accounts_batch_size_min: usize,
+        accounts_batch_size_span: usize, // inclusive
+        transfers_batch_size_min: usize,
+        transfers_batch_size_span: usize, // inclusive
+
+        pub fn generate(random: std.rand.Random, options: struct {
+            client_count: usize,
+            in_flight_max: usize,
+        }) Options {
+            return .{
+                .auditor_options = .{
+                    .accounts_max = 2 + random.uintLessThan(usize, 128),
+                    .account_id_permutation = IdPermutation.generate(random),
+                    .client_count = options.client_count,
+                    .transfers_pending_max = 256,
+                    .in_flight_max = options.in_flight_max,
+                },
+                .transfer_id_permutation = IdPermutation.generate(random),
+                .operations = .{
+                    .create_accounts = 1 + random.uintLessThan(usize, 10),
+                    .create_transfers = 1 + random.uintLessThan(usize, 100),
+                    .lookup_accounts = 1 + random.uintLessThan(usize, 20),
+                    .lookup_transfers = 1 + random.uintLessThan(usize, 20),
+                },
+                .create_account_invalid_probability = 1,
+                .create_transfer_invalid_probability = 1,
+                .create_transfer_limit_probability = random.uintLessThan(u8, 101),
+                .create_transfer_pending_probability = 1 + random.uintLessThan(u8, 100),
+                .create_transfer_post_probability = 1 + random.uintLessThan(u8, 50),
+                .create_transfer_void_probability = 1 + random.uintLessThan(u8, 50),
+                .lookup_account_invalid_probability = 1,
+                .lookup_transfer = .{
+                    .delivered = 1 + random.uintLessThan(usize, 10),
+                    .sending = 1 + random.uintLessThan(usize, 10),
+                },
+                .lookup_transfer_span_mean = 10 + random.uintLessThan(usize, 1000),
+                .account_limit_probability = random.uintLessThan(u8, 80),
+                .linked_valid_probability = random.uintLessThan(u8, 101),
+                // 100% chance because this only applies to consecutive invalid transfers, which are rare.
+                .linked_invalid_probability = 100,
+                // TODO(Timeouts): When timeouts are implemented in the StateMachine, change this to the
+                // (commented out) value so that timeouts can actually trigger.
+                .pending_timeout_mean = std.math.maxInt(u64) / 2,
+                // .pending_timeout_mean = 1 + random.uintLessThan(usize, 1_000_000_000 / 4),
+                .accounts_batch_size_min = 0,
+                .accounts_batch_size_span = 1 + random.uintLessThan(
+                    usize,
+                    StateMachine.constants.batch_max.create_accounts,
+                ),
+                .transfers_batch_size_min = 0,
+                .transfers_batch_size_span = 1 + random.uintLessThan(
+                    usize,
+                    StateMachine.constants.batch_max.create_transfers,
+                ),
+            };
+        }
+    };
+}
+
 /// Sample from a discrete distribution.
 /// Use integers instead of floating-point numbers to avoid nondeterminism on different hardware.
 fn sample_distribution(

package/src/tigerbeetle/src/state_machine.zig

@@ -8,6 +8,7 @@ const tracer = @import("tracer.zig");
 
 const tb = @import("tigerbeetle.zig");
 const snapshot_latest = @import("lsm/tree.zig").snapshot_latest;
+const WorkloadType = @import("state_machine/workload.zig").WorkloadType;
 
 const Account = tb.Account;
 const AccountFlags = tb.AccountFlags;
@@ -26,7 +27,6 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
 }) type {
     return struct {
         const StateMachine = @This();
-
         const Grid = @import("lsm/grid.zig").GridType(Storage);
         const GrooveType = @import("lsm/groove.zig").GrooveType;
         const ForestType = @import("lsm/forest.zig").ForestType;
@@ -49,6 +49,8 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
         );
         const PostedGroove = @import("lsm/posted_groove.zig").PostedGrooveType(Storage);
 
+        pub const Workload = WorkloadType(StateMachine);
+
         pub const Forest = ForestType(Storage, .{
             .accounts = AccountsGroove,
             .transfers = TransfersGroove,
@@ -180,13 +182,13 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
         /// Returns the header's timestamp.
         pub fn prepare(self: *StateMachine, operation: Operation, input: []u8) u64 {
             switch (operation) {
+                .reserved => unreachable,
                 .root => unreachable,
                 .register => {},
                 .create_accounts => self.prepare_timestamps(.create_accounts, input),
                 .create_transfers => self.prepare_timestamps(.create_transfers, input),
                 .lookup_accounts => {},
                 .lookup_transfers => {},
-                else => unreachable,
             }
             return self.prepare_timestamp;
         }
@@ -660,10 +662,7 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
             if (t.credit_account_id == t.debit_account_id) return .accounts_must_be_different;
 
             if (t.pending_id != 0) return .pending_id_must_be_zero;
-            if (t.flags.pending) {
-                // Otherwise, reserved amounts may never be released.
-                if (t.timeout == 0) return .pending_transfer_must_timeout;
-            } else {
+            if (!t.flags.pending) {
                 if (t.timeout != 0) return .timeout_reserved_for_pending_transfer;
             }
 
@@ -822,8 +821,9 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
             }
 
             assert(p.timestamp < t.timestamp);
-            assert(p.timeout > 0);
-            if (p.timestamp + p.timeout <= t.timestamp) return .pending_transfer_expired;
+            if (p.timeout > 0) {
+                if (p.timestamp + p.timeout <= t.timestamp) return .pending_transfer_expired;
+            }
 
             self.forest.grooves.transfers.put_no_clobber(&Transfer{
                 .id = t.id,
@@ -1113,6 +1113,7 @@ test "sum_overflows" {
 const TestContext = struct {
     const Storage = @import("test/storage.zig").Storage;
     const MessagePool = @import("message_pool.zig").MessagePool;
+    const data_file_size_min = @import("vsr/superblock.zig").data_file_size_min;
     const SuperBlock = @import("vsr/superblock.zig").SuperBlockType(Storage);
     const Grid = @import("lsm/grid.zig").GridType(Storage);
     const StateMachine = StateMachineType(Storage, .{
@@ -1147,7 +1148,11 @@ const TestContext = struct {
         };
         errdefer ctx.message_pool.deinit(allocator);
 
-        ctx.superblock = try SuperBlock.init(allocator, &ctx.storage, &ctx.message_pool);
+        ctx.superblock = try SuperBlock.init(allocator, .{
+            .storage = &ctx.storage,
+            .storage_size_limit = data_file_size_min,
+            .message_pool = &ctx.message_pool,
+        });
         errdefer ctx.superblock.deinit(allocator);
 
         // Pretend that the superblock is open so that the Forest can initialize.
@@ -1622,7 +1627,6 @@ test "create_transfers/lookup_transfers" {
         \\ transfer T1 A8 -0  _  _ T1   _ L0 C0 _ PEN _ _  _    0 credit_account_id_must_not_be_int_max
         \\ transfer T1 A8 A8  _  _ T1   _ L0 C0 _ PEN _ _  _    0 accounts_must_be_different
         \\ transfer T1 A8 A9  _  _ T1   _ L0 C0 _ PEN _ _  _    0 pending_id_must_be_zero
-        \\ transfer T1 A8 A9  _  _  _   _ L0 C0 _ PEN _ _  _    0 pending_transfer_must_timeout
         \\ transfer T1 A8 A9  _  _  _  -0 L0 C0 _   _ _ _  _    0 timeout_reserved_for_pending_transfer
         \\ transfer T1 A8 A9  _  _  _  -0 L0 C0 _ PEN _ _  _    0 ledger_must_not_be_zero
         \\ transfer T1 A8 A9  _  _  _  -0 L9 C0 _ PEN _ _  _    0 code_must_not_be_zero
@@ -1680,11 +1684,12 @@ test "create/lookup 2-phase transfers" {
         \\ transfer   T3 A1 A2  _ _   _   50 L1 C1 _ PEN   _   _ _ 15 ok
         \\ transfer   T4 A1 A2  _ _   _    1 L1 C1 _ PEN   _   _ _ 15 ok
         \\ transfer   T5 A1 A2 U9 _   _   50 L1 C1 _ PEN   _   _ _  7 ok
+        \\ transfer   T6 A1 A2  _ _   _    0 L1 C1 _ PEN   _   _ _  1 ok
         \\ commit create_transfers
 
         // Check balances before resolving.
-        \\ lookup_account A1 52 15  0  0
-        \\ lookup_account A2  0  0 52 15
+        \\ lookup_account A1 53 15  0  0
+        \\ lookup_account A2  0  0 53 15
         \\ commit lookup_accounts
 
         // Second phase.
@@ -1715,11 +1720,12 @@ test "create/lookup 2-phase transfers" {
         \\ transfer T102 A1 A2 U1 _  T3    _ L1 C1 _   _ POS   _ _ 13 pending_transfer_already_voided
         \\ transfer T102 A1 A2 U1 _  T4    _ L1 C1 _   _   _ VOI _ 15 pending_transfer_expired
         \\ transfer T105 A0 A0 U0 _  T5    _ L0 C0 _   _ POS   _ _  _ ok
+        \\ transfer T106 A0 A0 U0 _  T6    _ L1 C1 _   _ POS   _ _  0 ok
         \\ commit create_transfers
 
         // Check balances after resolving.
-        \\ lookup_account A1 15 35  0  0
-        \\ lookup_account A2  0  0 15 35
+        \\ lookup_account A1 15 36  0  0
+        \\ lookup_account A2  0  0 15 36
         \\ commit lookup_accounts
     );
 }

package/src/tigerbeetle/src/storage.zig

@@ -5,8 +5,8 @@ const assert = std.debug.assert;
 const log = std.log.scoped(.storage);
 
 const IO = @import("io.zig").IO;
+const FIFO = @import("fifo.zig").FIFO;
 const constants = @import("constants.zig");
-const fatal = @import("cli.zig").fatal;
 const vsr = @import("vsr.zig");
 
 pub const Storage = struct {
@@ -68,9 +68,17 @@ pub const Storage = struct {
         offset: u64,
     };
 
+    pub const NextTick = struct {
+        next: ?*NextTick = null,
+        callback: fn (next_tick: *NextTick) void,
+    };
+
     io: *IO,
     fd: os.fd_t,
 
+    next_tick_queue: FIFO(NextTick) = .{},
+    next_tick_completion: IO.Completion = undefined,
+
     pub fn init(io: *IO, fd: os.fd_t) !Storage {
         return Storage{
             .io = io,
@@ -79,6 +87,7 @@ pub const Storage = struct {
     }
 
     pub fn deinit(storage: *Storage) void {
+        assert(storage.next_tick_queue.empty());
         assert(storage.fd != IO.INVALID_FILE);
         storage.fd = IO.INVALID_FILE;
     }
@@ -90,6 +99,43 @@ pub const Storage = struct {
         };
     }
 
+    pub fn on_next_tick(
+        storage: *Storage,
+        callback: fn (next_tick: *Storage.NextTick) void,
+        next_tick: *Storage.NextTick,
+    ) void {
+        next_tick.* = .{ .callback = callback };
+
+        const was_empty = storage.next_tick_queue.empty();
+        storage.next_tick_queue.push(next_tick);
+
+        if (was_empty) {
+            storage.io.timeout(
+                *Storage,
+                storage,
+                timeout_callback,
+                &storage.next_tick_completion,
+                0, // 0ns timeout means to resolve as soon as possible - like a yield
+            );
+        }
+    }
+
+    fn timeout_callback(
+        storage: *Storage,
+        completion: *IO.Completion,
+        result: IO.TimeoutError!void,
+    ) void {
+        assert(completion == &storage.next_tick_completion);
+        _ = result catch |e| switch (e) {
+            error.Canceled => unreachable,
+            error.Unexpected => unreachable,
+        };
+
+        var queue = storage.next_tick_queue;
+        storage.next_tick_queue = .{};
+        while (queue.pop()) |next_tick| next_tick.callback(next_tick);
+    }
+
     pub fn read_sectors(
         self: *Storage,
         callback: fn (read: *Storage.Read) void,
@@ -113,18 +159,24 @@ pub const Storage = struct {
             .target_max = buffer.len,
         };
 
-        self.start_read(read, 0);
+        self.start_read(read, null);
         assert(read.target().len > 0);
     }
 
-    fn start_read(self: *Storage, read: *Storage.Read, bytes_read: usize) void {
-        assert(bytes_read <= read.target().len);
+    fn start_read(self: *Storage, read: *Storage.Read, bytes_read: ?usize) void {
+        const bytes = bytes_read orelse 0;
+        assert(bytes <= read.target().len);
 
-        read.offset += bytes_read;
-        read.buffer = read.buffer[bytes_read..];
+        read.offset += bytes;
+        read.buffer = read.buffer[bytes..];
 
         const target = read.target();
         if (target.len == 0) {
+            // Resolving the read inline means start_read() must not have been called from
+            // read_sectors(). If it was, this is a synchronous callback resolution and should
+            // be reported.
+            assert(bytes_read != null);
+
             read.callback(read);
             return;
         }