tigerbeetle-node 0.11.0 → 0.11.1

package/src/tigerbeetle/src/message_bus.zig

@@ -12,6 +12,7 @@ const log = std.log.scoped(.message_bus);
 const vsr = @import("vsr.zig");
 const Header = vsr.Header;
 
+const util = @import("util.zig");
 const RingBuffer = @import("ring_buffer.zig").RingBuffer;
 const IO = @import("io.zig").IO;
 const MessagePool = @import("message_pool.zig").MessagePool;
@@ -44,7 +45,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
         io: *IO,
 
         cluster: u32,
-        configuration: []std.net.Address,
+        configuration: []const std.net.Address,
 
         process: switch (process_type) {
             .replica => struct {
@@ -84,7 +85,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
         prng: std.rand.DefaultPrng,
 
         pub const Options = struct {
-            configuration: []std.net.Address,
+            configuration: []const std.net.Address,
             io: *IO,
         };
 
@@ -145,8 +146,19 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
             return bus;
         }
 
-        /// TODO This is required by the Client.
-        pub fn deinit(_: *Self, _: std.mem.Allocator) void {}
+        pub fn deinit(bus: *Self, allocator: std.mem.Allocator) void {
+            if (process_type == .replica) {
+                bus.process.clients.deinit(allocator);
+            }
+
+            for (bus.connections) |*connection| {
+                if (connection.recv_message) |message| bus.unref(message);
+                while (connection.send_queue.pop()) |message| bus.unref(message);
+            }
+            allocator.free(bus.connections);
+            allocator.free(bus.replicas);
+            allocator.free(bus.replicas_connect_attempts);
+        }
 
         fn init_tcp(io: *IO, address: std.net.Address) !os.socket_t {
             const fd = try io.open_socket(
@@ -738,7 +750,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
                 if (connection.recv_progress == header.size) return connection.recv_message.?.ref();
 
                 const message = bus.get_message();
-                mem.copy(u8, message.buffer, data[0..header.size]);
+                util.copy_disjoint(.inexact, u8, message.buffer, data[0..header.size]);
                 return message;
             }
 
@@ -811,7 +823,6 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
             }
 
             /// Acquires a free message if necessary and then calls `recv()`.
-            /// Terminates the connection if a free message cannot be obtained.
             /// If the connection has a `recv_message` and the message being parsed is
             /// at pole position then calls `recv()` immediately, otherwise copies any
             /// partially received message into a new Message and sets `recv_message`,
@@ -831,7 +842,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
                     assert(connection.recv_progress > 0);
                     assert(connection.recv_parsed > 0);
                     const data = recv_message.buffer[connection.recv_parsed..connection.recv_progress];
-                    mem.copy(u8, new_message.buffer, data);
+                    util.copy_disjoint(.inexact, u8, new_message.buffer, data);
                     connection.recv_progress = data.len;
                     connection.recv_parsed = 0;
                 } else {

package/src/tigerbeetle/src/message_pool.zig

@@ -37,7 +37,6 @@ pub const messages_max_replica = messages_max: {
     // Handle Replica.commit_op's reply:
     // (This is separate from the burst +1 because they may occur concurrently).
     sum += 1;
-    sum += 20; // TODO Our network simulator allows up to 20 messages for path_capacity_max.
 
     break :messages_max sum;
 };
@@ -51,7 +50,6 @@ pub const messages_max_client = messages_max: {
     sum += config.client_request_queue_max; // Client.request_queue
     // Handle bursts (e.g. Connection.parse_message, or sending a ping when the send queue is full).
     sum += 1;
-    sum += 20; // TODO Our network simulator allows up to 20 messages for path_capacity_max.
 
     break :messages_max sum;
 };
@@ -86,6 +84,8 @@ pub const MessagePool = struct {
     /// List of currently unused messages.
     free_list: ?*Message,
 
+    messages_max: usize,
+
     pub fn init(allocator: mem.Allocator, process_type: vsr.ProcessType) error{OutOfMemory}!MessagePool {
         return MessagePool.init_capacity(allocator, switch (process_type) {
             .replica => messages_max_replica,
@@ -96,6 +96,7 @@ pub const MessagePool = struct {
     pub fn init_capacity(allocator: mem.Allocator, messages_max: usize) error{OutOfMemory}!MessagePool {
         var pool: MessagePool = .{
             .free_list = null,
+            .messages_max = messages_max,
         };
         {
             var i: usize = 0;
@@ -121,11 +122,16 @@ pub const MessagePool = struct {
 
     /// Frees all messages that were unused or returned to the pool via unref().
     pub fn deinit(pool: *MessagePool, allocator: mem.Allocator) void {
+        var free_count: usize = 0;
         while (pool.free_list) |message| {
             pool.free_list = message.next;
             allocator.free(message.buffer);
             allocator.destroy(message);
+            free_count += 1;
         }
+        // If the MessagePool is being deinitialized, all messages should have already been
+        // released to the pool.
+        assert(free_count == pool.messages_max);
     }
 
     /// Get an unused message with a buffer of config.message_size_max.

package/src/tigerbeetle/src/ring_buffer.zig

@@ -3,15 +3,19 @@ const assert = std.debug.assert;
 const math = std.math;
 const mem = std.mem;
 
+const util = @import("util.zig");
+
 /// A First In, First Out ring buffer holding at most `count_max` elements.
 pub fn RingBuffer(
     comptime T: type,
-    comptime count_max: usize,
+    comptime count_max_: usize,
     comptime buffer_type: enum { array, pointer },
 ) type {
     return struct {
         const Self = @This();
 
+        pub const count_max = count_max_;
+
         buffer: switch (buffer_type) {
             .array => [count_max]T,
             .pointer => *[count_max]T,
@@ -145,8 +149,8 @@ pub fn RingBuffer(
             const pre_wrap_count = math.min(items.len, self.buffer.len - pre_wrap_start);
             const post_wrap_count = items.len - pre_wrap_count;
 
-            mem.copy(T, self.buffer[pre_wrap_start..], items[0..pre_wrap_count]);
-            mem.copy(T, self.buffer[0..post_wrap_count], items[pre_wrap_count..]);
+            util.copy_disjoint(.inexact, T, self.buffer[pre_wrap_start..], items[0..pre_wrap_count]);
+            util.copy_disjoint(.exact, T, self.buffer[0..post_wrap_count], items[pre_wrap_count..]);
 
             self.count += items.len;
         }

package/src/tigerbeetle/src/simulator.zig

@@ -14,6 +14,7 @@ const ClusterOptions = @import("test/cluster.zig").ClusterOptions;
 const Replica = @import("test/cluster.zig").Replica;
 const StateMachine = @import("test/cluster.zig").StateMachine;
 const StateChecker = @import("test/state_checker.zig").StateChecker;
+const StorageChecker = @import("test/storage_checker.zig").StorageChecker;
 const PartitionMode = @import("test/packet_simulator.zig").PartitionMode;
 const MessageBus = @import("test/message_bus.zig").MessageBus;
 const auditor = @import("test/accounting/auditor.zig");
@@ -28,8 +29,7 @@ const output = std.log.scoped(.state_checker);
 /// This will run much slower but will trace all logic across the cluster.
 const log_state_transitions_only = builtin.mode != .Debug;
 
-const log_health = std.log.scoped(.health);
-const log_faults = std.log.scoped(.faults);
+const log_simulator = std.log.scoped(.simulator);
 
 /// You can fine tune your log levels even further (debug/info/notice/warn/err/crit/alert/emerg):
 pub const log_level: std.log.Level = if (log_state_transitions_only) .info else .debug;
@@ -41,6 +41,7 @@ const cluster_id = 0;
 
 var cluster: *Cluster = undefined;
 var state_checker: *StateChecker = undefined;
+var storage_checker: *StorageChecker = undefined;
 
 pub fn main() !void {
     comptime {
@@ -103,7 +104,9 @@ pub fn main() !void {
         // TODO Compute an upper-bound for this based on requests_committed_max.
         .grid_size_max = 1024 * 1024 * 256,
         .seed = random.int(u64),
-        .on_change_state = on_change_replica,
+        .on_change_state = on_replica_change_state,
+        .on_compact = on_replica_compact,
+        .on_checkpoint = on_replica_checkpoint,
         .network_options = .{
             .packet_simulator_options = .{
                 .replica_count = replica_count,
@@ -134,11 +137,17 @@ pub fn main() !void {
             .write_latency_mean = 3 + random.uintLessThan(u16, 100),
             .read_fault_probability = random.uintLessThan(u8, 10),
             .write_fault_probability = random.uintLessThan(u8, 10),
+            // TODO Allow WAL faults on crash when replica_count=1 when redundant-header-repair
+            // is implemented after recovering with decision=fix. Otherwise we can end up with
+            // multiple crashes faulting first a redundant headers, then a prepare, upgrading
+            // a decision=fix to decision=vsr.
+            .crash_fault_probability = if (replica_count == 1) 0 else 80 + random.uintLessThan(u8, 21),
+            .faulty_superblock = true,
         },
         .health_options = .{
-            .crash_probability = 0.0001,
+            .crash_probability = 0.000001,
             .crash_stability = random.uintLessThan(u32, 1_000),
-            .restart_probability = 0.01,
+            .restart_probability = 0.0001,
             .restart_stability = random.uintLessThan(u32, 1_000),
         },
         .state_machine_options = .{
@@ -292,6 +301,12 @@ pub fn main() !void {
     );
     defer state_checker.deinit();
 
+    storage_checker = try allocator.create(StorageChecker);
+    defer allocator.destroy(storage_checker);
+
+    storage_checker.* = StorageChecker.init(allocator);
+    defer storage_checker.deinit();
+
     // The minimum number of healthy replicas required for a crashed replica to be able to recover.
     const replica_normal_min = replicas: {
         if (replica_count == 1) {
@@ -330,14 +345,14 @@ pub fn main() !void {
                 // complete the VSR recovery protocol either.
                 if (cluster.health[replica] == .up and crashes == 0) {
                     if (storage.faulty) {
-                        log_faults.debug("{}: disable storage faults", .{replica});
+                        log_simulator.debug("{}: disable storage faults", .{replica});
                         storage.faulty = false;
                     }
                 } else {
                     // When a journal recovers for the first time, enable its storage faults.
                     // Future crashes will recover in the presence of faults.
                     if (!storage.faulty) {
-                        log_faults.debug("{}: enable storage faults", .{replica});
+                        log_simulator.debug("{}: enable storage faults", .{replica});
                         storage.faulty = true;
                     }
                 }
@@ -364,7 +379,7 @@ pub fn main() !void {
                     }
 
                     if (!try cluster.crash_replica(replica.replica)) continue;
-                    log_health.debug("{}: crash replica", .{replica.replica});
+                    log_simulator.debug("{}: crash replica", .{replica.replica});
                     crashes -= 1;
                 },
                 .down => |*ticks| {
@@ -375,7 +390,7 @@ pub fn main() !void {
                     assert(replica.status == .recovering);
                     if (ticks.* == 0 and chance_f64(random, health_options.restart_probability)) {
                         cluster.health[replica.replica] = .{ .up = health_options.restart_stability };
-                        log_health.debug("{}: restart replica", .{replica.replica});
+                        log_simulator.debug("{}: restart replica", .{replica.replica});
                     }
                 },
             }
@@ -416,7 +431,7 @@ fn fatal(exit_code: ExitCode, comptime fmt_string: []const u8, args: anytype) no
 /// Returns true, `p` percent of the time, else false.
 fn chance_f64(random: std.rand.Random, p: f64) bool {
     assert(p <= 100.0);
-    return random.float(f64) < p;
+    return random.float(f64) * 100.0 < p;
 }
 
 /// Returns the next argument for the simulator or null (if none available)
@@ -425,12 +440,24 @@ fn args_next(args: *std.process.ArgIterator, allocator: std.mem.Allocator) ?[:0]
     return err_or_bytes catch @panic("Unable to extract next value from args");
 }
 
-fn on_change_replica(replica: *Replica) void {
+fn on_replica_change_state(replica: *const Replica) void {
     state_checker.check_state(replica.replica) catch |err| {
         fatal(.correctness, "state checker error: {}", .{err});
     };
 }
 
+fn on_replica_compact(replica: *const Replica) void {
+    storage_checker.replica_compact(replica) catch |err| {
+        fatal(.correctness, "storage checker error: {}", .{err});
+    };
+}
+
+fn on_replica_checkpoint(replica: *const Replica) void {
+    storage_checker.replica_checkpoint(replica) catch |err| {
+        fatal(.correctness, "storage checker error: {}", .{err});
+    };
+}
+
 /// Returns a random partitioning mode, excluding .custom
 fn random_partition_mode(random: std.rand.Random) PartitionMode {
     const typeInfo = @typeInfo(PartitionMode).Enum;

package/src/tigerbeetle/src/state_machine.zig

@@ -75,10 +75,10 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
                 pub const lookup_transfers = operation_batch_max(.lookup_transfers);
 
                 comptime {
-                    assert(create_accounts >= 0);
-                    assert(create_transfers >= 0);
-                    assert(lookup_accounts >= 0);
-                    assert(lookup_transfers >= 0);
+                    assert(create_accounts > 0);
+                    assert(create_transfers > 0);
+                    assert(lookup_accounts > 0);
+                    assert(lookup_transfers > 0);
                 }
 
                 fn operation_batch_max(comptime operation: Operation) usize {
@@ -571,7 +571,7 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
 
             if (self.get_account(a.id)) |e| return create_account_exists(a, e);
 
-            self.forest.grooves.accounts.put(a);
+            self.forest.grooves.accounts.put_no_clobber(a);
 
             self.commit_timestamp = a.timestamp;
             return .ok;
@@ -653,6 +653,7 @@ pub fn StateMachineType(comptime Storage: type, comptime constants_: struct {
             if (sum_overflows(t.amount, cr.credits_pending + cr.credits_posted)) {
                 return .overflows_credits;
             }
+            if (sum_overflows(t.timestamp, t.timeout)) return .overflows_timeout;
 
             if (dr.debits_exceed_credits(t.amount)) return .exceeds_credits;
             if (cr.credits_exceed_debits(t.amount)) return .exceeds_debits;
@@ -1074,23 +1075,19 @@ const TestContext = struct {
             allocator,
             4096,
             .{
-                .seed = 0,
                 .read_latency_min = 0,
                 .read_latency_mean = 0,
                 .write_latency_min = 0,
                 .write_latency_mean = 0,
-                .read_fault_probability = 0,
-                .write_fault_probability = 0,
-            },
-            0,
-            .{
-                .first_offset = 0,
-                .period = 0,
             },
         );
         errdefer ctx.storage.deinit(allocator);
 
-        ctx.message_pool = .{ .free_list = null };
+        ctx.message_pool = .{
+            .free_list = null,
+            .messages_max = 0,
+        };
+        errdefer ctx.message_pool.deinit(allocator);
 
         ctx.superblock = try SuperBlock.init(allocator, &ctx.storage, &ctx.message_pool);
         errdefer ctx.superblock.deinit(allocator);
@@ -1837,7 +1834,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 100,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 0,
                 .code = 0,
                 .amount = 0,
@@ -1850,7 +1847,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 100,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 0,
                 .code = 0,
                 .flags = .{ .pending = true },
@@ -1864,7 +1861,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 100,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 0,
                 .flags = .{ .pending = true },
@@ -1878,7 +1875,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 100,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 1,
                 .flags = .{ .pending = true },
@@ -1892,7 +1889,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 100,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 1,
                 .flags = .{ .pending = true },
@@ -1906,7 +1903,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 200,
-                .timeout = 1,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 1,
                 .flags = .{ .pending = true },
@@ -1920,9 +1917,11 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 2,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 1,
                 .amount = 1,
+                .flags = .{ .pending = true },
                 .timestamp = timestamp,
             }),
         },
@@ -1932,9 +1931,11 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 100,
                 .code = 1,
                 .amount = 1,
+                .flags = .{ .pending = true },
                 .timestamp = timestamp,
             }),
         },
@@ -1944,7 +1945,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
-                .timeout = 30000,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
                 .flags = .{ .pending = true },
@@ -1958,7 +1959,7 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
-                .timeout = 30000,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
                 .flags = .{ .pending = true },
@@ -1972,8 +1973,10 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
+                .flags = .{ .pending = true },
                 .amount = math.maxInt(u64) - accounts[1 - 1].debits_posted + 1,
                 .timestamp = timestamp,
             }),
@@ -1984,8 +1987,10 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
+                .flags = .{ .pending = true },
                 .amount = math.maxInt(u64) - accounts[3 - 1].credits_posted + 1,
                 .timestamp = timestamp,
             }),
@@ -1996,8 +2001,10 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
+                .flags = .{ .pending = true },
                 .amount = math.maxInt(u64) -
                     accounts[1 - 1].debits_pending -
                     accounts[1 - 1].debits_posted + 1,
@@ -2010,14 +2017,32 @@ test "create/lookup/rollback transfers" {
                 .id = 1,
                 .debit_account_id = 1,
                 .credit_account_id = 3,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
                 .ledger = 1,
                 .code = 1,
+                .flags = .{ .pending = true },
                 .amount = math.maxInt(u64) -
                     accounts[3 - 1].credits_pending -
                     accounts[3 - 1].credits_posted + 1,
                 .timestamp = timestamp,
             }),
         },
+        .{
+            .result = .overflows_timeout,
+            .object = mem.zeroInit(Transfer, .{
+                .id = 1,
+                .debit_account_id = 4,
+                .credit_account_id = 5,
+                .timeout = (std.math.maxInt(u64) - timestamp) + 1,
+                .ledger = 1,
+                .code = 1,
+                .flags = .{ .pending = true },
+                .amount = accounts[4 - 1].credits_posted -
+                    accounts[4 - 1].debits_pending -
+                    accounts[4 - 1].debits_posted + 1,
+                .timestamp = timestamp,
+            }),
+        },
         .{
             .result = .exceeds_credits,
             .object = mem.zeroInit(Transfer, .{

package/src/tigerbeetle/src/test/accounting/workload.zig

@@ -589,11 +589,15 @@ pub fn WorkloadType(comptime AccountingStateMachine: type) type {
                 .single_phase => {},
                 .pending => {
                     transfer.flags = .{ .pending = true };
-                    transfer.timeout = std.math.max(1, fuzz.random_int_exponential(
-                        self.random,
-                        u64,
-                        self.options.pending_timeout_mean,
-                    ));
+                    // Bound the timeout to ensure we never hit `overflows_timeout`.
+                    transfer.timeout = 1 + std.math.min(
+                        std.math.maxInt(u64) / 2,
+                        fuzz.random_int_exponential(
+                            self.random,
+                            u64,
+                            self.options.pending_timeout_mean,
+                        ),
+                    );
                 },
                 .post_pending, .void_pending => {
                     // Don't depend on `HashMap.keyIterator()` being deterministic.

package/src/tigerbeetle/src/test/cluster.zig

@@ -32,7 +32,9 @@ pub const ClusterOptions = struct {
     grid_size_max: usize,
 
     seed: u64,
-    on_change_state: fn (replica: *Replica) void,
+    on_change_state: fn (replica: *const Replica) void,
+    on_compact: fn (replica: *const Replica) void,
+    on_checkpoint: fn (replica: *const Replica) void,
 
     network_options: NetworkOptions,
     storage_options: Storage.Options,
@@ -122,21 +124,22 @@ pub const Cluster = struct {
         };
 
         var buffer: [config.replicas_max]Storage.FaultyAreas = undefined;
-        const faulty_areas = Storage.generate_faulty_areas(
+        const faulty_wal_areas = Storage.generate_faulty_wal_areas(
             prng,
             config.journal_size_max,
             options.replica_count,
             &buffer,
         );
-        assert(faulty_areas.len == options.replica_count);
+        assert(faulty_wal_areas.len == options.replica_count);
 
         for (cluster.storages) |*storage, replica_index| {
+            var storage_options = options.storage_options;
+            storage_options.replica_index = @intCast(u8, replica_index);
+            storage_options.faulty_wal_areas = faulty_wal_areas[replica_index];
             storage.* = try Storage.init(
                 allocator,
                 superblock_zone_size + config.journal_size_max + options.grid_size_max,
-                options.storage_options,
-                @intCast(u8, replica_index),
-                faulty_areas[replica_index],
+                storage_options,
             );
         }
         errdefer for (cluster.storages) |*storage| storage.deinit(allocator);
@@ -280,41 +283,18 @@ pub const Cluster = struct {
         const replica_time = replica.time;
         replica.deinit(cluster.allocator);
 
-        // The message bus and network should be left alone, as messages
-        // may still be inflight to/from this replica. However, we should
-        // do a check to ensure that we aren't leaking any messages when
-        // deinitializing the replica above.
-        const packet_simulator = &cluster.network.packet_simulator;
-        // The same message may be used for multiple network packets, so simply counting how
-        // many packets are inflight from the replica is insufficient, we need to dedup them.
-        var messages_in_network_set = std.AutoHashMap(*Message, void).init(cluster.allocator);
-        defer messages_in_network_set.deinit();
-
-        var target: u8 = 0;
-        while (target < packet_simulator.options.node_count) : (target += 1) {
-            const path = .{ .source = replica_index, .target = target };
-            const queue = packet_simulator.path_queue(path);
-            var it = queue.iterator();
-            while (it.next()) |data| {
-                try messages_in_network_set.put(data.packet.message, {});
-            }
-        }
-
-        const messages_in_network = messages_in_network_set.count();
-
+        // Ensure that none of the replica's messages leaked when it was deinitialized.
         var messages_in_pool: usize = 0;
         const message_bus = cluster.network.get_message_bus(.{ .replica = replica_index });
         {
             var it = message_bus.pool.free_list;
             while (it) |message| : (it = message.next) messages_in_pool += 1;
         }
-
-        const total_messages = message_pool.messages_max_replica;
-        assert(messages_in_network + messages_in_pool == total_messages);
+        assert(messages_in_pool == message_pool.messages_max_replica);
 
         // Logically it would make more sense to run this during restart, not immediately following
-        // the crash. But having it here allows the replica's MessageBus to initialized and start
-        // queueing packets, or collecting packets that are dropped by the network.
+        // the crash. But having it here allows the replica's MessageBus to initialize and begin
+        // queueing packets.
         //
         // Pass the old replica's Time through to the new replica. It will continue to be tick
         // while the replica is crashed, to ensure the clocks don't desyncronize too far to recover.
@@ -362,6 +342,8 @@ pub const Cluster = struct {
         assert(replica.status == .recovering);
 
         replica.on_change_state = cluster.options.on_change_state;
+        replica.on_compact = cluster.options.on_compact;
+        replica.on_checkpoint = cluster.options.on_checkpoint;
         cluster.network.link(replica.message_bus.process, &replica.message_bus);
     }
 };

package/src/tigerbeetle/src/test/conductor.zig

@@ -8,6 +8,7 @@ const assert = std.debug.assert;
 const log = std.log.scoped(.test_conductor);
 
 const vsr = @import("../vsr.zig");
+const util = @import("../util.zig");
 const config = @import("../config.zig");
 const IdPermutation = @import("id.zig").IdPermutation;
 const MessagePool = @import("../message_pool.zig").MessagePool;
@@ -310,7 +311,7 @@ pub fn ConductorType(
         /// Returns the Conductor's message.
         fn clone_message(self: *Self, message_client: *const Message) *Message {
             const message_conductor = self.message_pool.get_message();
-            std.mem.copy(u8, message_conductor.buffer, message_client.buffer);
+            util.copy_disjoint(.exact, u8, message_conductor.buffer, message_client.buffer);
             return message_conductor;
         }