tigerbeetle-node 0.11.8 → 0.11.10

package/src/tigerbeetle/src/{util.zig → stdx.zig}

@@ -1,3 +1,5 @@
+//! Extensions to the standard library -- things which could have been in std, but aren't.
+
 const std = @import("std");
 const assert = std.debug.assert;
 

package/src/tigerbeetle/src/storage.zig

@@ -77,6 +77,7 @@ pub const Storage = struct {
     fd: os.fd_t,
 
     next_tick_queue: FIFO(NextTick) = .{},
+    next_tick_completion_scheduled: bool = false,
     next_tick_completion: IO.Completion = undefined,
 
     pub fn init(io: *IO, fd: os.fd_t) !Storage {
@@ -105,11 +106,10 @@ pub const Storage = struct {
         next_tick: *Storage.NextTick,
     ) void {
         next_tick.* = .{ .callback = callback };
-
-        const was_empty = storage.next_tick_queue.empty();
         storage.next_tick_queue.push(next_tick);
 
-        if (was_empty) {
+        if (!storage.next_tick_completion_scheduled) {
+            storage.next_tick_completion_scheduled = true;
             storage.io.timeout(
                 *Storage,
                 storage,
@@ -131,9 +131,16 @@ pub const Storage = struct {
             error.Unexpected => unreachable,
         };
 
-        var queue = storage.next_tick_queue;
-        storage.next_tick_queue = .{};
-        while (queue.pop()) |next_tick| next_tick.callback(next_tick);
+        // Reset the scheduled flag after processing all tick entries
+        assert(storage.next_tick_completion_scheduled);
+        defer {
+            assert(storage.next_tick_completion_scheduled);
+            storage.next_tick_completion_scheduled = false;
+        }
+
+        while (storage.next_tick_queue.pop()) |next_tick| {
+            next_tick.callback(next_tick);
+        }
     }
 
     pub fn read_sectors(

package/src/tigerbeetle/src/{test → testing/cluster}/message_bus.zig

@@ -1,10 +1,10 @@
 const std = @import("std");
 const assert = std.debug.assert;
 
-const MessagePool = @import("../message_pool.zig").MessagePool;
+const MessagePool = @import("../../message_pool.zig").MessagePool;
 const Message = MessagePool.Message;
-const Header = @import("../vsr.zig").Header;
-const ProcessType = @import("../vsr.zig").ProcessType;
+const Header = @import("../../vsr.zig").Header;
+const ProcessType = @import("../../vsr.zig").ProcessType;
 
 const Network = @import("network.zig").Network;
 

package/src/tigerbeetle/src/{test → testing/cluster}/network.zig

@@ -3,25 +3,23 @@ const math = std.math;
 const mem = std.mem;
 const assert = std.debug.assert;
 
-const constants = @import("../constants.zig");
-const vsr = @import("../vsr.zig");
-const util = @import("../util.zig");
+const constants = @import("../../constants.zig");
+const vsr = @import("../../vsr.zig");
+const stdx = @import("../../stdx.zig");
 
-const MessagePool = @import("../message_pool.zig").MessagePool;
+const MessagePool = @import("../../message_pool.zig").MessagePool;
 const Message = MessagePool.Message;
 
 const MessageBus = @import("message_bus.zig").MessageBus;
 const Process = @import("message_bus.zig").Process;
 
-const PacketSimulator = @import("packet_simulator.zig").PacketSimulator;
-const PacketSimulatorOptions = @import("packet_simulator.zig").PacketSimulatorOptions;
-const PacketSimulatorPath = @import("packet_simulator.zig").Path;
+const PacketSimulatorType = @import("../packet_simulator.zig").PacketSimulatorType;
+const PacketSimulatorOptions = @import("../packet_simulator.zig").PacketSimulatorOptions;
+const PacketSimulatorPath = @import("../packet_simulator.zig").Path;
 
 const log = std.log.scoped(.network);
 
-pub const NetworkOptions = struct {
-    packet_simulator_options: PacketSimulatorOptions,
-};
+pub const NetworkOptions = PacketSimulatorOptions;
 
 pub const Network = struct {
     pub const Packet = struct {
@@ -41,10 +39,10 @@ pub const Network = struct {
     allocator: std.mem.Allocator,
 
     options: NetworkOptions,
-    packet_simulator: PacketSimulator(Packet),
+    packet_simulator: PacketSimulatorType(Packet),
 
     // TODO(Zig) If this stored a ?*MessageBus, then a process's bus could be set to `null` while
-    // the replica is crashed, and replaced when it is destroy. Zig complains:
+    // the replica is crashed, and replaced when it is destroyed. But Zig complains:
     //
     //   ./src/test/message_bus.zig:20:24: error: struct 'test.message_bus.MessageBus' depends on itself
     //   pub const MessageBus = struct {
@@ -53,6 +51,7 @@ pub const Network = struct {
     //       network: *Network,
     //       ^
     buses: std.ArrayListUnmanaged(*MessageBus),
+    buses_enabled: std.ArrayListUnmanaged(bool),
     processes: std.ArrayListUnmanaged(u128),
     /// A pool of messages that are in the network (sent, but not yet delivered).
     message_pool: MessagePool,
@@ -69,13 +68,13 @@ pub const Network = struct {
         var buses = try std.ArrayListUnmanaged(*MessageBus).initCapacity(allocator, process_count);
         errdefer buses.deinit(allocator);
 
+        var buses_enabled = try std.ArrayListUnmanaged(bool).initCapacity(allocator, process_count);
+        errdefer buses_enabled.deinit(allocator);
+
         var processes = try std.ArrayListUnmanaged(u128).initCapacity(allocator, process_count);
         errdefer processes.deinit(allocator);
 
-        var packet_simulator = try PacketSimulator(Packet).init(
-            allocator,
-            options.packet_simulator_options,
-        );
+        var packet_simulator = try PacketSimulatorType(Packet).init(allocator, options);
         errdefer packet_simulator.deinit(allocator);
 
         // Count:
@@ -89,8 +88,7 @@ pub const Network = struct {
             allocator,
             // +1 so we can allocate an extra packet when all packet queues are at capacity,
             // so that `PacketSimulator.submit_packet` can choose which packet to drop.
-            1 + @as(usize, options.packet_simulator_options.path_maximum_capacity) *
-                path_count,
+            1 + @as(usize, options.path_maximum_capacity) * path_count,
         );
         errdefer message_pool.deinit(allocator);
 
@@ -99,6 +97,7 @@ pub const Network = struct {
             .options = options,
             .packet_simulator = packet_simulator,
             .buses = buses,
+            .buses_enabled = buses_enabled,
             .processes = processes,
             .message_pool = message_pool,
         };
@@ -106,11 +105,16 @@ pub const Network = struct {
 
     pub fn deinit(network: *Network) void {
         network.buses.deinit(network.allocator);
+        network.buses_enabled.deinit(network.allocator);
         network.processes.deinit(network.allocator);
         network.packet_simulator.deinit(network.allocator);
         network.message_pool.deinit(network.allocator);
     }
 
+    pub fn tick(network: *Network) void {
+        network.packet_simulator.tick();
+    }
+
     pub fn link(network: *Network, process: Process, message_bus: *MessageBus) void {
         const raw_process = switch (process) {
             .replica => |replica| replica,
@@ -128,10 +132,21 @@ pub const Network = struct {
         } else {
             network.processes.appendAssumeCapacity(raw_process);
             network.buses.appendAssumeCapacity(message_bus);
+            network.buses_enabled.appendAssumeCapacity(true);
         }
         assert(network.processes.items.len == network.buses.items.len);
     }
 
+    pub fn process_enable(network: *Network, process: Process) void {
+        assert(!network.buses_enabled.items[network.process_to_address(process)]);
+        network.buses_enabled.items[network.process_to_address(process)] = true;
+    }
+
+    pub fn process_disable(network: *Network, process: Process) void {
+        assert(network.buses_enabled.items[network.process_to_address(process)]);
+        network.buses_enabled.items[network.process_to_address(process)] = false;
+    }
+
     pub fn send_message(network: *Network, message: *Message, path: Path) void {
         log.debug("send_message: {} > {}: {}", .{
             path.source,
@@ -142,7 +157,7 @@ pub const Network = struct {
         const network_message = network.message_pool.get_message();
         defer network.message_pool.unref(network_message);
 
-        util.copy_disjoint(.exact, u8, network_message.buffer, message.buffer);
+        stdx.copy_disjoint(.exact, u8, network_message.buffer, message.buffer);
 
         network.packet_simulator.submit_packet(
             .{
@@ -157,7 +172,7 @@ pub const Network = struct {
         );
     }
 
-    fn process_to_address(network: *Network, process: Process) u8 {
+    fn process_to_address(network: *const Network, process: Process) u8 {
         for (network.processes.items) |p, i| {
             if (std.meta.eql(raw_process_to_process(p), process)) return @intCast(u8, i);
         }
@@ -171,11 +186,20 @@ pub const Network = struct {
     fn deliver_message(packet: Packet, path: PacketSimulatorPath) void {
         const network = packet.network;
 
+        if (!network.buses_enabled.items[path.target]) {
+            log.debug("deliver_message: {} > {}: {} (dropped; target is down)", .{
+                path.source,
+                path.target,
+                packet.message.header.command,
+            });
+            return;
+        }
+
         const target_bus = network.buses.items[path.target];
         const target_message = target_bus.get_message();
         defer target_bus.unref(target_message);
 
-        util.copy_disjoint(.exact, u8, target_message.buffer, packet.message.buffer);
+        stdx.copy_disjoint(.exact, u8, target_message.buffer, packet.message.buffer);
 
         const process_path = .{
             .source = raw_process_to_process(network.processes.items[path.source]),
@@ -194,7 +218,7 @@ pub const Network = struct {
             const sector_ceil = vsr.sector_ceil(target_message.header.size);
             if (target_message.header.size != sector_ceil) {
                 assert(target_message.header.size < sector_ceil);
-                assert(target_message.buffer.len == constants.message_size_max + constants.sector_size);
+                assert(target_message.buffer.len == constants.message_size_max);
                 mem.set(u8, target_message.buffer[target_message.header.size..sector_ceil], 0);
             }
         }

package/src/tigerbeetle/src/testing/cluster/state_checker.zig

@@ -0,0 +1,169 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const mem = std.mem;
+
+const constants = @import("../../constants.zig");
+const vsr = @import("../../vsr.zig");
+const RingBuffer = @import("../../ring_buffer.zig").RingBuffer;
+
+const message_pool = @import("../../message_pool.zig");
+const MessagePool = message_pool.MessagePool;
+const Message = MessagePool.Message;
+
+const ReplicaSet = std.StaticBitSet(constants.replicas_max);
+const Commits = std.ArrayList(struct {
+    header: vsr.Header,
+    replicas: ReplicaSet = ReplicaSet.initEmpty(),
+});
+
+const log = std.log.scoped(.state_checker);
+
+pub fn StateCheckerType(comptime Client: type, comptime Replica: type) type {
+    return struct {
+        const Self = @This();
+
+        replica_count: u8,
+
+        commits: Commits,
+        commit_mins: [constants.replicas_max]u64 = [_]u64{0} ** constants.replicas_max,
+
+        replicas: []const Replica,
+        clients: []const Client,
+
+        /// The number of times the canonical state has been advanced.
+        requests_committed: u64 = 0,
+
+        pub fn init(
+            allocator: mem.Allocator,
+            cluster: u32,
+            replicas: []const Replica,
+            clients: []const Client,
+        ) !Self {
+            const root_prepare = vsr.Header.root_prepare(cluster);
+
+            var commits = Commits.init(allocator);
+            errdefer commits.deinit();
+
+            var commit_replicas = ReplicaSet.initEmpty();
+            for (replicas) |_, i| commit_replicas.set(i);
+            try commits.append(.{
+                .header = root_prepare,
+                .replicas = commit_replicas,
+            });
+
+            return Self{
+                .replica_count = @intCast(u8, replicas.len),
+                .commits = commits,
+                .replicas = replicas,
+                .clients = clients,
+            };
+        }
+
+        pub fn deinit(state_checker: *Self) void {
+            state_checker.commits.deinit();
+        }
+
+        pub fn check_state(state_checker: *Self, replica_index: u8) !void {
+            const replica = &state_checker.replicas[replica_index];
+
+            const commit_root = replica.superblock.working.vsr_state.commit_min_checksum;
+
+            const commit_a = state_checker.commit_mins[replica_index];
+            const commit_b = replica.commit_min;
+
+            const header_b = replica.journal.header_with_op(replica.commit_min);
+            assert(header_b != null or replica.commit_min == replica.op_checkpoint());
+            assert(header_b == null or header_b.?.op == commit_b);
+
+            const checksum_a = state_checker.commits.items[commit_a].header.checksum;
+            const checksum_b = if (header_b) |h| h.checksum else commit_root;
+
+            assert(checksum_b != commit_root or replica.commit_min == replica.superblock.working.vsr_state.commit_min);
+            assert((commit_a == commit_b) == (checksum_a == checksum_b));
+
+            if (checksum_a == checksum_b) return;
+
+            assert(commit_b < commit_a or commit_a + 1 == commit_b);
+            state_checker.commit_mins[replica_index] = commit_b;
+
+            // If some other replica has already reached this state, then it will be in the commit history:
+            if (replica.commit_min < state_checker.commits.items.len) {
+                state_checker.commits.items[replica.commit_min].replicas.set(replica_index);
+
+                assert(replica.commit_min < state_checker.commits.items.len);
+                // A replica may transition more than once to the same state, for example, when
+                // restarting after a crash and replaying the log. The more important invariant is that
+                // the cluster as a whole may not transition to the same state more than once, and once
+                // transitioned may not regress.
+                log.info("{d:0>4}/{d:0>4} {x:0>32} > {x:0>32} {}", .{
+                    replica.commit_min,
+                    state_checker.requests_committed,
+                    checksum_a,
+                    checksum_b,
+                    replica_index,
+                });
+                return;
+            }
+
+            if (header_b == null) return;
+            assert(header_b.?.parent == checksum_a);
+            assert(header_b.?.op > 0);
+            assert(header_b.?.command == .prepare);
+            assert(header_b.?.operation != .reserved);
+
+            // The replica has transitioned to state `b` that is not yet in the commit history.
+            // Check if this is a valid new state based on the originating client's inflight request.
+            const client = for (state_checker.clients) |*client| {
+                if (client.id == header_b.?.client) break client;
+            } else unreachable;
+
+            if (client.request_queue.empty()) {
+                return error.ReplicaTransitionedToInvalidState;
+            }
+
+            const request = client.request_queue.head_ptr_const().?;
+            assert(request.message.header.client == header_b.?.client);
+            assert(request.message.header.request == header_b.?.request);
+            assert(request.message.header.command == .request);
+            assert(request.message.header.operation == header_b.?.operation);
+            assert(request.message.header.size == header_b.?.size);
+            // `checksum_body` will not match; the leader's StateMachine updated the timestamps in the
+            // prepare body's accounts/transfers.
+
+            state_checker.requests_committed += 1;
+            assert(state_checker.requests_committed == header_b.?.op);
+
+            log.info("     {d:0>4} {x:0>32} > {x:0>32} {}", .{
+                state_checker.requests_committed,
+                checksum_a,
+                checksum_b,
+                replica_index,
+            });
+
+            assert(state_checker.commits.items.len == header_b.?.op);
+            state_checker.commits.append(.{ .header = header_b.?.* }) catch unreachable;
+            state_checker.commits.items[header_b.?.op].replicas.set(replica_index);
+        }
+
+        pub fn convergence(state_checker: *Self) bool {
+            const a = state_checker.commits.items.len - 1;
+            for (state_checker.commit_mins[0..state_checker.replica_count]) |b| {
+                if (b != a) {
+                    return false;
+                }
+            }
+
+            for (state_checker.commits.items) |commit, i| {
+                assert(commit.replicas.count() == state_checker.replica_count);
+                assert(commit.header.command == .prepare);
+                assert(commit.header.op == i);
+                if (i > 0) {
+                    const previous = state_checker.commits.items[i - 1].header;
+                    assert(commit.header.parent == previous.checksum);
+                    assert(commit.header.view >= previous.view);
+                }
+            }
+            return true;
+        }
+    };
+}

package/src/tigerbeetle/src/testing/cluster/storage_checker.zig

@@ -0,0 +1,202 @@
+//! Verify deterministic storage.
+//!
+//! At each replica compact and checkpoint, check that storage is byte-for-byte identical across
+//! replicas.
+//!
+//! Areas verified at compaction (half-measure):
+//! - Acquired Grid blocks (ignores skipped recovery compactions)
+//!  TODO Because ManifestLog acquires blocks potentially several beats prior to actually writing
+//!  the block, this check will need to be removed or use a different strategy.
+//!
+//! Areas verified at checkpoint:
+//! - WAL prepares
+//! - SuperBlock Manifest, FreeSet, ClientTable
+//! - Acquired Grid blocks
+//!
+//! Areas not verified:
+//! - SuperBlock headers, which hold replica-specific state.
+//! - WAL headers, which may differ because the WAL writes deliberately corrupt redundant headers
+//!   to faulty slots to ensure recovery is consistent.
+//! - Non-allocated Grid blocks, which may differ due to state transfer.
+const std = @import("std");
+const assert = std.debug.assert;
+const log = std.log.scoped(.storage_checker);
+
+const constants = @import("../../constants.zig");
+const vsr = @import("../../vsr.zig");
+const superblock = @import("../../vsr/superblock.zig");
+const SuperBlockHeader = superblock.SuperBlockHeader;
+const Storage = @import("../storage.zig").Storage;
+
+/// After each compaction half measure, save the cumulative hash of all acquired grid blocks.
+///
+/// (Track half-measures instead of beats because the on-disk state mid-compaction is
+/// nondeterministic; it depends on IO progress.)
+const Compactions = std.ArrayList(u128);
+
+/// Maps from op_checkpoint to cumulative storage checksum.
+///
+/// Not every checkpoint is necessarily recorded — a replica calls on_checkpoint *at most* once.
+/// For example, a replica will not call on_checkpoint if it crashes (during a checkpoint) after
+/// writing 2 superblock copies. (This could be repeated by other replicas, causing a checkpoint
+/// op to be skipped in Checkpoints).
+const Checkpoints = std.AutoHashMap(u64, Checkpoint);
+
+const Checkpoint = struct {
+    // The superblock trailers are an XOR of all copies of all respective trailers, not the
+    // `SuperBlockHeader.{trailer}_checksum`.
+    checksum_superblock_manifest: u128,
+    checksum_superblock_free_set: u128,
+    checksum_superblock_client_table: u128,
+    checksum_wal_prepares: u128,
+    checksum_grid: u128,
+};
+
+pub fn StorageCheckerType(comptime Replica: type) type {
+    return struct {
+        const Self = @This();
+
+        allocator: std.mem.Allocator,
+        compactions: Compactions,
+        checkpoints: Checkpoints,
+
+        pub fn init(allocator: std.mem.Allocator) Self {
+            var compactions = Compactions.init(allocator);
+            errdefer compactions.deinit();
+
+            var checkpoints = Checkpoints.init(allocator);
+            errdefer checkpoints.deinit();
+
+            return Self{
+                .allocator = allocator,
+                .compactions = compactions,
+                .checkpoints = checkpoints,
+            };
+        }
+
+        pub fn deinit(checker: *Self) void {
+            checker.compactions.deinit();
+            checker.checkpoints.deinit();
+        }
+
+        pub fn replica_compact(checker: *Self, replica: *const Replica) !void {
+            // TODO(Beat Compaction) Remove when deterministic beat compaction is fixed.
+            // Until then this is too noisy.
+            if (1 == 1) return;
+
+            // If we are recovering from a crash, don't test the checksum until we are caught up.
+            // Until then our grid's checksum is too far ahead.
+            if (replica.superblock.working.vsr_state.op_compacted(replica.commit_min)) return;
+
+            // TODO(Beat Compaction) Remove when deterministic beat compaction is implemented.
+            const half_measure_beat_count = @divExact(constants.lsm_batch_multiple, 2);
+            if ((replica.commit_min + 1) % half_measure_beat_count != 0) return;
+
+            const checksum = checksum_grid(replica);
+            log.debug("{}: replica_compact: op={} area=grid checksum={}", .{
+                replica.replica,
+                replica.commit_min,
+                checksum,
+            });
+
+            // -1 since we never compact op=1.
+            const compactions_index = @divExact(replica.commit_min + 1, half_measure_beat_count) - 1;
+            if (compactions_index == checker.compactions.items.len) {
+                try checker.compactions.append(checksum);
+            } else {
+                const checksum_expect = checker.compactions.items[compactions_index];
+                if (checksum_expect != checksum) {
+                    log.err("{}: replica_compact: mismatch area=grid expect={} actual={}", .{
+                        replica.replica,
+                        checksum_expect,
+                        checksum,
+                    });
+                    return error.StorageMismatch;
+                }
+            }
+        }
+
+        pub fn replica_checkpoint(checker: *Self, replica: *const Replica) !void {
+            const storage = replica.superblock.storage;
+            const working = replica.superblock.working;
+
+            // TODO(Beat Compaction) Remove when deterministic storage is fixed.
+            // Until then this is too noisy.
+            if (1 == 1) return;
+
+            var checkpoint = Checkpoint{
+                .checksum_superblock_manifest = 0,
+                .checksum_superblock_free_set = 0,
+                .checksum_superblock_client_table = 0,
+                .checksum_wal_prepares = checksum_wal_prepares(storage),
+                .checksum_grid = checksum_grid(replica),
+            };
+
+            inline for (.{ .manifest, .free_set, .client_table }) |trailer| {
+                const trailer_area = @field(superblock.areas, trailer);
+                const trailer_size = @field(working, @tagName(trailer) ++ "_size");
+                var copy: u8 = 0;
+                while (copy < constants.superblock_copies) : (copy += 1) {
+                    @field(checkpoint, "checksum_superblock_" ++ @tagName(trailer.field)) |=
+                        vsr.checksum(storage.memory[trailer_area.offset(copy)..][0..trailer_size]);
+                }
+            }
+
+            inline for (std.meta.fields(Checkpoint)) |field| {
+                log.debug("{}: replica_checkpoint: checkpoint={} area={s} value={}", .{
+                    replica.replica,
+                    replica.op_checkpoint(),
+                    field.name,
+                    @field(checkpoint, field.name),
+                });
+            }
+
+            const checkpoint_expect = checker.checkpoints.get(replica.op_checkpoint()) orelse {
+                // This replica is the first to reach op_checkpoint.
+                try checker.checkpoints.putNoClobber(replica.op_checkpoint(), checkpoint);
+                return;
+            };
+
+            var fail: bool = false;
+            inline for (std.meta.fields(Checkpoint)) |field| {
+                const field_actual = @field(checkpoint, field.name);
+                const field_expect = @field(checkpoint_expect, field.name);
+                if (!std.meta.eql(field_expect, field_actual)) {
+                    fail = true;
+                    log.debug("{}: replica_checkpoint: mismatch area={s} expect={} actual={}", .{
+                        replica.replica,
+                        field.name,
+                        @field(checkpoint_expect, field.name),
+                        @field(checkpoint, field.name),
+                    });
+                }
+            }
+            if (fail) return error.StorageMismatch;
+        }
+
+        fn checksum_wal_prepares(storage: *const Storage) u128 {
+            var checksum: u128 = 0;
+            for (storage.wal_prepares()) |*prepare| {
+                assert(prepare.header.valid_checksum());
+                assert(prepare.header.command == .prepare);
+
+                // Only checksum the actual message header+body. Any leftover space is nondeterministic,
+                // because the current prepare may have overwritten a longer message.
+                checksum ^= vsr.checksum(std.mem.asBytes(prepare)[0..prepare.header.size]);
+            }
+            return checksum;
+        }
+
+        fn checksum_grid(replica: *const Replica) u128 {
+            const storage = replica.superblock.storage;
+            var acquired = replica.superblock.free_set.blocks.iterator(.{ .kind = .unset });
+            var checksum: u128 = 0;
+            while (acquired.next()) |address_index| {
+                const block = storage.grid_block(address_index + 1);
+                const block_header = std.mem.bytesToValue(vsr.Header, block[0..@sizeOf(vsr.Header)]);
+                checksum ^= vsr.checksum(block[0..block_header.size]);
+            }
+            return checksum;
+        }
+    };
+}