tigerbeetle-node 0.11.12 → 0.12.0

package/src/tigerbeetle/src/testing/cluster/network.zig

@@ -1,237 +0,0 @@
-const std = @import("std");
-const math = std.math;
-const mem = std.mem;
-const assert = std.debug.assert;
-
-const constants = @import("../../constants.zig");
-const vsr = @import("../../vsr.zig");
-const stdx = @import("../../stdx.zig");
-
-const MessagePool = @import("../../message_pool.zig").MessagePool;
-const Message = MessagePool.Message;
-
-const MessageBus = @import("message_bus.zig").MessageBus;
-const Process = @import("message_bus.zig").Process;
-
-const PacketSimulatorType = @import("../packet_simulator.zig").PacketSimulatorType;
-const PacketSimulatorOptions = @import("../packet_simulator.zig").PacketSimulatorOptions;
-const PacketSimulatorPath = @import("../packet_simulator.zig").Path;
-
-const log = std.log.scoped(.network);
-
-pub const NetworkOptions = PacketSimulatorOptions;
-
-pub const Network = struct {
-    pub const Packet = struct {
-        network: *Network,
-        message: *Message,
-
-        pub fn deinit(packet: *const Packet) void {
-            packet.network.message_pool.unref(packet.message);
-        }
-    };
-
-    pub const Path = struct {
-        source: Process,
-        target: Process,
-    };
-
-    allocator: std.mem.Allocator,
-
-    options: NetworkOptions,
-    packet_simulator: PacketSimulatorType(Packet),
-
-    // TODO(Zig) If this stored a ?*MessageBus, then a process's bus could be set to `null` while
-    // the replica is crashed, and replaced when it is destroyed. But Zig complains:
-    //
-    //   ./src/test/message_bus.zig:20:24: error: struct 'test.message_bus.MessageBus' depends on itself
-    //   pub const MessageBus = struct {
-    //                          ^
-    //   ./src/test/message_bus.zig:21:5: note: while checking this field
-    //       network: *Network,
-    //       ^
-    buses: std.ArrayListUnmanaged(*MessageBus),
-    buses_enabled: std.ArrayListUnmanaged(bool),
-    processes: std.ArrayListUnmanaged(u128),
-    /// A pool of messages that are in the network (sent, but not yet delivered).
-    message_pool: MessagePool,
-
-    pub fn init(
-        allocator: std.mem.Allocator,
-        replica_count: u8,
-        client_count: u8,
-        options: NetworkOptions,
-    ) !Network {
-        const process_count = client_count + replica_count;
-        assert(process_count <= std.math.maxInt(u8));
-
-        var buses = try std.ArrayListUnmanaged(*MessageBus).initCapacity(allocator, process_count);
-        errdefer buses.deinit(allocator);
-
-        var buses_enabled = try std.ArrayListUnmanaged(bool).initCapacity(allocator, process_count);
-        errdefer buses_enabled.deinit(allocator);
-
-        var processes = try std.ArrayListUnmanaged(u128).initCapacity(allocator, process_count);
-        errdefer processes.deinit(allocator);
-
-        var packet_simulator = try PacketSimulatorType(Packet).init(allocator, options);
-        errdefer packet_simulator.deinit(allocator);
-
-        // Count:
-        // - replica → replica paths (excluding self-loops)
-        // - replica → client paths
-        // - client → replica paths
-        // but not client→client paths; clients never message one another.
-        const path_count = @as(usize, replica_count) * @as(usize, replica_count - 1) +
-            2 * @as(usize, replica_count) * @as(usize, client_count);
-        const message_pool = try MessagePool.init_capacity(
-            allocator,
-            // +1 so we can allocate an extra packet when all packet queues are at capacity,
-            // so that `PacketSimulator.submit_packet` can choose which packet to drop.
-            1 + @as(usize, options.path_maximum_capacity) * path_count,
-        );
-        errdefer message_pool.deinit(allocator);
-
-        return Network{
-            .allocator = allocator,
-            .options = options,
-            .packet_simulator = packet_simulator,
-            .buses = buses,
-            .buses_enabled = buses_enabled,
-            .processes = processes,
-            .message_pool = message_pool,
-        };
-    }
-
-    pub fn deinit(network: *Network) void {
-        network.buses.deinit(network.allocator);
-        network.buses_enabled.deinit(network.allocator);
-        network.processes.deinit(network.allocator);
-        network.packet_simulator.deinit(network.allocator);
-        network.message_pool.deinit(network.allocator);
-    }
-
-    pub fn tick(network: *Network) void {
-        network.packet_simulator.tick();
-    }
-
-    pub fn link(network: *Network, process: Process, message_bus: *MessageBus) void {
-        const raw_process = switch (process) {
-            .replica => |replica| replica,
-            .client => |client| blk: {
-                assert(client >= constants.replicas_max);
-                break :blk client;
-            },
-        };
-
-        for (network.processes.items) |existing_process, i| {
-            if (existing_process == raw_process) {
-                network.buses.items[i] = message_bus;
-                break;
-            }
-        } else {
-            network.processes.appendAssumeCapacity(raw_process);
-            network.buses.appendAssumeCapacity(message_bus);
-            network.buses_enabled.appendAssumeCapacity(true);
-        }
-        assert(network.processes.items.len == network.buses.items.len);
-    }
-
-    pub fn process_enable(network: *Network, process: Process) void {
-        assert(!network.buses_enabled.items[network.process_to_address(process)]);
-        network.buses_enabled.items[network.process_to_address(process)] = true;
-    }
-
-    pub fn process_disable(network: *Network, process: Process) void {
-        assert(network.buses_enabled.items[network.process_to_address(process)]);
-        network.buses_enabled.items[network.process_to_address(process)] = false;
-    }
-
-    pub fn send_message(network: *Network, message: *Message, path: Path) void {
-        log.debug("send_message: {} > {}: {}", .{
-            path.source,
-            path.target,
-            message.header.command,
-        });
-
-        const network_message = network.message_pool.get_message();
-        defer network.message_pool.unref(network_message);
-
-        stdx.copy_disjoint(.exact, u8, network_message.buffer, message.buffer);
-
-        network.packet_simulator.submit_packet(
-            .{
-                .message = network_message.ref(),
-                .network = network,
-            },
-            deliver_message,
-            .{
-                .source = network.process_to_address(path.source),
-                .target = network.process_to_address(path.target),
-            },
-        );
-    }
-
-    fn process_to_address(network: *const Network, process: Process) u8 {
-        for (network.processes.items) |p, i| {
-            if (std.meta.eql(raw_process_to_process(p), process)) return @intCast(u8, i);
-        }
-        unreachable;
-    }
-
-    pub fn get_message_bus(network: *Network, process: Process) *MessageBus {
-        return network.buses.items[network.process_to_address(process)];
-    }
-
-    fn deliver_message(packet: Packet, path: PacketSimulatorPath) void {
-        const network = packet.network;
-        const process_path = .{
-            .source = raw_process_to_process(network.processes.items[path.source]),
-            .target = raw_process_to_process(network.processes.items[path.target]),
-        };
-
-        if (!network.buses_enabled.items[path.target]) {
-            log.debug("deliver_message: {} > {}: {} (dropped; target is down)", .{
-                process_path.source,
-                process_path.target,
-                packet.message.header.command,
-            });
-            return;
-        }
-
-        const target_bus = network.buses.items[path.target];
-        const target_message = target_bus.get_message();
-        defer target_bus.unref(target_message);
-
-        stdx.copy_disjoint(.exact, u8, target_message.buffer, packet.message.buffer);
-
-        log.debug("deliver_message: {} > {}: {}", .{
-            process_path.source,
-            process_path.target,
-            packet.message.header.command,
-        });
-
-        if (target_message.header.command == .request or
-            target_message.header.command == .prepare)
-        {
-            const sector_ceil = vsr.sector_ceil(target_message.header.size);
-            if (target_message.header.size != sector_ceil) {
-                assert(target_message.header.size < sector_ceil);
-                assert(target_message.buffer.len == constants.message_size_max);
-                mem.set(u8, target_message.buffer[target_message.header.size..sector_ceil], 0);
-            }
-        }
-
-        target_bus.on_message_callback(target_bus, target_message);
-    }
-
-    fn raw_process_to_process(raw: u128) Process {
-        switch (raw) {
-            0...(constants.replicas_max - 1) => return .{ .replica = @intCast(u8, raw) },
-            else => {
-                assert(raw >= constants.replicas_max);
-                return .{ .client = raw };
-            },
-        }
-    }
-};

package/src/tigerbeetle/src/testing/cluster/state_checker.zig

@@ -1,169 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const mem = std.mem;
-
-const constants = @import("../../constants.zig");
-const vsr = @import("../../vsr.zig");
-const RingBuffer = @import("../../ring_buffer.zig").RingBuffer;
-
-const message_pool = @import("../../message_pool.zig");
-const MessagePool = message_pool.MessagePool;
-const Message = MessagePool.Message;
-
-const ReplicaSet = std.StaticBitSet(constants.replicas_max);
-const Commits = std.ArrayList(struct {
-    header: vsr.Header,
-    replicas: ReplicaSet = ReplicaSet.initEmpty(),
-});
-
-const log = std.log.scoped(.state_checker);
-
-pub fn StateCheckerType(comptime Client: type, comptime Replica: type) type {
-    return struct {
-        const Self = @This();
-
-        replica_count: u8,
-
-        commits: Commits,
-        commit_mins: [constants.replicas_max]u64 = [_]u64{0} ** constants.replicas_max,
-
-        replicas: []const Replica,
-        clients: []const Client,
-
-        /// The number of times the canonical state has been advanced.
-        requests_committed: u64 = 0,
-
-        pub fn init(
-            allocator: mem.Allocator,
-            cluster: u32,
-            replicas: []const Replica,
-            clients: []const Client,
-        ) !Self {
-            const root_prepare = vsr.Header.root_prepare(cluster);
-
-            var commits = Commits.init(allocator);
-            errdefer commits.deinit();
-
-            var commit_replicas = ReplicaSet.initEmpty();
-            for (replicas) |_, i| commit_replicas.set(i);
-            try commits.append(.{
-                .header = root_prepare,
-                .replicas = commit_replicas,
-            });
-
-            return Self{
-                .replica_count = @intCast(u8, replicas.len),
-                .commits = commits,
-                .replicas = replicas,
-                .clients = clients,
-            };
-        }
-
-        pub fn deinit(state_checker: *Self) void {
-            state_checker.commits.deinit();
-        }
-
-        pub fn check_state(state_checker: *Self, replica_index: u8) !void {
-            const replica = &state_checker.replicas[replica_index];
-
-            const commit_root = replica.superblock.working.vsr_state.commit_min_checksum;
-
-            const commit_a = state_checker.commit_mins[replica_index];
-            const commit_b = replica.commit_min;
-
-            const header_b = replica.journal.header_with_op(replica.commit_min);
-            assert(header_b != null or replica.commit_min == replica.op_checkpoint());
-            assert(header_b == null or header_b.?.op == commit_b);
-
-            const checksum_a = state_checker.commits.items[commit_a].header.checksum;
-            const checksum_b = if (header_b) |h| h.checksum else commit_root;
-
-            assert(checksum_b != commit_root or replica.commit_min == replica.superblock.working.vsr_state.commit_min);
-            assert((commit_a == commit_b) == (checksum_a == checksum_b));
-
-            if (checksum_a == checksum_b) return;
-
-            assert(commit_b < commit_a or commit_a + 1 == commit_b);
-            state_checker.commit_mins[replica_index] = commit_b;
-
-            // If some other replica has already reached this state, then it will be in the commit history:
-            if (replica.commit_min < state_checker.commits.items.len) {
-                state_checker.commits.items[replica.commit_min].replicas.set(replica_index);
-
-                assert(replica.commit_min < state_checker.commits.items.len);
-                // A replica may transition more than once to the same state, for example, when
-                // restarting after a crash and replaying the log. The more important invariant is that
-                // the cluster as a whole may not transition to the same state more than once, and once
-                // transitioned may not regress.
-                log.info("{d:0>4}/{d:0>4} {x:0>32} > {x:0>32} {}", .{
-                    replica.commit_min,
-                    state_checker.requests_committed,
-                    checksum_a,
-                    checksum_b,
-                    replica_index,
-                });
-                return;
-            }
-
-            if (header_b == null) return;
-            assert(header_b.?.parent == checksum_a);
-            assert(header_b.?.op > 0);
-            assert(header_b.?.command == .prepare);
-            assert(header_b.?.operation != .reserved);
-
-            // The replica has transitioned to state `b` that is not yet in the commit history.
-            // Check if this is a valid new state based on the originating client's inflight request.
-            const client = for (state_checker.clients) |*client| {
-                if (client.id == header_b.?.client) break client;
-            } else unreachable;
-
-            if (client.request_queue.empty()) {
-                return error.ReplicaTransitionedToInvalidState;
-            }
-
-            const request = client.request_queue.head_ptr_const().?;
-            assert(request.message.header.client == header_b.?.client);
-            assert(request.message.header.request == header_b.?.request);
-            assert(request.message.header.command == .request);
-            assert(request.message.header.operation == header_b.?.operation);
-            assert(request.message.header.size == header_b.?.size);
-            // `checksum_body` will not match; the leader's StateMachine updated the timestamps in the
-            // prepare body's accounts/transfers.
-
-            state_checker.requests_committed += 1;
-            assert(state_checker.requests_committed == header_b.?.op);
-
-            log.info("     {d:0>4} {x:0>32} > {x:0>32} {}", .{
-                state_checker.requests_committed,
-                checksum_a,
-                checksum_b,
-                replica_index,
-            });
-
-            assert(state_checker.commits.items.len == header_b.?.op);
-            state_checker.commits.append(.{ .header = header_b.?.* }) catch unreachable;
-            state_checker.commits.items[header_b.?.op].replicas.set(replica_index);
-        }
-
-        pub fn convergence(state_checker: *Self) bool {
-            const a = state_checker.commits.items.len - 1;
-            for (state_checker.commit_mins[0..state_checker.replica_count]) |b| {
-                if (b != a) {
-                    return false;
-                }
-            }
-
-            for (state_checker.commits.items) |commit, i| {
-                assert(commit.replicas.count() == state_checker.replica_count);
-                assert(commit.header.command == .prepare);
-                assert(commit.header.op == i);
-                if (i > 0) {
-                    const previous = state_checker.commits.items[i - 1].header;
-                    assert(commit.header.parent == previous.checksum);
-                    assert(commit.header.view >= previous.view);
-                }
-            }
-            return true;
-        }
-    };
-}

package/src/tigerbeetle/src/testing/cluster/storage_checker.zig

@@ -1,202 +0,0 @@
-//! Verify deterministic storage.
-//!
-//! At each replica compact and checkpoint, check that storage is byte-for-byte identical across
-//! replicas.
-//!
-//! Areas verified at compaction (half-measure):
-//! - Acquired Grid blocks (ignores skipped recovery compactions)
-//!  TODO Because ManifestLog acquires blocks potentially several beats prior to actually writing
-//!  the block, this check will need to be removed or use a different strategy.
-//!
-//! Areas verified at checkpoint:
-//! - WAL prepares
-//! - SuperBlock Manifest, FreeSet, ClientTable
-//! - Acquired Grid blocks
-//!
-//! Areas not verified:
-//! - SuperBlock headers, which hold replica-specific state.
-//! - WAL headers, which may differ because the WAL writes deliberately corrupt redundant headers
-//!   to faulty slots to ensure recovery is consistent.
-//! - Non-allocated Grid blocks, which may differ due to state transfer.
-const std = @import("std");
-const assert = std.debug.assert;
-const log = std.log.scoped(.storage_checker);
-
-const constants = @import("../../constants.zig");
-const vsr = @import("../../vsr.zig");
-const superblock = @import("../../vsr/superblock.zig");
-const SuperBlockHeader = superblock.SuperBlockHeader;
-const Storage = @import("../storage.zig").Storage;
-
-/// After each compaction half measure, save the cumulative hash of all acquired grid blocks.
-///
-/// (Track half-measures instead of beats because the on-disk state mid-compaction is
-/// nondeterministic; it depends on IO progress.)
-const Compactions = std.ArrayList(u128);
-
-/// Maps from op_checkpoint to cumulative storage checksum.
-///
-/// Not every checkpoint is necessarily recorded — a replica calls on_checkpoint *at most* once.
-/// For example, a replica will not call on_checkpoint if it crashes (during a checkpoint) after
-/// writing 2 superblock copies. (This could be repeated by other replicas, causing a checkpoint
-/// op to be skipped in Checkpoints).
-const Checkpoints = std.AutoHashMap(u64, Checkpoint);
-
-const Checkpoint = struct {
-    // The superblock trailers are an XOR of all copies of all respective trailers, not the
-    // `SuperBlockHeader.{trailer}_checksum`.
-    checksum_superblock_manifest: u128,
-    checksum_superblock_free_set: u128,
-    checksum_superblock_client_table: u128,
-    checksum_wal_prepares: u128,
-    checksum_grid: u128,
-};
-
-pub fn StorageCheckerType(comptime Replica: type) type {
-    return struct {
-        const Self = @This();
-
-        allocator: std.mem.Allocator,
-        compactions: Compactions,
-        checkpoints: Checkpoints,
-
-        pub fn init(allocator: std.mem.Allocator) Self {
-            var compactions = Compactions.init(allocator);
-            errdefer compactions.deinit();
-
-            var checkpoints = Checkpoints.init(allocator);
-            errdefer checkpoints.deinit();
-
-            return Self{
-                .allocator = allocator,
-                .compactions = compactions,
-                .checkpoints = checkpoints,
-            };
-        }
-
-        pub fn deinit(checker: *Self) void {
-            checker.compactions.deinit();
-            checker.checkpoints.deinit();
-        }
-
-        pub fn replica_compact(checker: *Self, replica: *const Replica) !void {
-            // TODO(Beat Compaction) Remove when deterministic beat compaction is fixed.
-            // Until then this is too noisy.
-            if (1 == 1) return;
-
-            // If we are recovering from a crash, don't test the checksum until we are caught up.
-            // Until then our grid's checksum is too far ahead.
-            if (replica.superblock.working.vsr_state.op_compacted(replica.commit_min)) return;
-
-            // TODO(Beat Compaction) Remove when deterministic beat compaction is implemented.
-            const half_measure_beat_count = @divExact(constants.lsm_batch_multiple, 2);
-            if ((replica.commit_min + 1) % half_measure_beat_count != 0) return;
-
-            const checksum = checksum_grid(replica);
-            log.debug("{}: replica_compact: op={} area=grid checksum={}", .{
-                replica.replica,
-                replica.commit_min,
-                checksum,
-            });
-
-            // -1 since we never compact op=1.
-            const compactions_index = @divExact(replica.commit_min + 1, half_measure_beat_count) - 1;
-            if (compactions_index == checker.compactions.items.len) {
-                try checker.compactions.append(checksum);
-            } else {
-                const checksum_expect = checker.compactions.items[compactions_index];
-                if (checksum_expect != checksum) {
-                    log.err("{}: replica_compact: mismatch area=grid expect={} actual={}", .{
-                        replica.replica,
-                        checksum_expect,
-                        checksum,
-                    });
-                    return error.StorageMismatch;
-                }
-            }
-        }
-
-        pub fn replica_checkpoint(checker: *Self, replica: *const Replica) !void {
-            const storage = replica.superblock.storage;
-            const working = replica.superblock.working;
-
-            // TODO(Beat Compaction) Remove when deterministic storage is fixed.
-            // Until then this is too noisy.
-            if (1 == 1) return;
-
-            var checkpoint = Checkpoint{
-                .checksum_superblock_manifest = 0,
-                .checksum_superblock_free_set = 0,
-                .checksum_superblock_client_table = 0,
-                .checksum_wal_prepares = checksum_wal_prepares(storage),
-                .checksum_grid = checksum_grid(replica),
-            };
-
-            inline for (.{ .manifest, .free_set, .client_table }) |trailer| {
-                const trailer_area = @field(superblock.areas, trailer);
-                const trailer_size = @field(working, @tagName(trailer) ++ "_size");
-                var copy: u8 = 0;
-                while (copy < constants.superblock_copies) : (copy += 1) {
-                    @field(checkpoint, "checksum_superblock_" ++ @tagName(trailer.field)) |=
-                        vsr.checksum(storage.memory[trailer_area.offset(copy)..][0..trailer_size]);
-                }
-            }
-
-            inline for (std.meta.fields(Checkpoint)) |field| {
-                log.debug("{}: replica_checkpoint: checkpoint={} area={s} value={}", .{
-                    replica.replica,
-                    replica.op_checkpoint(),
-                    field.name,
-                    @field(checkpoint, field.name),
-                });
-            }
-
-            const checkpoint_expect = checker.checkpoints.get(replica.op_checkpoint()) orelse {
-                // This replica is the first to reach op_checkpoint.
-                try checker.checkpoints.putNoClobber(replica.op_checkpoint(), checkpoint);
-                return;
-            };
-
-            var fail: bool = false;
-            inline for (std.meta.fields(Checkpoint)) |field| {
-                const field_actual = @field(checkpoint, field.name);
-                const field_expect = @field(checkpoint_expect, field.name);
-                if (!std.meta.eql(field_expect, field_actual)) {
-                    fail = true;
-                    log.debug("{}: replica_checkpoint: mismatch area={s} expect={} actual={}", .{
-                        replica.replica,
-                        field.name,
-                        @field(checkpoint_expect, field.name),
-                        @field(checkpoint, field.name),
-                    });
-                }
-            }
-            if (fail) return error.StorageMismatch;
-        }
-
-        fn checksum_wal_prepares(storage: *const Storage) u128 {
-            var checksum: u128 = 0;
-            for (storage.wal_prepares()) |*prepare| {
-                assert(prepare.header.valid_checksum());
-                assert(prepare.header.command == .prepare);
-
-                // Only checksum the actual message header+body. Any leftover space is nondeterministic,
-                // because the current prepare may have overwritten a longer message.
-                checksum ^= vsr.checksum(std.mem.asBytes(prepare)[0..prepare.header.size]);
-            }
-            return checksum;
-        }
-
-        fn checksum_grid(replica: *const Replica) u128 {
-            const storage = replica.superblock.storage;
-            var acquired = replica.superblock.free_set.blocks.iterator(.{ .kind = .unset });
-            var checksum: u128 = 0;
-            while (acquired.next()) |address_index| {
-                const block = storage.grid_block(address_index + 1);
-                const block_header = std.mem.bytesToValue(vsr.Header, block[0..@sizeOf(vsr.Header)]);
-                checksum ^= vsr.checksum(block[0..block_header.size]);
-            }
-            return checksum;
-        }
-    };
-}