tigerbeetle-node 0.10.0 → 0.11.1

package/src/tigerbeetle/src/test/state_checker.zig

@@ -3,11 +3,13 @@ const assert = std.debug.assert;
 const mem = std.mem;
 
 const config = @import("../config.zig");
+const vsr = @import("../vsr.zig");
 
 const Cluster = @import("cluster.zig").Cluster;
 const Network = @import("network.zig").Network;
 const Storage = @import("storage.zig").Storage;
-const StateMachine = @import("state_machine.zig").StateMachineType(Storage);
+const Client = @import("cluster.zig").Client;
+const Replica = @import("cluster.zig").Replica;
 
 const message_pool = @import("../message_pool.zig");
 const MessagePool = message_pool.MessagePool;
@@ -15,47 +17,50 @@ const Message = MessagePool.Message;
 
 const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
 
-const RequestQueue = RingBuffer(u128, config.client_request_queue_max, .array);
 const StateTransitions = std.AutoHashMap(u128, u64);
 
 const log = std.log.scoped(.state_checker);
 
 pub const StateChecker = struct {
-    /// Indexed by client index as used by Cluster.
-    client_requests: [config.clients_max]RequestQueue =
-        [_]RequestQueue{.{}} ** config.clients_max,
-
     /// Indexed by replica index.
-    state_machine_states: [config.replicas_max]u128,
+    replica_states: [config.replicas_max]u128 = [_]u128{0} ** config.replicas_max,
 
+    /// Keyed by committed `message.header.checksum`.
+    ///
+    /// The first state is always `root_prepare.checksum`, since the root prepare doesn't
+    /// commit normally.
     history: StateTransitions,
 
-    /// The highest cannonical state reached by the cluster.
-    state: u128,
+    root: u128,
 
-    /// The number of times the cannonical state has been advanced.
-    transitions: u64 = 0,
+    // TODO When StateChecker is owned by the Simulation, use @fieldParentPtr to get these.
+    replicas: []const Replica,
+    clients: []const Client,
 
-    pub fn init(allocator: mem.Allocator, cluster: *Cluster) !StateChecker {
-        const state = cluster.replicas[0].state_machine.state;
+    /// The highest canonical state reached by the cluster.
+    state: u128 = 0,
 
-        var state_machine_states: [config.replicas_max]u128 = undefined;
-        for (cluster.replicas) |*replica, i| {
-            const state_machine = &replica.state_machine;
-            assert(state_machine.state == state);
-            state_machine_states[i] = state_machine.state;
-        }
+    /// The number of times the canonical state has been advanced.
+    requests_committed: u64 = 0,
 
+    pub fn init(
+        allocator: mem.Allocator,
+        cluster: u32,
+        replicas: []const Replica,
+        clients: []const Client,
+    ) !StateChecker {
         var history = StateTransitions.init(allocator);
         errdefer history.deinit();
 
+        const root_checksum = vsr.Header.root_prepare(cluster).checksum;
+
         var state_checker = StateChecker{
-            .state_machine_states = state_machine_states,
             .history = history,
-            .state = state,
+            .root = root_checksum,
+            .replicas = replicas,
+            .clients = clients,
         };
-
-        try state_checker.history.putNoClobber(state, state_checker.transitions);
+        try state_checker.history.putNoClobber(root_checksum, state_checker.requests_committed);
 
         return state_checker;
     }
@@ -64,14 +69,24 @@ pub const StateChecker = struct {
         state_checker.history.deinit();
     }
 
-    pub fn check_state(state_checker: *StateChecker, replica: u8) void {
-        const cluster = @fieldParentPtr(Cluster, "state_checker", state_checker);
+    pub fn check_state(state_checker: *StateChecker, replica_index: u8) !void {
+        const replica = state_checker.replicas[replica_index];
+        const commit_header = header: {
+            if (replica.journal.recovered) {
+                const commit_header = replica.journal.header_with_op(replica.commit_min);
+                assert(commit_header != null or replica.commit_min == replica.op_checkpoint);
+                break :header replica.journal.header_with_op(replica.commit_min);
+            } else {
+                // Still recovering.
+                break :header null;
+            }
+        };
 
-        const a = state_checker.state_machine_states[replica];
-        const b = cluster.replicas[replica].state_machine.state;
+        const a = state_checker.replica_states[replica_index];
+        const b = if (commit_header) |h| h.checksum else state_checker.root;
 
         if (b == a) return;
-        state_checker.state_machine_states[replica] = b;
+        state_checker.replica_states[replica_index] = b;
 
         // If some other replica has already reached this state, then it will be in the history:
         if (state_checker.history.get(b)) |transition| {
@@ -81,65 +96,72 @@ pub const StateChecker = struct {
             // transitioned may not regress.
             log.info(
                 "{d:0>4}/{d:0>4} {x:0>32} > {x:0>32} {}",
-                .{ transition, state_checker.transitions, a, b, replica },
+                .{ transition, state_checker.requests_committed, a, b, replica_index },
             );
             return;
         }
 
+        if (commit_header == null) return;
+        assert(commit_header.?.parent == a);
+        assert(commit_header.?.op > 0);
+        assert(commit_header.?.command == .prepare);
+        assert(commit_header.?.operation != .reserved);
+
         // The replica has transitioned to state `b` that is not yet in the history.
-        // Check if this is a valid new state based on all currently inflight client requests.
-        for (state_checker.client_requests) |*queue| {
-            if (queue.head_ptr()) |input| {
-                if (b == StateMachine.hash(state_checker.state, std.mem.asBytes(input))) {
-                    const transitions_executed = state_checker.history.get(a).?;
-                    if (transitions_executed < state_checker.transitions) {
-                        @panic("replica skipped interim transitions");
-                    } else {
-                        assert(transitions_executed == state_checker.transitions);
-                    }
-
-                    state_checker.state = b;
-                    state_checker.transitions += 1;
-
-                    log.info("     {d:0>4} {x:0>32} > {x:0>32} {}", .{
-                        state_checker.transitions,
-                        a,
-                        b,
-                        replica,
-                    });
-
-                    state_checker.history.putNoClobber(b, state_checker.transitions) catch {
-                        @panic("state checker unable to allocate memory for history.put()");
-                    };
-
-                    // As soon as we reach a valid state we must pop the inflight request.
-                    // We cannot wait until the client receives the reply because that would allow
-                    // the inflight request to be used to reach other states in the interim.
-                    // We must therefore use our own queue rather than the clients' queues.
-                    _ = queue.pop();
-                    return;
-                }
-            }
+        // Check if this is a valid new state based on the originating client's inflight request.
+        const client = for (state_checker.clients) |*client| {
+            if (client.id == commit_header.?.client) break client;
+        } else unreachable;
+
+        if (client.request_queue.empty()) {
+            return error.ReplicaTransitionedToInvalidState;
         }
 
-        @panic("replica transitioned to an invalid state");
+        const request = client.request_queue.head_ptr_const().?;
+        assert(request.message.header.client == commit_header.?.client);
+        assert(request.message.header.request == commit_header.?.request);
+        assert(request.message.header.command == .request);
+        assert(request.message.header.operation == commit_header.?.operation);
+        assert(request.message.header.size == commit_header.?.size);
+        // `checksum_body` will not match; the leader's StateMachine updated the timestamps in the
+        // prepare body's accounts/transfers.
+
+        const transitions_executed = state_checker.history.get(a).?;
+        if (transitions_executed < state_checker.requests_committed) {
+            return error.ReplicaSkippedInterimTransitions;
+        } else {
+            assert(transitions_executed == state_checker.requests_committed);
+        }
+
+        state_checker.state = b;
+        state_checker.requests_committed += 1;
+        assert(state_checker.requests_committed == commit_header.?.op);
+
+        log.info("     {d:0>4} {x:0>32} > {x:0>32} {}", .{
+            state_checker.requests_committed,
+            a,
+            b,
+            replica_index,
+        });
+
+        state_checker.history.putNoClobber(b, state_checker.requests_committed) catch {
+            @panic("state checker unable to allocate memory for history.put()");
+        };
     }
 
     pub fn convergence(state_checker: *StateChecker) bool {
-        const cluster = @fieldParentPtr(Cluster, "state_checker", state_checker);
-
-        const a = state_checker.state_machine_states[0];
-        for (state_checker.state_machine_states[1..cluster.options.replica_count]) |b| {
+        const a = state_checker.replica_states[0];
+        for (state_checker.replica_states[1..state_checker.replicas[0].replica_count]) |b| {
             if (b != a) return false;
         }
 
         const transitions_executed = state_checker.history.get(a).?;
-        if (transitions_executed < state_checker.transitions) {
+        if (transitions_executed < state_checker.requests_committed) {
             // Cluster reached convergence but on a regressed state.
             // A replica reached the transition limit, crashed, then repaired.
             return false;
         } else {
-            assert(transitions_executed == state_checker.transitions);
+            assert(transitions_executed == state_checker.requests_committed);
         }
 
         return true;

package/src/tigerbeetle/src/test/state_machine.zig

@@ -15,7 +15,7 @@ pub fn StateMachineType(comptime Storage: type) type {
             root,
             register,
 
-            hash,
+            random,
         };
 
         /// Minimum/mean number of ticks to perform the specified operation.
@@ -28,7 +28,6 @@ pub fn StateMachineType(comptime Storage: type) type {
         };
 
         options: Options,
-        state: u128,
         prng: std.rand.DefaultPrng,
         prepare_timestamp: u64 = 0,
         commit_timestamp: u64 = 0,
@@ -38,7 +37,6 @@ pub fn StateMachineType(comptime Storage: type) type {
 
         pub fn init(_: std.mem.Allocator, _: *Grid, options: Options) !StateMachine {
             return StateMachine{
-                .state = hash(0, std.mem.asBytes(&options.seed)),
                 .options = options,
                 .prng = std.rand.DefaultPrng.init(options.seed),
             };
@@ -46,6 +44,10 @@ pub fn StateMachineType(comptime Storage: type) type {
 
         pub fn deinit(_: *StateMachine, _: std.mem.Allocator) void {}
 
+        // TODO This is dead code — tick() has been removed from the StateMachine
+        // interface. If we start using the test StateMachine again, tick will need
+        // to be called explicitly from the simulator to ensure async operations can
+        // finish.
         pub fn tick(state_machine: *StateMachine) void {
             if (state_machine.callback) |callback| {
                 if (state_machine.callback_ticks == 0) {
@@ -98,32 +100,16 @@ pub fn StateMachineType(comptime Storage: type) type {
             input: []const u8,
             output: []u8,
         ) usize {
-            _ = op;
+            _ = state_machine;
+            _ = client;
+            _ = input;
+            _ = output;
+            assert(op != 0);
 
             switch (operation) {
                 .reserved, .root => unreachable,
                 .register => return 0,
-
-                // TODO: instead of always using the first 32 bytes of the output
-                // buffer, get tricky and use a random but deterministic slice
-                // of it, filling the rest with 0s.
-                .hash => {
-                    // Fold the input into our current state, creating a hash chain.
-                    // Hash the input with the client ID since small inputs may collide across clients.
-                    const client_input = hash(client, input);
-                    const new_state = hash(state_machine.state, std.mem.asBytes(&client_input));
-
-                    log.debug("state={x} input={x} input.len={} new state={x}", .{
-                        state_machine.state,
-                        client_input,
-                        input.len,
-                        new_state,
-                    });
-
-                    state_machine.state = new_state;
-                    std.mem.copy(u8, output, std.mem.asBytes(&state_machine.state));
-                    return @sizeOf(@TypeOf(state_machine.state));
-                },
+                .random => return 0,
             }
         }
 
@@ -142,23 +128,13 @@ pub fn StateMachineType(comptime Storage: type) type {
         pub fn checkpoint(
             state_machine: *StateMachine,
             callback: fn (*StateMachine) void,
-            op: u64,
         ) void {
-            _ = op;
             assert(state_machine.callback == null);
             assert(state_machine.callback_ticks == 0);
             state_machine.callback = callback;
             state_machine.callback_ticks = state_machine.latency(state_machine.options.checkpoint_mean);
         }
 
-        pub fn hash(state: u128, input: []const u8) u128 {
-            var key: [32]u8 = [_]u8{0} ** 32;
-            std.mem.copy(u8, key[0..16], std.mem.asBytes(&state));
-            var target: [32]u8 = undefined;
-            std.crypto.hash.Blake3.hash(input, &target, .{ .key = key });
-            return @bitCast(u128, target[0..16].*);
-        }
-
         fn latency(state_machine: *StateMachine, mean: u64) u64 {
             return state_machine.prng.random().uintLessThan(u64, 2 * mean);
         }