tigerbeetle-node 0.3.3 → 0.5.0

package/src/tigerbeetle/src/test/packet_simulator.zig

@@ -0,0 +1,371 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const math = std.math;
+
+const log = std.log.scoped(.packet_simulator);
+
+pub const PacketSimulatorOptions = struct {
+    /// Mean for the exponential distribution used to calculate forward delay.
+    one_way_delay_mean: u64,
+    one_way_delay_min: u64,
+
+    packet_loss_probability: u8,
+    packet_replay_probability: u8,
+    seed: u64,
+
+    replica_count: u8,
+    client_count: u8,
+    node_count: u8,
+
+    /// How the partitions should be generated
+    partition_mode: PartitionMode,
+
+    /// Probability per tick that a partition will occur
+    partition_probability: u8,
+
+    /// Probability per tick that a partition will resolve
+    unpartition_probability: u8,
+
+    /// Minimum time a partition lasts
+    partition_stability: u32,
+
+    /// Minimum time the cluster is fully connected until it is partitioned again
+    unpartition_stability: u32,
+
+    /// The maximum number of in-flight packets a path can have before packets are randomly dropped.
+    path_maximum_capacity: u8,
+
+    /// Mean for the exponential distribution used to calculate how long a path is clogged for.
+    path_clog_duration_mean: u64,
+    path_clog_probability: u8,
+};
+
+pub const Path = struct {
+    source: u8,
+    target: u8,
+};
+
+/// Determines how the partitions are created. Partitions
+/// are two-way, i.e. if i cannot communicate with j, then
+/// j cannot communicate with i.
+///
+/// Only replicas are partitioned. There will always be exactly two partitions.
+pub const PartitionMode = enum {
+    /// Draws the size of the partition uniformly at random from (1, n-1).
+    /// Replicas are randomly assigned a partition.
+    uniform_size,
+
+    /// Assigns each node to a partition uniformly at random. This biases towards
+    /// equal-size partitions.
+    uniform_partition,
+
+    /// Isolates exactly one replica.
+    isolate_single,
+
+    /// User-defined partitioning algorithm.
+    custom,
+};
+
+/// A fully connected network of nodes used for testing. Simulates the fault model:
+/// Packets may be dropped.
+/// Packets may be delayed.
+/// Packets may be replayed.
+pub const PacketStatistics = enum(u8) {
+    dropped_due_to_partition,
+    dropped_due_to_congestion,
+    dropped,
+    replay,
+};
+pub fn PacketSimulator(comptime Packet: type) type {
+    return struct {
+        const Self = @This();
+
+        const Data = struct {
+            expiry: u64,
+            callback: fn (packet: Packet, path: Path) void,
+            packet: Packet,
+        };
+
+        /// A send and receive path between each node in the network. We use the `path` function to
+        /// index it.
+        paths: []std.PriorityQueue(Data),
+
+        /// We can arbitrary clog a path until a tick.
+        path_clogged_till: []u64,
+        ticks: u64 = 0,
+        options: PacketSimulatorOptions,
+        prng: std.rand.DefaultPrng,
+        stats: [@typeInfo(PacketStatistics).Enum.fields.len]u32 = [_]u32{0} **
+            @typeInfo(PacketStatistics).Enum.fields.len,
+
+        is_partitioned: bool,
+        partition: []bool,
+        replicas: []u8,
+        stability: u32,
+
+        pub fn init(allocator: *std.mem.Allocator, options: PacketSimulatorOptions) !Self {
+            assert(options.one_way_delay_mean >= options.one_way_delay_min);
+            var self = Self{
+                .paths = try allocator.alloc(
+                    std.PriorityQueue(Data),
+                    @as(usize, options.node_count) * options.node_count,
+                ),
+                .path_clogged_till = try allocator.alloc(
+                    u64,
+                    @as(usize, options.node_count) * options.node_count,
+                ),
+                .options = options,
+                .prng = std.rand.DefaultPrng.init(options.seed),
+
+                .is_partitioned = false,
+                .stability = options.unpartition_stability,
+                .partition = try allocator.alloc(bool, @as(usize, options.replica_count)),
+                .replicas = try allocator.alloc(u8, @as(usize, options.replica_count)),
+            };
+
+            for (self.replicas) |_, i| {
+                self.replicas[i] = @intCast(u8, i);
+            }
+
+            for (self.paths) |*queue| {
+                queue.* = std.PriorityQueue(Data).init(allocator, Self.order_packets);
+                try queue.ensureCapacity(options.path_maximum_capacity);
+            }
+
+            for (self.path_clogged_till) |*clogged_till| {
+                clogged_till.* = 0;
+            }
+
+            return self;
+        }
+
+        pub fn deinit(self: *Self, allocator: *std.mem.Allocator) void {
+            for (self.paths) |*queue| {
+                while (queue.popOrNull()) |*data| data.packet.deinit();
+                queue.deinit();
+            }
+            allocator.free(self.paths);
+        }
+
+        fn order_packets(a: Data, b: Data) math.Order {
+            return math.order(a.expiry, b.expiry);
+        }
+
+        fn should_drop(self: *Self) bool {
+            return self.prng.random.uintAtMost(u8, 100) < self.options.packet_loss_probability;
+        }
+
+        fn path_index(self: *Self, path: Path) usize {
+            assert(path.source < self.options.node_count and path.target < self.options.node_count);
+
+            return @as(usize, path.source) * self.options.node_count + path.target;
+        }
+
+        pub fn path_queue(self: *Self, path: Path) *std.PriorityQueue(Data) {
+            var index = self.path_index(path);
+            return &self.paths[@as(usize, path.source) * self.options.node_count + path.target];
+        }
+
+        fn is_clogged(self: *Self, path: Path) bool {
+            return self.path_clogged_till[self.path_index(path)] > self.ticks;
+        }
+
+        fn should_clog(self: *Self, path: Path) bool {
+            return self.prng.random.uintAtMost(u8, 100) < self.options.path_clog_probability;
+        }
+
+        fn clog_for(self: *Self, path: Path, ticks: u64) void {
+            const clog_expiry = &self.path_clogged_till[self.path_index(path)];
+            clog_expiry.* = self.ticks + ticks;
+            log.debug("Path path.source={} path.target={} clogged for ticks={}", .{
+                path.source,
+                path.target,
+                ticks,
+            });
+        }
+
+        fn should_replay(self: *Self) bool {
+            return self.prng.random.uintAtMost(u8, 100) < self.options.packet_replay_probability;
+        }
+
+        fn should_partition(self: *Self) bool {
+            return self.prng.random.uintAtMost(u8, 100) < self.options.partition_probability;
+        }
+
+        fn should_unpartition(self: *Self) bool {
+            return self.prng.random.uintAtMost(u8, 100) < self.options.unpartition_probability;
+        }
+
+        /// Return a value produced using an exponential distribution with
+        /// the minimum and mean specified in self.options
+        fn one_way_delay(self: *Self) u64 {
+            const min = self.options.one_way_delay_min;
+            const mean = self.options.one_way_delay_mean;
+            return min + @floatToInt(u64, @intToFloat(f64, mean - min) * self.prng.random.floatExp(f64));
+        }
+
+        /// Partitions the network. Guaranteed to isolate at least one replica.
+        fn partition_network(
+            self: *Self,
+        ) void {
+            assert(self.options.replica_count > 1);
+
+            self.is_partitioned = true;
+            self.stability = self.options.partition_stability;
+
+            switch (self.options.partition_mode) {
+                .uniform_size => {
+                    // Exclude cases sz == 0 and sz == replica_count
+                    const sz =
+                        1 + self.prng.random.uintAtMost(u8, self.options.replica_count - 2);
+                    self.prng.random.shuffle(u8, self.replicas);
+                    for (self.replicas) |r, i| {
+                        self.partition[r] = i < sz;
+                    }
+                },
+                .uniform_partition => {
+                    var only_same = true;
+                    self.partition[0] =
+                        self.prng.random.uintLessThan(u8, 2) == 1;
+
+                    var i: usize = 1;
+                    while (i < self.options.replica_count) : (i += 1) {
+                        self.partition[i] =
+                            self.prng.random.uintLessThan(u8, 2) == 1;
+                        only_same =
+                            only_same and (self.partition[i - 1] == self.partition[i]);
+                    }
+
+                    if (only_same) {
+                        const n = self.prng.random.uintLessThan(u8, self.options.replica_count);
+                        self.partition[n] = true;
+                    }
+                },
+                .isolate_single => {
+                    for (self.replicas) |_, i| {
+                        self.partition[i] = false;
+                    }
+                    const n = self.prng.random.uintLessThan(u8, self.options.replica_count);
+                    self.partition[n] = true;
+                },
+                // Put your own partitioning logic here.
+                .custom => unreachable,
+            }
+        }
+
+        fn unpartition_network(
+            self: *Self,
+        ) void {
+            self.is_partitioned = false;
+            self.stability = self.options.unpartition_stability;
+
+            for (self.replicas) |_, i| {
+                self.partition[i] = false;
+            }
+        }
+
+        fn replicas_are_in_different_partitions(self: *Self, from: u8, to: u8) bool {
+            return from < self.options.replica_count and
+                to < self.options.replica_count and
+                self.partition[from] != self.partition[to];
+        }
+
+        pub fn tick(self: *Self) void {
+            self.ticks += 1;
+
+            if (self.stability > 0) {
+                self.stability -= 1;
+            } else {
+                if (self.is_partitioned) {
+                    if (self.should_unpartition()) {
+                        self.unpartition_network();
+                        log.alert("unpartitioned network: partition={d}", .{self.partition});
+                    }
+                } else {
+                    if (self.options.replica_count > 1 and self.should_partition()) {
+                        self.partition_network();
+                        log.alert("partitioned network: partition={d}", .{self.partition});
+                    }
+                }
+            }
+
+            var from: u8 = 0;
+            while (from < self.options.node_count) : (from += 1) {
+                var to: u8 = 0;
+                while (to < self.options.node_count) : (to += 1) {
+                    const path = .{ .source = from, .target = to };
+                    if (self.is_clogged(path)) continue;
+
+                    const queue = self.path_queue(path);
+                    while (queue.peek()) |*data| {
+                        if (data.expiry > self.ticks) break;
+                        _ = queue.remove();
+
+                        if (self.is_partitioned and
+                            self.replicas_are_in_different_partitions(from, to))
+                        {
+                            self.stats[@enumToInt(PacketStatistics.dropped_due_to_partition)] += 1;
+                            log.alert("dropped packet (different partitions): from={} to={}", .{ from, to });
+                            data.packet.deinit(path);
+                            continue;
+                        }
+
+                        if (self.should_drop()) {
+                            self.stats[@enumToInt(PacketStatistics.dropped)] += 1;
+                            log.alert("dropped packet from={} to={}.", .{ from, to });
+                            data.packet.deinit(path);
+                            continue;
+                        }
+
+                        if (self.should_replay()) {
+                            self.submit_packet(data.packet, data.callback, path);
+
+                            log.debug("replayed packet from={} to={}", .{ from, to });
+                            self.stats[@enumToInt(PacketStatistics.replay)] += 1;
+
+                            data.callback(data.packet, path);
+                        } else {
+                            log.debug("delivering packet from={} to={}", .{ from, to });
+                            data.callback(data.packet, path);
+                            data.packet.deinit(path);
+                        }
+                    }
+
+                    const reverse_path: Path = .{ .source = to, .target = from };
+
+                    if (self.should_clog(reverse_path)) {
+                        log.debug("reverse path clogged", .{});
+                        const mean = @intToFloat(f64, self.options.path_clog_duration_mean);
+                        const ticks = @floatToInt(u64, mean * self.prng.random.floatExp(f64));
+                        self.clog_for(reverse_path, ticks);
+                    }
+                }
+            }
+        }
+
+        pub fn submit_packet(
+            self: *Self,
+            packet: Packet,
+            callback: fn (packet: Packet, path: Path) void,
+            path: Path,
+        ) void {
+            const queue = self.path_queue(path);
+            var queue_length = queue.count();
+            if (queue_length + 1 > queue.capacity()) {
+                const index = self.prng.random.uintLessThanBiased(u64, queue_length);
+                const data = queue.removeIndex(index);
+                data.packet.deinit(path);
+                log.alert("submit_packet: {} reached capacity, dropped packet={}", .{
+                    path,
+                    index,
+                });
+            }
+
+            queue.add(.{
+                .expiry = self.ticks + self.one_way_delay(),
+                .packet = packet,
+                .callback = callback,
+            }) catch unreachable;
+        }
+    };
+}

package/src/tigerbeetle/src/test/state_checker.zig

@@ -0,0 +1,142 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const mem = std.mem;
+
+const config = @import("../config.zig");
+
+const Cluster = @import("cluster.zig").Cluster;
+const Network = @import("network.zig").Network;
+const StateMachine = @import("state_machine.zig").StateMachine;
+
+const MessagePool = @import("../message_pool.zig").MessagePool;
+const Message = MessagePool.Message;
+
+const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
+
+const RequestQueue = RingBuffer(u128, config.message_bus_messages_max - 1);
+const StateTransitions = std.AutoHashMap(u128, u64);
+
+const log = std.log.scoped(.state_checker);
+
+pub const StateChecker = struct {
+    /// Indexed by client index as used by Cluster.
+    client_requests: [config.clients_max]RequestQueue =
+        [_]RequestQueue{.{}} ** config.clients_max,
+
+    /// Indexed by replica index.
+    state_machine_states: [config.replicas_max]u128,
+
+    history: StateTransitions,
+
+    /// The highest cannonical state reached by the cluster.
+    state: u128,
+
+    /// The number of times the cannonical state has been advanced.
+    transitions: u64 = 0,
+
+    pub fn init(allocator: *mem.Allocator, cluster: *Cluster) !StateChecker {
+        const state = cluster.state_machines[0].state;
+
+        var state_machine_states: [config.replicas_max]u128 = undefined;
+        for (cluster.state_machines) |state_machine, i| {
+            assert(state_machine.state == state);
+            state_machine_states[i] = state_machine.state;
+        }
+
+        var history = StateTransitions.init(allocator);
+        errdefer history.deinit();
+
+        var state_checker = StateChecker{
+            .state_machine_states = state_machine_states,
+            .history = history,
+            .state = state,
+        };
+
+        try state_checker.history.putNoClobber(state, state_checker.transitions);
+
+        return state_checker;
+    }
+
+    pub fn deinit(state_checker: *StateChecker) void {
+        state_checker.history.deinit();
+    }
+
+    pub fn check_state(state_checker: *StateChecker, replica: u8) void {
+        const cluster = @fieldParentPtr(Cluster, "state_checker", state_checker);
+
+        const a = state_checker.state_machine_states[replica];
+        const b = cluster.state_machines[replica].state;
+
+        if (b == a) return;
+        state_checker.state_machine_states[replica] = b;
+
+        // If some other replica has already reached this state, then it will be in the history:
+        if (state_checker.history.get(b)) |transition| {
+            // A replica may transition more than once to the same state, for example, when
+            // restarting after a crash and replaying the log. The more important invariant is that
+            // the cluster as a whole may not transition to the same state more than once, and once
+            // transitioned may not regress.
+            log.info(
+                "{d:0>4}/{d:0>4} {x:0>32} > {x:0>32} {}",
+                .{ transition, state_checker.transitions, a, b, replica },
+            );
+            return;
+        }
+
+        // The replica has transitioned to state `b` that is not yet in the history.
+        // Check if this is a valid new state based on all currently inflight client requests.
+        for (state_checker.client_requests) |*queue| {
+            if (queue.head_ptr()) |input| {
+                if (b == StateMachine.hash(state_checker.state, std.mem.asBytes(input))) {
+                    const transitions_executed = state_checker.history.get(a).?;
+                    if (transitions_executed < state_checker.transitions) {
+                        @panic("replica skipped interim transitions");
+                    } else {
+                        assert(transitions_executed == state_checker.transitions);
+                    }
+
+                    state_checker.state = b;
+                    state_checker.transitions += 1;
+
+                    log.info("     {d:0>4} {x:0>32} > {x:0>32} {}", .{
+                        state_checker.transitions,
+                        a,
+                        b,
+                        replica,
+                    });
+
+                    state_checker.history.putNoClobber(b, state_checker.transitions) catch {
+                        @panic("state checker unable to allocate memory for history.put()");
+                    };
+
+                    // As soon as we reach a valid state we must pop the inflight request.
+                    // We cannot wait until the client receives the reply because that would allow
+                    // the inflight request to be used to reach other states in the interim.
+                    // We must therefore use our own queue rather than the clients' queues.
+                    _ = queue.pop();
+                    return;
+                }
+            }
+        }
+
+        @panic("replica transitioned to an invalid state");
+    }
+
+    pub fn convergence(state_checker: *StateChecker) bool {
+        const cluster = @fieldParentPtr(Cluster, "state_checker", state_checker);
+
+        const a = state_checker.state_machine_states[0];
+        for (state_checker.state_machine_states[1..cluster.options.replica_count]) |b| {
+            if (b != a) return false;
+        }
+
+        const transitions_executed = state_checker.history.get(a).?;
+        if (transitions_executed < state_checker.transitions) {
+            @panic("cluster reached convergence but on a regressed state");
+        } else {
+            assert(transitions_executed == state_checker.transitions);
+        }
+
+        return true;
+    }
+};

package/src/tigerbeetle/src/test/state_machine.zig

@@ -0,0 +1,71 @@
+const std = @import("std");
+
+const log = std.log.scoped(.state_machine);
+
+pub const StateMachine = struct {
+    pub const Operation = enum(u8) {
+        /// Operations reserved by VR protocol (for all state machines):
+        reserved,
+        init,
+        register,
+
+        hash,
+    };
+
+    state: u128,
+
+    pub fn init(seed: u64) StateMachine {
+        return .{ .state = hash(0, std.mem.asBytes(&seed)) };
+    }
+
+    pub fn prepare(
+        state_machine: *StateMachine,
+        realtime: i64,
+        operation: Operation,
+        input: []u8,
+    ) void {
+        // TODO: use realtime in some way to test the system
+    }
+
+    pub fn commit(
+        state_machine: *StateMachine,
+        client: u128,
+        operation: Operation,
+        input: []const u8,
+        output: []u8,
+    ) usize {
+        switch (operation) {
+            .reserved, .init => unreachable,
+            .register => return 0,
+
+            // TODO: instead of always using the first 32 bytes of the output
+            // buffer, get tricky and use a random but deterministic slice
+            // of it, filling the rest with 0s.
+            .hash => {
+                // Fold the input into our current state, creating a hash chain.
+                // Hash the input with the client ID since small inputs may collide across clients.
+                const client_input = hash(client, input);
+                const new_state = hash(state_machine.state, std.mem.asBytes(&client_input));
+
+                log.debug("state={x} input={x} input.len={} new state={x}", .{
+                    state_machine.state,
+                    client_input,
+                    input.len,
+                    new_state,
+                });
+
+                state_machine.state = new_state;
+                std.mem.copy(u8, output, std.mem.asBytes(&state_machine.state));
+                return @sizeOf(@TypeOf(state_machine.state));
+            },
+        }
+    }
+
+    pub fn hash(state: u128, input: []const u8) u128 {
+        var key: [32]u8 = [_]u8{0} ** 32;
+        std.mem.copy(u8, key[0..16], std.mem.asBytes(&state));
+        var target: [32]u8 = undefined;
+        std.crypto.hash.Blake3.hash(input, &target, .{ .key = key });
+        return @bitCast(u128, target[0..16].*);
+    }
+};