tigerbeetle-node 0.11.8 → 0.11.9

package/src/tigerbeetle/src/lsm/tree_fuzz.zig

@@ -4,7 +4,7 @@ const allocator = testing.allocator;
 const assert = std.debug.assert;
 
 const constants = @import("../constants.zig");
-const fuzz = @import("../test/fuzz.zig");
+const fuzz = @import("../testing/fuzz.zig");
 const vsr = @import("../vsr.zig");
 
 const log = std.log.scoped(.lsm_tree_fuzz);
@@ -13,7 +13,7 @@ const tracer = @import("../tracer.zig");
 const MessagePool = @import("../message_pool.zig").MessagePool;
 const Transfer = @import("../tigerbeetle.zig").Transfer;
 const Account = @import("../tigerbeetle.zig").Account;
-const Storage = @import("../test/storage.zig").Storage;
+const Storage = @import("../testing/storage.zig").Storage;
 const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
     .message_body_size_max = constants.message_body_size_max,
 });
@@ -62,6 +62,7 @@ const Key = packed struct {
     }
 };
 
+const FuzzOpTag = std.meta.Tag(FuzzOp);
 const FuzzOp = union(enum) {
     // TODO Test range queries.
     compact: struct {
@@ -72,14 +73,13 @@ const FuzzOp = union(enum) {
     remove: Key.Value,
     get: Key,
 };
-const FuzzOpTag = std.meta.Tag(FuzzOp);
 
-const cluster = 32;
-const replica = 4;
-const node_count = 1024;
 const batch_size_max = constants.message_size_max - @sizeOf(vsr.Header);
 const commit_entries_max = @divFloor(batch_size_max, @sizeOf(Key.Value));
 
+const cluster = 32;
+const replica = 4;
+const node_count = 1024;
 const tree_options = .{
     .commit_entries_max = commit_entries_max,
     // This is the smallest size that set_associative_cache will allow us.
@@ -87,7 +87,6 @@ const tree_options = .{
 };
 
 const puts_since_compact_max = commit_entries_max;
-
 const compacts_per_checkpoint = std.math.divCeil(
     usize,
     constants.journal_slot_count,
@@ -98,6 +97,7 @@ fn EnvironmentType(comptime table_usage: TableUsage) type {
     return struct {
         const Environment = @This();
 
+        const Tree = @import("tree.zig").TreeType(Table, Storage, "Key.Value");
         const Table = TableType(
             Key,
             Key.Value,
@@ -108,17 +108,17 @@ fn EnvironmentType(comptime table_usage: TableUsage) type {
             Key.tombstone_from_key,
             table_usage,
         );
-        const Tree = @import("tree.zig").TreeType(Table, Storage, "Key.Value");
 
         const State = enum {
-            uninit,
             init,
-            formatted,
+            superblock_format,
             superblock_open,
+            tree_init,
             tree_open,
-            tree_compacting,
-            tree_checkpointing,
-            superblock_checkpointing,
+            fuzzing,
+            tree_compact,
+            tree_checkpoint,
+            superblock_checkpoint,
             tree_lookup,
         };
 
@@ -126,70 +126,40 @@ fn EnvironmentType(comptime table_usage: TableUsage) type {
         storage: *Storage,
         message_pool: MessagePool,
         superblock: SuperBlock,
-        superblock_context: SuperBlock.Context = undefined,
+        superblock_context: SuperBlock.Context,
         grid: Grid,
         node_pool: NodePool,
         tree: Tree,
-        // We need @fieldParentPtr() of tree, so we can't use an optional Tree.
-        tree_exists: bool,
-        lookup_context: Tree.LookupContext = undefined,
-        lookup_value: ?*const Key.Value = null,
-        checkpoint_op: ?u64 = null,
-
-        fn init(env: *Environment, storage: *Storage) !void {
-            env.state = .uninit;
+        lookup_context: Tree.LookupContext,
+        lookup_value: ?*const Key.Value,
+        checkpoint_op: ?u64,
 
+        pub fn run(storage: *Storage, fuzz_ops: []const FuzzOp) !void {
+            var env: Environment = undefined;
+            env.state = .init;
             env.storage = storage;
-            errdefer env.storage.deinit(allocator);
 
             env.message_pool = try MessagePool.init(allocator, .replica);
-            errdefer env.message_pool.deinit(allocator);
+            defer env.message_pool.deinit(allocator);
 
             env.superblock = try SuperBlock.init(allocator, .{
                 .storage = env.storage,
                 .storage_size_limit = constants.storage_size_max,
                 .message_pool = &env.message_pool,
             });
-            errdefer env.superblock.deinit(allocator);
+            defer env.superblock.deinit(allocator);
 
             env.grid = try Grid.init(allocator, &env.superblock);
-            errdefer env.grid.deinit(allocator);
+            defer env.grid.deinit(allocator);
 
             env.node_pool = try NodePool.init(allocator, node_count);
-            errdefer env.node_pool.deinit(allocator);
+            defer env.node_pool.deinit(allocator);
 
-            // Tree must be initialized with an open superblock.
             env.tree = undefined;
-            env.tree_exists = false;
-
-            env.state = .init;
-        }
-
-        fn deinit(env: *Environment) void {
-            assert(env.state != .uninit);
-
-            if (env.tree_exists) {
-                env.tree.deinit(allocator);
-                env.tree_exists = false;
-            }
-            env.node_pool.deinit(allocator);
-            env.grid.deinit(allocator);
-            env.superblock.deinit(allocator);
-            env.message_pool.deinit(allocator);
-
-            env.state = .uninit;
-        }
+            env.lookup_value = null;
+            env.checkpoint_op = null;
 
-        fn tick(env: *Environment) void {
-            // env.grid.tick();
-            env.storage.tick();
-        }
-
-        fn tick_until_state_change(env: *Environment, current_state: State, next_state: State) void {
-            // Sometimes IO completes synchronously (eg if cached), so we might already be in next_state before ticking.
-            assert(env.state == current_state or env.state == next_state);
-            while (env.state == current_state) env.tick();
-            assert(env.state == next_state);
+            try env.open_then_apply(fuzz_ops);
         }
 
         fn change_state(env: *Environment, current_state: State, next_state: State) void {
@@ -197,123 +167,129 @@ fn EnvironmentType(comptime table_usage: TableUsage) type {
             env.state = next_state;
         }
 
-        pub fn format(storage: *Storage) !void {
-            var env: Environment = undefined;
-
-            try env.init(storage);
-            defer env.deinit();
+        fn tick_until_state_change(env: *Environment, current_state: State, next_state: State) void {
+            // Sometimes operations complete synchronously so we might already be in next_state before ticking.
+            //assert(env.state == current_state or env.state == next_state);
+            while (env.state == current_state) env.storage.tick();
+            assert(env.state == next_state);
+        }
 
-            assert(env.state == .init);
+        pub fn open_then_apply(env: *Environment, fuzz_ops: []const FuzzOp) !void {
+            env.change_state(.init, .superblock_format);
             env.superblock.format(superblock_format_callback, &env.superblock_context, .{
                 .cluster = cluster,
                 .replica = replica,
             });
-            env.tick_until_state_change(.init, .formatted);
+
+            env.tick_until_state_change(.superblock_format, .superblock_open);
+            env.superblock.open(superblock_open_callback, &env.superblock_context);
+
+            env.tick_until_state_change(.superblock_open, .tree_init);
+            env.tree = try Tree.init(allocator, &env.node_pool, &env.grid, tree_options);
+            defer env.tree.deinit(allocator);
+
+            env.change_state(.tree_init, .tree_open);
+            env.tree.open(tree_open_callback);
+
+            env.tick_until_state_change(.tree_open, .fuzzing);
+            try env.apply(fuzz_ops);
         }
 
         fn superblock_format_callback(superblock_context: *SuperBlock.Context) void {
             const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
-            env.change_state(.init, .formatted);
-        }
-
-        pub fn open(env: *Environment) void {
-            assert(env.state == .init);
-            env.superblock.open(superblock_open_callback, &env.superblock_context);
-            env.tick_until_state_change(.init, .tree_open);
+            env.change_state(.superblock_format, .superblock_open);
         }
 
         fn superblock_open_callback(superblock_context: *SuperBlock.Context) void {
             const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
-            env.change_state(.init, .superblock_open);
-            env.tree = Tree.init(allocator, &env.node_pool, &env.grid, tree_options) catch unreachable;
-            env.tree_exists = true;
-            env.tree.open(tree_open_callback);
+            env.change_state(.superblock_open, .tree_init);
         }
 
         fn tree_open_callback(tree: *Tree) void {
             const env = @fieldParentPtr(@This(), "tree", tree);
-            env.change_state(.superblock_open, .tree_open);
+            env.change_state(.tree_open, .fuzzing);
         }
 
         pub fn compact(env: *Environment, op: u64) void {
-            env.change_state(.tree_open, .tree_compacting);
+            env.change_state(.fuzzing, .tree_compact);
             env.tree.compact(tree_compact_callback, op);
-            env.tick_until_state_change(.tree_compacting, .tree_open);
+            env.tick_until_state_change(.tree_compact, .fuzzing);
         }
 
         fn tree_compact_callback(tree: *Tree) void {
             const env = @fieldParentPtr(@This(), "tree", tree);
-            env.change_state(.tree_compacting, .tree_open);
+            env.change_state(.tree_compact, .fuzzing);
         }
 
         pub fn checkpoint(env: *Environment, op: u64) void {
+            assert(env.checkpoint_op == null);
             env.checkpoint_op = op - constants.lsm_batch_multiple;
-            env.change_state(.tree_open, .tree_checkpointing);
+
+            env.change_state(.fuzzing, .tree_checkpoint);
             env.tree.checkpoint(tree_checkpoint_callback);
-            env.tick_until_state_change(.tree_checkpointing, .superblock_checkpointing);
-            env.tick_until_state_change(.superblock_checkpointing, .tree_open);
+            env.tick_until_state_change(.tree_checkpoint, .superblock_checkpoint);
+            env.tick_until_state_change(.superblock_checkpoint, .fuzzing);
         }
 
         fn tree_checkpoint_callback(tree: *Tree) void {
             const env = @fieldParentPtr(@This(), "tree", tree);
-            env.change_state(.tree_checkpointing, .superblock_checkpointing);
+            const op = env.checkpoint_op.?;
+            env.checkpoint_op = null;
+
+            env.change_state(.tree_checkpoint, .superblock_checkpoint);
             env.superblock.checkpoint(superblock_checkpoint_callback, &env.superblock_context, .{
                 .commit_min_checksum = env.superblock.working.vsr_state.commit_min_checksum + 1,
-                .commit_min = env.checkpoint_op.?,
-                .commit_max = env.checkpoint_op.? + 1,
-                .view_normal = 0,
+                .commit_min = op,
+                .commit_max = op + 1,
+                .log_view = 0,
                 .view = 0,
             });
-            env.checkpoint_op = null;
         }
 
         fn superblock_checkpoint_callback(superblock_context: *SuperBlock.Context) void {
             const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
-            env.change_state(.superblock_checkpointing, .tree_open);
+            env.change_state(.superblock_checkpoint, .fuzzing);
         }
 
-        fn get(env: *Environment, key: Key) ?*const Key.Value {
+        pub fn get(env: *Environment, key: Key) ?*const Key.Value {
+            env.change_state(.fuzzing, .tree_lookup);
+
             if (env.tree.lookup_from_memory(env.tree.lookup_snapshot_max, key)) |value| {
+                env.change_state(.tree_lookup, .fuzzing);
                 return Tree.unwrap_tombstone(value);
-            } else {
-                env.change_state(.tree_open, .tree_lookup);
-                env.lookup_context = undefined;
-                env.lookup_value = null;
-                env.tree.lookup_from_levels(get_callback, &env.lookup_context, env.tree.lookup_snapshot_max, key);
-                env.tick_until_state_change(.tree_lookup, .tree_open);
-                return env.lookup_value;
             }
+
+            env.lookup_value = null;
+            env.tree.lookup_from_levels(get_callback, &env.lookup_context, env.tree.lookup_snapshot_max, key);
+            env.tick_until_state_change(.tree_lookup, .fuzzing);
+            return env.lookup_value;
         }
 
         fn get_callback(lookup_context: *Tree.LookupContext, value: ?*const Key.Value) void {
             const env = @fieldParentPtr(Environment, "lookup_context", lookup_context);
             assert(env.lookup_value == null);
             env.lookup_value = value;
-            env.change_state(.tree_lookup, .tree_open);
+            env.change_state(.tree_lookup, .fuzzing);
         }
 
-        fn run(storage: *Storage, fuzz_ops: []const FuzzOp) !void {
-            var env: Environment = undefined;
-
-            try env.init(storage);
-            defer env.deinit();
-
-            // Open the superblock then tree.
-            env.open();
-
+        pub fn apply(env: *Environment, fuzz_ops: []const FuzzOp) !void {
             // The tree should behave like a simple key-value data-structure.
             // We'll compare it to a hash map.
             var model = std.hash_map.AutoHashMap(Key, Key.Value).init(allocator);
             defer model.deinit();
 
             for (fuzz_ops) |fuzz_op, fuzz_op_index| {
+                assert(env.state == .fuzzing);
                 log.debug("Running fuzz_ops[{}/{}] == {}", .{ fuzz_op_index, fuzz_ops.len, fuzz_op });
-                const storage_size_used = storage.size_used();
-                log.debug("storage.size_used = {}/{}", .{ storage_size_used, storage.size });
+
+                const storage_size_used = env.storage.size_used();
+                log.debug("storage.size_used = {}/{}", .{ storage_size_used, env.storage.size });
+
                 const model_size = model.count() * @sizeOf(Key.Value);
                 log.debug("space_amplification = {d:.2}", .{
                     @intToFloat(f64, storage_size_used) / @intToFloat(f64, model_size),
                 });
+
                 // Apply fuzz_op to the tree and the model.
                 switch (fuzz_op) {
                     .compact => |compact| {
@@ -489,14 +465,10 @@ pub fn main() !void {
     // TODO Use inline switch after upgrading to zig 0.10
     switch (table_usage) {
         .general => {
-            const Environment = EnvironmentType(.general);
-            try Environment.format(&storage);
-            try Environment.run(&storage, fuzz_ops);
+            try EnvironmentType(.general).run(&storage, fuzz_ops);
         },
         .secondary_index => {
-            const Environment = EnvironmentType(.secondary_index);
-            try Environment.format(&storage);
-            try Environment.run(&storage, fuzz_ops);
+            try EnvironmentType(.secondary_index).run(&storage, fuzz_ops);
         },
     }
 

package/src/tigerbeetle/src/message_bus.zig

@@ -12,7 +12,7 @@ const log = std.log.scoped(.message_bus);
 const vsr = @import("vsr.zig");
 const Header = vsr.Header;
 
-const util = @import("util.zig");
+const stdx = @import("stdx.zig");
 const RingBuffer = @import("ring_buffer.zig").RingBuffer;
 const IO = @import("io.zig").IO;
 const MessagePool = @import("message_pool.zig").MessagePool;
@@ -750,7 +750,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
                 if (connection.recv_progress == header.size) return connection.recv_message.?.ref();
 
                 const message = bus.get_message();
-                util.copy_disjoint(.inexact, u8, message.buffer, data[0..header.size]);
+                stdx.copy_disjoint(.inexact, u8, message.buffer, data[0..header.size]);
                 return message;
             }
 
@@ -771,7 +771,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
                     const sector_ceil = vsr.sector_ceil(message.header.size);
                     if (message.header.size != sector_ceil) {
                         assert(message.header.size < sector_ceil);
-                        assert(message.buffer.len == constants.message_size_max + constants.sector_size);
+                        assert(message.buffer.len == constants.message_size_max);
                         mem.set(u8, message.buffer[message.header.size..sector_ceil], 0);
                     }
                 }
@@ -842,7 +842,7 @@ fn MessageBusType(comptime process_type: vsr.ProcessType) type {
                     assert(connection.recv_progress > 0);
                     assert(connection.recv_parsed > 0);
                     const data = recv_message.buffer[connection.recv_parsed..connection.recv_progress];
-                    util.copy_disjoint(.inexact, u8, new_message.buffer, data);
+                    stdx.copy_disjoint(.inexact, u8, new_message.buffer, data);
                     connection.recv_progress = data.len;
                     connection.recv_parsed = 0;
                 } else {

package/src/tigerbeetle/src/message_pool.zig

@@ -13,10 +13,6 @@ comptime {
     assert(constants.message_size_max % constants.sector_size == 0);
 }
 
-/// Add an extra sector_size bytes to allow a partially received subsequent
-/// message to be shifted to make space for 0 padding to vsr.sector_ceil.
-const message_size_max_padded = constants.message_size_max + constants.sector_size;
-
 /// The number of full-sized messages allocated at initialization by the replica message pool.
 /// There must be enough messages to ensure that the replica can always progress, to avoid deadlock.
 pub const messages_max_replica = messages_max: {
@@ -26,7 +22,8 @@ pub const messages_max_replica = messages_max: {
     sum += constants.journal_iops_write_max; // Journal writes
     sum += constants.clients_max; // SuperBlock.client_table
     sum += 1; // Replica.loopback_queue
-    sum += constants.pipeline_max; // Replica.pipeline
+    sum += constants.pipeline_prepare_queue_max; // Replica.Pipeline{Queue|Cache}
+    sum += constants.pipeline_request_queue_max; // Replica.Pipeline{Queue|Cache}
     sum += 1; // Replica.commit_prepare
     // Replica.do_view_change_from_all_replicas quorum:
     // Replica.recovery_response_quorum is only used for recovery and does not increase the limit.
@@ -67,7 +64,7 @@ pub const MessagePool = struct {
     pub const Message = struct {
         // TODO: replace this with a header() function to save memory
         header: *Header,
-        buffer: []align(constants.sector_size) u8,
+        buffer: *align(constants.sector_size) [constants.message_size_max]u8,
         references: u32 = 0,
         next: ?*Message,
 
@@ -105,13 +102,13 @@ pub const MessagePool = struct {
                 const buffer = try allocator.allocAdvanced(
                     u8,
                     constants.sector_size,
-                    message_size_max_padded,
+                    constants.message_size_max,
                     .exact,
                 );
                 const message = try allocator.create(Message);
                 message.* = .{
                     .header = mem.bytesAsValue(Header, buffer[0..@sizeOf(Header)]),
-                    .buffer = buffer,
+                    .buffer = buffer[0..constants.message_size_max],
                     .next = pool.free_list,
                 };
                 pool.free_list = message;
@@ -126,7 +123,7 @@ pub const MessagePool = struct {
         var free_count: usize = 0;
         while (pool.free_list) |message| {
             pool.free_list = message.next;
-            allocator.free(message.buffer);
+            allocator.free(@as([]const u8, message.buffer));
             allocator.destroy(message);
             free_count += 1;
         }
@@ -151,7 +148,7 @@ pub const MessagePool = struct {
     pub fn unref(pool: *MessagePool, message: *Message) void {
         message.references -= 1;
         if (message.references == 0) {
-            if (builtin.mode == .Debug) mem.set(u8, message.buffer, undefined);
+            mem.set(u8, message.buffer, 0);
             message.next = pool.free_list;
             pool.free_list = message;
         }

package/src/tigerbeetle/src/ring_buffer.zig

@@ -3,7 +3,7 @@ const assert = std.debug.assert;
 const math = std.math;
 const mem = std.mem;
 
-const util = @import("util.zig");
+const stdx = @import("stdx.zig");
 
 /// A First In, First Out ring buffer holding at most `count_max` elements.
 pub fn RingBuffer(
@@ -47,36 +47,38 @@ pub fn RingBuffer(
 
         // TODO Add doc comments to these functions:
         pub inline fn head(self: Self) ?T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return self.buffer[self.index];
         }
 
         pub inline fn head_ptr(self: *Self) ?*T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return &self.buffer[self.index];
         }
 
         pub inline fn head_ptr_const(self: *const Self) ?*const T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return &self.buffer[self.index];
         }
 
         pub inline fn tail(self: Self) ?T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return self.buffer[(self.index + self.count - 1) % self.buffer.len];
         }
 
         pub inline fn tail_ptr(self: *Self) ?*T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return &self.buffer[(self.index + self.count - 1) % self.buffer.len];
         }
 
         pub inline fn tail_ptr_const(self: *const Self) ?*const T {
-            if (self.empty()) return null;
+            if (count_max == 0 or self.empty()) return null;
             return &self.buffer[(self.index + self.count - 1) % self.buffer.len];
         }
 
         pub inline fn get_ptr(self: *Self, index: usize) ?*T {
+            if (count_max == 0) unreachable;
+
             if (index < self.count) {
                 return &self.buffer[(self.index + index) % self.buffer.len];
             } else {
@@ -86,17 +88,17 @@ pub fn RingBuffer(
         }
 
         pub inline fn next_tail(self: Self) ?T {
-            if (self.full()) return null;
+            if (count_max == 0 or self.full()) return null;
             return self.buffer[(self.index + self.count) % self.buffer.len];
         }
 
         pub inline fn next_tail_ptr(self: *Self) ?*T {
-            if (self.full()) return null;
+            if (count_max == 0 or self.full()) return null;
             return &self.buffer[(self.index + self.count) % self.buffer.len];
         }
 
         pub inline fn next_tail_ptr_const(self: *const Self) ?*const T {
-            if (self.full()) return null;
+            if (count_max == 0 or self.full()) return null;
             return &self.buffer[(self.index + self.count) % self.buffer.len];
         }
 
@@ -143,14 +145,15 @@ pub fn RingBuffer(
         }
 
         pub fn push_slice(self: *Self, items: []const T) error{NoSpaceLeft}!void {
+            if (count_max == 0) return error.NoSpaceLeft;
             if (self.count + items.len > self.buffer.len) return error.NoSpaceLeft;
 
             const pre_wrap_start = (self.index + self.count) % self.buffer.len;
             const pre_wrap_count = math.min(items.len, self.buffer.len - pre_wrap_start);
             const post_wrap_count = items.len - pre_wrap_count;
 
-            util.copy_disjoint(.inexact, T, self.buffer[pre_wrap_start..], items[0..pre_wrap_count]);
-            util.copy_disjoint(.exact, T, self.buffer[0..post_wrap_count], items[pre_wrap_count..]);
+            stdx.copy_disjoint(.inexact, T, self.buffer[pre_wrap_start..], items[0..pre_wrap_count]);
+            stdx.copy_disjoint(.exact, T, self.buffer[0..post_wrap_count], items[pre_wrap_count..]);
 
             self.count += items.len;
         }
@@ -174,6 +177,7 @@ pub fn RingBuffer(
             count: usize = 0,
 
             pub fn next(it: *Iterator) ?T {
+                if (count_max == 0) return null;
                 // TODO Use next_ptr() internally to avoid duplicating this code.
                 assert(it.count <= it.ring.count);
                 if (it.count == it.ring.count) return null;
@@ -183,6 +187,7 @@ pub fn RingBuffer(
 
             pub fn next_ptr(it: *Iterator) ?*const T {
                 assert(it.count <= it.ring.count);
+                if (count_max == 0) return null;
                 if (it.count == it.ring.count) return null;
                 defer it.count += 1;
                 return &it.ring.buffer[(it.ring.index + it.count) % it.ring.buffer.len];
@@ -201,6 +206,7 @@ pub fn RingBuffer(
 
             pub fn next_ptr(it: *IteratorMutable) ?*T {
                 assert(it.count <= it.ring.count);
+                if (count_max == 0) return null;
                 if (it.count == it.ring.count) return null;
                 defer it.count += 1;
                 return &it.ring.buffer[(it.ring.index + it.count) % it.ring.buffer.len];
@@ -387,3 +393,7 @@ test "RingBuffer: pop_tail" {
     try testing.expectEqual(@as(?u32, null), lifo.pop_tail());
     try testing.expect(lifo.empty());
 }
+
+test "RingBuffer: count_max=0" {
+    std.testing.refAllDecls(RingBuffer(u32, 0, .array));
+}