tigerbeetle-node 0.11.0 → 0.11.2

package/src/tigerbeetle/src/lsm/forest_fuzz.zig

@@ -38,7 +38,10 @@ const Environment = struct {
     const cluster = 32;
     const replica = 4;
     // TODO Is this appropriate for the number of fuzz_ops we want to run?
-    const size_max = vsr.Zone.superblock.size().? + vsr.Zone.wal.size().? + 1024 * 1024 * 1024;
+    const size_max = vsr.Zone.superblock.size().? +
+        vsr.Zone.wal_headers.size().? +
+        vsr.Zone.wal_prepares.size().? +
+        1024 * 1024 * 1024;
 
     const node_count = 1024;
     // This is the smallest size that set_associative_cache will allow us.
@@ -80,6 +83,7 @@ const Environment = struct {
     forest: Forest,
     // We need @fieldParentPtr() of forest, so we can't use an optional Forest.
     forest_exists: bool,
+    checkpoint_op: ?u64 = null,
 
     fn init(env: *Environment, storage: *Storage) !void {
         env.state = .uninit;
@@ -184,7 +188,8 @@ const Environment = struct {
         env.change_state(.forest_compacting, .forest_open);
     }
 
-    pub fn checkpoint(env: *Environment) void {
+    pub fn checkpoint(env: *Environment, op: u64) void {
+        env.checkpoint_op = op - config.lsm_batch_multiple;
         env.change_state(.forest_open, .forest_checkpointing);
         env.forest.checkpoint(forest_checkpoint_callback);
         env.tick_until_state_change(.forest_checkpointing, .superblock_checkpointing);
@@ -194,7 +199,14 @@ const Environment = struct {
     fn forest_checkpoint_callback(forest: *Forest) void {
         const env = @fieldParentPtr(@This(), "forest", forest);
         env.change_state(.forest_checkpointing, .superblock_checkpointing);
-        env.superblock.checkpoint(superblock_checkpoint_callback, &env.superblock_context);
+        env.superblock.checkpoint(superblock_checkpoint_callback, &env.superblock_context, .{
+            .commit_min_checksum = env.superblock.working.vsr_state.commit_min_checksum + 1,
+            .commit_min = env.checkpoint_op.?,
+            .commit_max = env.checkpoint_op.? + 1,
+            .view_normal = 0,
+            .view = 0,
+        });
+        env.checkpoint_op = null;
     }
 
     fn superblock_checkpoint_callback(superblock_context: *SuperBlock.Context) void {
@@ -236,17 +248,17 @@ const Environment = struct {
 
         for (fuzz_ops) |fuzz_op, fuzz_op_index| {
             log.debug("Running fuzz_ops[{}/{}] == {}", .{ fuzz_op_index, fuzz_ops.len, fuzz_op });
-            //TODO(@djg) Restore these when dj-vopr-workload merges.
-            //const storage_size_used = storage.size_used();
-            //log.debug("storage.size_used = {}/{}", .{ storage_size_used, storage.size });
-            //const model_size = model.count() * @sizeOf(Account);
-            //log.debug("space_amplification = {d:.2}", .{@intToFloat(f64, storage_size_used) / @intToFloat(f64, model_size)});
+            const storage_size_used = storage.size_used();
+            log.debug("storage.size_used = {}/{}", .{ storage_size_used, storage.size });
+            const model_size = model.count() * @sizeOf(Account);
+            log.debug("space_amplification = {d:.2}", .{
+                @intToFloat(f64, storage_size_used) / @intToFloat(f64, model_size),
+            });
             // Apply fuzz_op to the forest and the model.
             switch (fuzz_op) {
                 .compact => |compact| {
                     env.compact(compact.op);
-                    if (compact.checkpoint)
-                        env.checkpoint();
+                    if (compact.checkpoint) env.checkpoint(compact.op);
                 },
                 .put_account => |account| {
                     env.forest.grooves.accounts.put(&account);
@@ -274,27 +286,9 @@ const Environment = struct {
     }
 };
 
-pub fn run_fuzz_ops(fuzz_ops: []const FuzzOp) !void {
+pub fn run_fuzz_ops(storage_options: Storage.Options, fuzz_ops: []const FuzzOp) !void {
     // Init mocked storage.
-    var storage = try Storage.init(
-        allocator,
-        Environment.size_max,
-        Storage.Options{
-            // We don't apply storage faults yet, so this seed doesn't matter.
-            .seed = 0xdeadbeef,
-            .read_latency_min = 0,
-            .read_latency_mean = 0,
-            .write_latency_min = 0,
-            .write_latency_mean = 0,
-            .read_fault_probability = 0,
-            .write_fault_probability = 0,
-        },
-        0,
-        .{
-            .first_offset = 0,
-            .period = 0,
-        },
-    );
+    var storage = try Storage.init(allocator, Environment.size_max, storage_options);
     defer storage.deinit(allocator);
 
     try Environment.format(&storage);
@@ -354,6 +348,7 @@ pub fn generate_fuzz_ops(random: std.rand.Random) ![]const FuzzOp {
                 const checkpoint =
                     // Can only checkpoint on the last beat of the bar.
                     compact_op % config.lsm_batch_multiple == config.lsm_batch_multiple - 1 and
+                    compact_op > config.lsm_batch_multiple and
                     // Checkpoint at roughly the same rate as log wraparound.
                     random.uintLessThan(usize, Environment.compacts_per_checkpoint) == 0;
                 break :compact FuzzOp{
@@ -402,11 +397,18 @@ pub fn generate_fuzz_ops(random: std.rand.Random) ![]const FuzzOp {
 pub fn main() !void {
     const fuzz_args = try fuzz.parse_fuzz_args(allocator);
     var rng = std.rand.DefaultPrng.init(fuzz_args.seed);
+    const random = rng.random();
 
-    const fuzz_ops = try generate_fuzz_ops(rng.random());
+    const fuzz_ops = try generate_fuzz_ops(random);
     defer allocator.free(fuzz_ops);
 
-    try run_fuzz_ops(fuzz_ops);
+    try run_fuzz_ops(Storage.Options{
+        .seed = random.int(u64),
+        .read_latency_min = 0,
+        .read_latency_mean = 0 + fuzz.random_int_exponential(random, u64, 20),
+        .write_latency_min = 0,
+        .write_latency_mean = 0 + fuzz.random_int_exponential(random, u64, 20),
+    }, fuzz_ops);
 
     log.info("Passed!", .{});
 }

package/src/tigerbeetle/src/lsm/grid.zig

@@ -4,11 +4,13 @@ const mem = std.mem;
 
 const config = @import("../config.zig");
 const vsr = @import("../vsr.zig");
+const free_set = @import("../vsr/superblock_free_set.zig");
 
 const SuperBlockType = vsr.SuperBlockType;
 const FIFO = @import("../fifo.zig").FIFO;
 const IOPS = @import("../iops.zig").IOPS;
 const SetAssociativeCache = @import("set_associative_cache.zig").SetAssociativeCache;
+const util = @import("../util.zig");
 
 const log = std.log.scoped(.grid);
 
@@ -90,6 +92,7 @@ pub fn GridType(comptime Storage: type) type {
 
         pub const BlockPtr = *align(config.sector_size) [block_size]u8;
         pub const BlockPtrConst = *align(config.sector_size) const [block_size]u8;
+        pub const Reservation = free_set.Reservation;
 
         pub const Write = struct {
             callback: fn (*Grid.Write) void,
@@ -166,31 +169,46 @@ pub fn GridType(comptime Storage: type) type {
         }
 
         pub fn tick(grid: *Grid) void {
-            
             // Resolve reads that were seen in the cache during start_read()
             // but deferred to be asynchronously resolved on the next tick.
             //
-            // Make a copy to avoid a potential infinite loop here.
-            // If any new reads are added to the read_cached_queue they will be started on
-            // the next tick.
-            var copy = grid.read_cached_queue;
-            grid.read_cached_queue = .{};
-            while (copy.pop()) |read| {
+            // Drain directly from the queue so that new cache reads (added upon completion of old
+            // cache reads) that can be serviced immediately aren't deferred until the next tick
+            // (which may be milliseconds later due to IO.run_for_ns). This is necessary to ensure
+            // that groove prefetch completes promptly.
+            //
+            // Even still, we cap the reads processed to prevent going over
+            // any implicit time slice expected of Grid.tick(). This limit is fairly arbitrary.
+            var retry_max: u32 = 100_000;
+            while (grid.read_cached_queue.pop()) |read| {
                 if (grid.cache.get(read.address)) |block| {
-                    read.callback(read, block);   
+                    read.callback(read, block);
                 } else {
                     grid.start_read(read);
                 }
+
+                retry_max -= 1;
+                if (retry_max == 0) break;
             }
         }
 
-        pub fn acquire(grid: *Grid) u64 {
-            // We will reject incoming data before it reaches the point
-            // where storage is full, so this assertion is safe.
-            return grid.superblock.free_set.acquire().?;
+        /// Returning null indicates that there are not enough free blocks to fill the reservation.
+        pub fn reserve(grid: *Grid, blocks_count: usize) ?Reservation {
+            return grid.superblock.free_set.reserve(blocks_count);
+        }
+
+        /// Forfeit a reservation.
+        pub fn forfeit(grid: *Grid, reservation: Reservation) void {
+            return grid.superblock.free_set.forfeit(reservation);
+        }
+
+        /// Returns a just-allocated block.
+        /// The caller is responsible for not acquiring more blocks than they reserved.
+        pub fn acquire(grid: *Grid, reservation: Reservation) u64 {
+            return grid.superblock.free_set.acquire(reservation).?;
         }
 
-        /// This function should be used to release addresses, instead of release_at_checkpoint()
+        /// This function should be used to release addresses, instead of release()
         /// on the free set directly, as this also demotes the address within the block cache.
         /// This reduces conflict misses in the block cache, by freeing ways soon after they are
         /// released.
@@ -199,12 +217,12 @@ pub fn GridType(comptime Storage: type) type {
         /// checkpoint.
         ///
         /// Asserts that the address is not currently being read from or written to.
-        pub fn release_at_checkpoint(grid: *Grid, address: u64) void {
+        pub fn release(grid: *Grid, address: u64) void {
             grid.assert_not_writing(address, null);
             grid.assert_not_reading(address, null);
 
             grid.cache.demote(address);
-            grid.superblock.free_set.release_at_checkpoint(address);
+            grid.superblock.free_set.release(address);
         }
 
         /// Assert that the address is not currently being written to.
@@ -320,7 +338,7 @@ pub fn GridType(comptime Storage: type) type {
                 grid,
                 completed_write.address,
             );
-            mem.copy(u8, cached_block, completed_write.block);
+            util.copy_disjoint(.exact, u8, cached_block, completed_write.block);
 
             grid.write_iops.release(iop);
 
@@ -438,8 +456,8 @@ pub fn GridType(comptime Storage: type) type {
             // read_block_callback(), but that would issue a new call to read_sectors().
             iop.reads.push(read);
             {
-                // Make a copy here to avoid an infinite loop from pending_reads being 
-                // re-added to read_queue after not matching the current read. 
+                // Make a copy here to avoid an infinite loop from pending_reads being
+                // re-added to read_queue after not matching the current read.
                 var copy = grid.read_queue;
                 grid.read_queue = .{};
                 while (copy.pop()) |pending_read| {
@@ -449,7 +467,7 @@ pub fn GridType(comptime Storage: type) type {
                     } else {
                         grid.read_queue.push(pending_read);
                     }
-                } 
+                }
             }
 
             grid.superblock.storage.read_sectors(
@@ -490,7 +508,7 @@ pub fn GridType(comptime Storage: type) type {
                 assert(header.operation == block_type.operation());
 
                 // NOTE: read callbacks resolved here could queue up reads into this very iop.
-                // This extends this while loop, but that's fine as it keeps the callbacks 
+                // This extends this while loop, but that's fine as it keeps the callbacks
                 // asynchronous to themselves (preventing something like a stack-overflow).
                 while (iop.reads.pop()) |read| {
                     assert(read.address == address);
@@ -530,7 +548,7 @@ pub fn GridType(comptime Storage: type) type {
             grid.read_iops.release(iop);
 
             // Always iterate through the full list of pending reads instead of just one to ensure
-            // that those serviced from the cache don't prevent others waiting for an IOP from 
+            // that those serviced from the cache don't prevent others waiting for an IOP from
             // seeing the IOP that was just released.
             var copy = grid.read_queue;
             grid.read_queue = .{};

package/src/tigerbeetle/src/lsm/groove.zig

@@ -587,6 +587,7 @@ pub fn GrooveType(
                         id_tree_value.timestamp,
                     )) |object| {
                         assert(!ObjectTreeHelpers(Object).tombstone(object));
+                        assert(object.id == id);
                         groove.prefetch_objects.putAssumeCapacity(object.*, {});
                     } else {
                         // The id was in the IdTree's value cache, but not in the ObjectTree's

package/src/tigerbeetle/src/lsm/k_way_merge.zig

@@ -17,7 +17,7 @@ pub fn KWayMergeIterator(
     comptime stream_peek: fn (
         context: *const Context,
         stream_index: u32,
-    ) error{Empty, Drained}!Key,
+    ) error{ Empty, Drained }!Key,
     comptime stream_pop: fn (context: *Context, stream_index: u32) Value,
     /// Returns true if stream A has higher precedence than stream B.
     /// This is used to deduplicate values across streams.
@@ -69,7 +69,7 @@ pub fn KWayMergeIterator(
             var stream_index: u32 = 0;
             while (stream_index < stream_count_max) : (stream_index += 1) {
                 it.keys[it.k] = stream_peek(context, stream_index) catch |err| switch (err) {
-                    // On initialization, the streams should either have data already 
+                    // On initialization, the streams should either have data already
                     // buffered up to peek or be empty and have no more values to produce.
                     error.Drained => unreachable,
                     error.Empty => continue,
@@ -225,7 +225,7 @@ fn TestContext(comptime k_max: u32) type {
             return math.order(a, b);
         }
 
-        fn stream_peek(context: *const Self, stream_index: u32) error{Empty, Drained}!u32 {
+        fn stream_peek(context: *const Self, stream_index: u32) error{ Empty, Drained }!u32 {
             // TODO: test for Drained somehow as well
             const stream = context.streams[stream_index];
             if (stream.len == 0) return error.Empty;

package/src/tigerbeetle/src/lsm/level_iterator.zig

@@ -283,7 +283,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
         /// - error.Empty when there are no values remaining to iterate.
         /// - error.Drained when the iterator isn't empty, but the values 
         ///   still need to be buffered into memory via tick().
-        pub fn peek(it: LevelIterator) error{Empty, Drained}!Key {
+        pub fn peek(it: LevelIterator) error{ Empty, Drained }!Key {
             if (it.values.head_ptr_const()) |value| return key_from_value(value);
 
             const scope = it.tables.head_ptr_const() orelse {

package/src/tigerbeetle/src/lsm/manifest.zig

@@ -217,6 +217,8 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
             table: *TableInfo,
         ) void {
             const manifest_level = &manifest.levels[level];
+
+            assert(table.snapshot_max >= snapshot);
             manifest_level.set_snapshot_max(snapshot, table);
             assert(table.snapshot_max == snapshot);
 
@@ -545,8 +547,16 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
             return true;
         }
 
+        pub fn reserve(manifest: *Manifest) void {
+            assert(manifest.compact_callback == null);
+            assert(manifest.checkpoint_callback == null);
+
+            manifest.manifest_log.reserve();
+        }
+
         pub fn compact(manifest: *Manifest, callback: Callback) void {
             assert(manifest.compact_callback == null);
+            assert(manifest.checkpoint_callback == null);
             manifest.compact_callback = callback;
 
             manifest.manifest_log.compact(manifest_log_compact_callback);
@@ -555,6 +565,7 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
         fn manifest_log_compact_callback(manifest_log: *ManifestLog) void {
             const manifest = @fieldParentPtr(Manifest, "manifest_log", manifest_log);
             assert(manifest.compact_callback != null);
+            assert(manifest.checkpoint_callback == null);
 
             const callback = manifest.compact_callback.?;
             manifest.compact_callback = null;
@@ -562,6 +573,7 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
         }
 
         pub fn checkpoint(manifest: *Manifest, callback: Callback) void {
+            assert(manifest.compact_callback == null);
             assert(manifest.checkpoint_callback == null);
             manifest.checkpoint_callback = callback;
 
@@ -570,6 +582,7 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
 
         fn manifest_log_checkpoint_callback(manifest_log: *ManifestLog) void {
             const manifest = @fieldParentPtr(Manifest, "manifest_log", manifest_log);
+            assert(manifest.compact_callback == null);
             assert(manifest.checkpoint_callback != null);
 
             const callback = manifest.checkpoint_callback.?;

package/src/tigerbeetle/src/lsm/manifest_level.zig

@@ -1,52 +1,3 @@
-//! A ManifestLevel is an in-memory collection of the table metadata for a single level of a tree.
-//!
-//! For a given level and snapshot, there may be gaps in the key ranges of the visible tables,
-//! but the key ranges are disjoint.
-//!
-//! A level's tables can be visualized in 2D as a partitioned rectangle.
-//! For example, given the ManifestLevel tables (with values chosen for visualization, not realism):
-//!
-//!          label   A   B   C   D   E   F   G   H   I   J   K   L   M
-//!        key_min   0   4  12  16   4   8  12  26   4  25   4  16  24
-//!        key_max   3  11  15  19   7  11  15  27   7  27  11  19  27
-//!   snapshot_min   1   1   1   1   3   3   3   3   5   5   7   7   7
-//!   snapshot_max   9   3   3   7   5   7   9   5   7   7   9   9   9
-//!
-//!     0         1         2
-//!     0   4   8   2   6   0   4   8
-//!   9┌───┬───────┬───┬───┬───┬───┐
-//!    │   │   K   │   │ L │###│ M │
-//!   7│   ├───┬───┤   ├───┤###└┬──┤
-//!    │   │ I │   │ G │   │####│ J│
-//!   5│ A ├───┤ F │   │   │####└┬─┤
-//!    │   │ E │   │   │ D │#####│H│
-//!   3│   ├───┴───┼───┤   │#####└─┤
-//!    │   │   B   │ C │   │#######│
-//!   1└───┴───────┴───┴───┴───────┘
-//!
-//! Example iterations:
-//!
-//!   visibility  snapshots   direction  key_min  key_max  tables
-//!      visible          2   ascending        0       28  A, B, C, D
-//!      visible          4   ascending        0       28  A, E, F, G, D, H
-//!      visible          6  descending       12       28  J, D, G
-//!      visible          8   ascending        0       28  A, K, G, L, M
-//!    invisible    2, 4, 6   ascending        0       28  K, L, M
-//!
-//! Legend:
-//!
-//! * "#" represents a gap — no tables cover these keys during the snapshot.
-//! * The horizontal axis represents the key range.
-//! * The vertical axis represents the snapshot range.
-//! * Each rectangle is a table within the manifest level.
-//! * The sides of each rectangle depict:
-//!   * left:   table.key_min (the diagram is inclusive, and the table.key_min is inclusive)
-//!   * right:  table.key_max (the diagram is EXCLUSIVE, but the table.key_max is INCLUSIVE)
-//!   * bottom: table.snapshot_min (inclusive)
-//!   * top:    table.snapshot_max (inclusive)
-//! * (Not depicted: tables may have `table.key_min == table.key_max`.)
-//! * (Not depicted: the newest set of tables would have `table.snapshot_max == maxInt(u64)`.)
-//!
 const std = @import("std");
 const assert = std.debug.assert;
 const math = std.math;