tigerbeetle-node 0.11.8 → 0.11.9

package/src/tigerbeetle/src/lsm/segmented_array.zig

@@ -4,8 +4,8 @@ const assert = std.debug.assert;
 const math = std.math;
 const mem = std.mem;
 
-const util = @import("../util.zig");
-const div_ceil = @import("../util.zig").div_ceil;
+const stdx = @import("../stdx.zig");
+const div_ceil = @import("../stdx.zig").div_ceil;
 const binary_search_values_raw = @import("binary_search.zig").binary_search_values_raw;
 const binary_search_keys = @import("binary_search.zig").binary_search_keys;
 const Direction = @import("direction.zig").Direction;
@@ -280,13 +280,13 @@ fn SegmentedArrayType(
 
             const total = array.count(a) + @intCast(u32, elements.len);
             if (total <= node_capacity) {
-                util.copy_right(
+                stdx.copy_right(
                     .inexact,
                     T,
                     a_pointer[cursor.relative_index + elements.len ..],
                     a_pointer[cursor.relative_index..array.count(a)],
                 );
-                util.copy_disjoint(.inexact, T, a_pointer[cursor.relative_index..], elements);
+                stdx.copy_disjoint(.inexact, T, a_pointer[cursor.relative_index..], elements);
 
                 array.increment_indexes_after(a, @intCast(u32, elements.len));
                 return;
@@ -348,7 +348,7 @@ fn SegmentedArrayType(
 
             if (a_half < cursor.relative_index) {
                 // Move the part of `a` that is past the half-way point into `b`.
-                util.copy_right(
+                stdx.copy_right(
                     .inexact,
                     T,
                     b_half_pointer,
@@ -383,10 +383,10 @@ fn SegmentedArrayType(
             const source_a = source[0..target_a.len];
             const source_b = source[target_a.len..];
             if (target_b.ptr != source_b.ptr) {
-                util.copy_right(.exact, T, target_b, source_b);
+                stdx.copy_right(.exact, T, target_b, source_b);
             }
             if (target_a.ptr != source_a.ptr) {
-                util.copy_right(.exact, T, target_a, source_a);
+                stdx.copy_right(.exact, T, target_a, source_a);
             }
         }
 
@@ -395,13 +395,13 @@ fn SegmentedArrayType(
             assert(node <= array.node_count);
             assert(array.node_count + 1 <= node_count_max);
 
-            util.copy_right(
+            stdx.copy_right(
                 .exact,
                 ?*[node_capacity]T,
                 array.nodes[node + 1 .. array.node_count + 1],
                 array.nodes[node..array.node_count],
             );
-            util.copy_right(
+            stdx.copy_right(
                 .exact,
                 u32,
                 array.indexes[node + 1 .. array.node_count + 2],
@@ -470,7 +470,7 @@ fn SegmentedArrayType(
 
             // Remove elements from exactly one node:
             if (a_remaining + remove_count <= array.count(a)) {
-                util.copy_left(
+                stdx.copy_left(
                     .inexact,
                     T,
                     a_pointer[a_remaining..],
@@ -497,7 +497,7 @@ fn SegmentedArrayType(
             assert(a_remaining > 0 or b_remaining.len > 0);
 
             if (a_remaining >= half) {
-                util.copy_left(.inexact, T, b_pointer, b_remaining);
+                stdx.copy_left(.inexact, T, b_pointer, b_remaining);
 
                 array.indexes[b] = array.indexes[a] + a_remaining;
                 array.decrement_indexes_after(b, remove_count);
@@ -514,7 +514,7 @@ fn SegmentedArrayType(
                 assert(a_remaining < half and b_remaining.len < half);
                 assert(a_remaining + b_remaining.len <= node_capacity);
 
-                util.copy_disjoint(.inexact, T, a_pointer[a_remaining..], b_remaining);
+                stdx.copy_disjoint(.inexact, T, a_pointer[a_remaining..], b_remaining);
 
                 array.indexes[b] = array.indexes[a] + a_remaining + @intCast(u32, b_remaining.len);
                 array.decrement_indexes_after(b, remove_count);
@@ -576,7 +576,7 @@ fn SegmentedArrayType(
 
             const total = array.count(a) + @intCast(u32, b_elements.len);
             if (total <= node_capacity) {
-                util.copy_disjoint(.inexact, T, a_pointer[array.count(a)..], b_elements);
+                stdx.copy_disjoint(.inexact, T, a_pointer[array.count(a)..], b_elements);
 
                 array.indexes[b] = array.indexes[b + 1];
                 array.remove_empty_node_at(node_pool, b);
@@ -588,13 +588,13 @@ fn SegmentedArrayType(
                 assert(a_half >= b_half);
                 assert(a_half + b_half == total);
 
-                util.copy_disjoint(
+                stdx.copy_disjoint(
                     .exact,
                     T,
                     a_pointer[array.count(a)..a_half],
                     b_elements[0 .. a_half - array.count(a)],
                 );
-                util.copy_left(.inexact, T, b_pointer, b_elements[a_half - array.count(a) ..]);
+                stdx.copy_left(.inexact, T, b_pointer, b_elements[a_half - array.count(a) ..]);
 
                 array.indexes[b] = array.indexes[a] + a_half;
 
@@ -616,13 +616,13 @@ fn SegmentedArrayType(
                 @ptrCast(NodePool.Node, @alignCast(NodePool.node_alignment, array.nodes[node].?)),
             );
 
-            util.copy_left(
+            stdx.copy_left(
                 .exact,
                 ?*[node_capacity]T,
                 array.nodes[node .. array.node_count - 1],
                 array.nodes[node + 1 .. array.node_count],
             );
-            util.copy_left(
+            stdx.copy_left(
                 .exact,
                 u32,
                 array.indexes[node..array.node_count],

package/src/tigerbeetle/src/lsm/segmented_array_fuzz.zig

@@ -1,6 +1,6 @@
 const std = @import("std");
 
-const fuzz = @import("../test/fuzz.zig");
+const fuzz = @import("../testing/fuzz.zig");
 const segmented_array = @import("segmented_array.zig");
 
 pub fn main() !void {

package/src/tigerbeetle/src/lsm/set_associative_cache.zig

@@ -8,7 +8,7 @@ const meta = std.meta;
 const Vector = meta.Vector;
 
 const constants = @import("../constants.zig");
-const div_ceil = @import("../util.zig").div_ceil;
+const div_ceil = @import("../stdx.zig").div_ceil;
 const verify = constants.verify;
 
 pub const Layout = struct {

package/src/tigerbeetle/src/lsm/table.zig

@@ -8,12 +8,13 @@ const vsr = @import("../vsr.zig");
 const binary_search = @import("binary_search.zig");
 const bloom_filter = @import("bloom_filter.zig");
 
-const util = @import("../util.zig");
-const div_ceil = util.div_ceil;
+const stdx = @import("../stdx.zig");
+const div_ceil = stdx.div_ceil;
 const eytzinger = @import("eytzinger.zig").eytzinger;
 const snapshot_latest = @import("tree.zig").snapshot_latest;
 
 const BlockType = @import("grid.zig").BlockType;
+const alloc_block = @import("grid.zig").alloc_block;
 const TableInfoType = @import("manifest.zig").TableInfoType;
 
 pub const TableUsage = enum {
@@ -484,13 +485,13 @@ pub fn TableType(
             data_blocks_in_filter: u32 = 0,
 
             pub fn init(allocator: mem.Allocator) !Builder {
-                const index_block = try allocator.alignedAlloc(u8, constants.sector_size, block_size);
+                const index_block = try alloc_block(allocator);
                 errdefer allocator.free(index_block);
 
-                const filter_block = try allocator.alignedAlloc(u8, constants.sector_size, block_size);
+                const filter_block = try alloc_block(allocator);
                 errdefer allocator.free(filter_block);
 
-                const data_block = try allocator.alignedAlloc(u8, constants.sector_size, block_size);
+                const data_block = try alloc_block(allocator);
                 errdefer allocator.free(data_block);
 
                 return Builder{
@@ -527,7 +528,7 @@ pub fn TableType(
                 const values_max = data_block_values(builder.data_block);
                 assert(values_max.len == data.value_count_max);
 
-                util.copy_disjoint(.inexact, Value, values_max[builder.value..], values);
+                stdx.copy_disjoint(.inexact, Value, values_max[builder.value..], values);
                 builder.value += @intCast(u32, values.len);
 
                 for (values) |*value| {
@@ -596,8 +597,6 @@ pub fn TableType(
 
                 const values_padding = mem.sliceAsBytes(values_max[builder.value..]);
                 const block_padding = block[data.padding_offset..][0..data.padding_size];
-                mem.set(u8, values_padding, 0);
-                mem.set(u8, block_padding, 0);
                 assert(compare_keys(key_from_value(&values[values.len - 1]), key_max) == .eq);
 
                 const header_bytes = block[0..@sizeOf(vsr.Header)];
@@ -712,17 +711,6 @@ pub fn TableType(
 
                 const index_block = builder.index_block;
 
-                const index_data_keys_padding = index_data_keys(index_block)[builder.data_block_count..];
-                const index_data_keys_padding_bytes = mem.sliceAsBytes(index_data_keys_padding);
-                mem.set(u8, index_data_keys_padding_bytes, 0);
-                mem.set(u64, index_data_addresses(index_block)[builder.data_block_count..], 0);
-                mem.set(u128, index_data_checksums(index_block)[builder.data_block_count..], 0);
-
-                mem.set(u64, index_filter_addresses(index_block)[builder.filter_block_count..], 0);
-                mem.set(u128, index_filter_checksums(index_block)[builder.filter_block_count..], 0);
-
-                mem.set(u8, index_block[index.padding_offset..][0..index.padding_size], 0);
-
                 const header_bytes = index_block[0..@sizeOf(vsr.Header)];
                 const header = mem.bytesAsValue(vsr.Header, header_bytes);
 
@@ -992,7 +980,7 @@ pub fn TableType(
             key_min: ?Key,
             key_max: ?Key,
         ) void {
-            if (Storage != @import("../test/storage.zig").Storage)
+            if (Storage != @import("../testing/storage.zig").Storage)
                 // Too complicated to do async verification
                 return;
 

package/src/tigerbeetle/src/lsm/table_immutable.zig

@@ -4,7 +4,7 @@ const math = std.math;
 const assert = std.debug.assert;
 
 const constants = @import("../constants.zig");
-const div_ceil = @import("../util.zig").div_ceil;
+const div_ceil = @import("../stdx.zig").div_ceil;
 const binary_search = @import("binary_search.zig");
 const snapshot_latest = @import("tree.zig").snapshot_latest;
 

package/src/tigerbeetle/src/lsm/table_iterator.zig

@@ -5,7 +5,7 @@ const assert = std.debug.assert;
 
 const constants = @import("../constants.zig");
 
-const util = @import("../util.zig");
+const stdx = @import("../stdx.zig");
 const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
 const ManifestType = @import("manifest.zig").ManifestType;
 const GridType = @import("grid.zig").GridType;
@@ -208,7 +208,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
             // Copy the bytes read into a buffer owned by the iterator since the Grid
             // only guarantees the provided pointer to be valid in this callback.
-            util.copy_disjoint(.exact, u8, it.index_block, block);
+            stdx.copy_disjoint(.exact, u8, it.index_block, block);
 
             if (it.index_block_callback) |callback| {
                 it.index_block_callback = null;
@@ -240,7 +240,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
             // Copy the bytes read into a buffer owned by the iterator since the Grid
             // only guarantees the provided pointer to be valid in this callback.
-            util.copy_disjoint(.exact, u8, it.data_blocks.next_tail().?, block);
+            stdx.copy_disjoint(.exact, u8, it.data_blocks.next_tail().?, block);
 
             it.data_blocks.advance_tail();
             it.data_block_index += 1;

package/src/tigerbeetle/src/lsm/table_mutable.zig

@@ -4,7 +4,7 @@ const math = std.math;
 const assert = std.debug.assert;
 
 const constants = @import("../constants.zig");
-const div_ceil = @import("../util.zig").div_ceil;
+const div_ceil = @import("../stdx.zig").div_ceil;
 const SetAssociativeCache = @import("set_associative_cache.zig").SetAssociativeCache;
 
 /// Range queries are not supported on the TableMutable, it must first be made immutable.
@@ -43,6 +43,13 @@ pub fn TableMutableType(comptime Table: type) type {
         value_count_max: u32,
         values: Values = .{},
 
+        /// Rather than using values.count(), we count how many values we could have had if every
+        /// operation had been on a different key. This means that mistakes in calculating
+        /// commit_entries_max are much easier to catch when fuzzing, rather than requiring very
+        /// specific workloads.
+        /// Invariant: value_count_worst_case <= value_count_max
+        value_count_worst_case: u32 = 0,
+
         /// This is used to accelerate point lookups and is not used for range queries.
         /// Secondary index trees used only for range queries can therefore set this to null.
         ///
@@ -98,6 +105,8 @@ pub fn TableMutableType(comptime Table: type) type {
         }
 
         pub fn put(table: *TableMutable, value: *const Value) void {
+            assert(table.value_count_worst_case < table.value_count_max);
+            table.value_count_worst_case += 1;
             switch (usage) {
                 .secondary_index => {
                     const existing = table.values.fetchRemove(value.*);
@@ -123,6 +132,8 @@ pub fn TableMutableType(comptime Table: type) type {
         }
 
         pub fn remove(table: *TableMutable, value: *const Value) void {
+            assert(table.value_count_worst_case < table.value_count_max);
+            table.value_count_worst_case += 1;
             switch (usage) {
                 .secondary_index => {
                     const existing = table.values.fetchRemove(value.*);
@@ -152,11 +163,12 @@ pub fn TableMutableType(comptime Table: type) type {
         /// assumes that none of the batch's keys are already in `table.values`.
         pub fn can_commit_batch(table: *TableMutable, batch_count: u32) bool {
             assert(batch_count <= table.value_count_max);
-            return (table.count() + batch_count) <= table.value_count_max;
+            return (table.value_count_worst_case + batch_count) <= table.value_count_max;
         }
 
         pub fn clear(table: *TableMutable) void {
             assert(table.values.count() > 0);
+            table.value_count_worst_case = 0;
             table.values.clearRetainingCapacity();
             assert(table.values.count() == 0);
         }

package/src/tigerbeetle/src/lsm/tree.zig

@@ -10,7 +10,7 @@ const log = std.log.scoped(.tree);
 const tracer = @import("../tracer.zig");
 
 const constants = @import("../constants.zig");
-const div_ceil = @import("../util.zig").div_ceil;
+const div_ceil = @import("../stdx.zig").div_ceil;
 const eytzinger = @import("eytzinger.zig").eytzinger;
 const vsr = @import("../vsr.zig");
 const bloom_filter = @import("bloom_filter.zig");
@@ -145,6 +145,7 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
 
         compaction_io_pending: usize,
         compaction_callback: ?fn (*Tree) void,
+        compaction_next_tick: Grid.NextTick = undefined,
 
         checkpoint_callback: ?fn (*Tree) void,
         open_callback: ?fn (*Tree) void,
@@ -336,7 +337,8 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             }
 
             if (index_block_count == 0) {
-                callback(context, null);
+                context.callback = callback;
+                tree.grid.on_next_tick(lookup_invalid_tick_callback, &context.next_tick);
                 return;
             }
 
@@ -346,6 +348,7 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             context.* = .{
                 .tree = tree,
                 .completion = undefined,
+                .next_tick = undefined,
 
                 .key = key,
                 .fingerprint = fingerprint,
@@ -360,12 +363,18 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             context.read_index_block();
         }
 
+        fn lookup_invalid_tick_callback(next_tick: *Grid.NextTick) void {
+            const context = @fieldParentPtr(LookupContext, "next_tick", next_tick);
+            context.callback(context, null);
+        }
+
         pub const LookupContext = struct {
             const Read = Grid.Read;
             const BlockPtrConst = Grid.BlockPtrConst;
 
             tree: *Tree,
             completion: Read,
+            next_tick: Grid.NextTick,
 
             key: Key,
             fingerprint: bloom_filter.Fingerprint,
@@ -560,8 +569,8 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
                     tree.compact_mutable_table_into_immutable();
                 }
 
-                // TODO Defer this callback until tick() to avoid stack growth.
-                callback(tree);
+                tree.compaction_callback = callback;
+                tree.grid.on_next_tick(compact_skip_tick_callback, &tree.compaction_next_tick);
                 return;
             }
 
@@ -589,6 +598,13 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             tree.compact_drive();
         }
 
+        fn compact_skip_tick_callback(next_tick: *Grid.NextTick) void {
+            const tree = @fieldParentPtr(Tree, "compaction_next_tick", next_tick);
+            const callback = tree.compaction_callback.?;
+            tree.compaction_callback = null;
+            callback(tree);
+        }
+
         fn compact_start(tree: *Tree, callback: fn (*Tree) void) void {
             assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback == null);
@@ -858,7 +874,7 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
                 // We are at the end of a half-bar, but the compactions have not finished.
                 // We keep ticking them until they finish.
                 log.debug(tree_name ++ ": compact_done: driving outstanding compactions", .{});
-                tree.compact_drive();
+                tree.grid.on_next_tick(compact_drive_tick_callback, &tree.compaction_next_tick);
                 return;
             }
 
@@ -936,6 +952,16 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             tree.manifest.compact(compact_manifest_callback);
         }
 
+        /// Asynchronously continue to drive the compactions when they haven't finished at the time
+        /// they were supposed to at the end of a half-bar.
+        fn compact_drive_tick_callback(next_tick: *Grid.NextTick) void {
+            const tree = @fieldParentPtr(Tree, "compaction_next_tick", next_tick);
+            assert(tree.compaction_io_pending == 0);
+            assert(tree.compaction_callback != null);
+            assert(tree.compaction_op == tree.lookup_snapshot_max);
+            tree.compact_drive();
+        }
+
         /// Called after the last beat of a full compaction bar.
         fn compact_mutable_table_into_immutable(tree: *Tree) void {
             assert(tree.table_immutable.free);