tigerbeetle-node 0.11.5 → 0.11.7

package/src/tigerbeetle/src/lsm/groove.zig

@@ -131,6 +131,7 @@ fn IndexTreeType(
         Key.sentinel_key,
         Key.tombstone,
         Key.tombstone_from_key,
+        .secondary_index,
     );
 
     return TreeType(Table, Storage, tree_name);
@@ -254,6 +255,7 @@ pub fn GrooveType(
             ObjectTreeHelpers(Object).sentinel_key,
             ObjectTreeHelpers(Object).tombstone,
             ObjectTreeHelpers(Object).tombstone_from_key,
+            .general,
         );
 
         const tree_name = @typeName(Object);
@@ -269,6 +271,7 @@ pub fn GrooveType(
             IdTreeValue.sentinel_key,
             IdTreeValue.tombstone,
             IdTreeValue.tombstone_from_key,
+            .general,
         );
 
         const tree_name = @typeName(Object) ++ ".id";

package/src/tigerbeetle/src/lsm/level_iterator.zig

@@ -68,6 +68,9 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
         values: ValuesRingBuffer,
         tables: TablesRingBuffer,
 
+        // Used for verifying key order when constants.verify == true.
+        key_prev: ?Key,
+
         pub fn init(allocator: mem.Allocator) !LevelIterator {
             var values = try ValuesRingBuffer.init(allocator);
             errdefer values.deinit(allocator);
@@ -96,6 +99,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
                         .{ .table_iterator = table_b },
                     },
                 },
+                .key_prev = null,
             };
         }
 
@@ -138,6 +142,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
                 .direction = context.direction,
                 .values = .{ .buffer = it.values.buffer },
                 .tables = .{ .buffer = it.tables.buffer },
+                .key_prev = null,
             };
 
             assert(it.key_exclusive == null);
@@ -281,7 +286,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
         /// Returns either:
         /// - the next Key, if available.
         /// - error.Empty when there are no values remaining to iterate.
-        /// - error.Drained when the iterator isn't empty, but the values 
+        /// - error.Drained when the iterator isn't empty, but the values
         ///   still need to be buffered into memory via tick().
         pub fn peek(it: LevelIterator) error{ Empty, Drained }!Key {
             if (it.values.head_ptr_const()) |value| return key_from_value(value);
@@ -299,6 +304,18 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
 
         /// This may only be called after peek() returns a Key (and not Empty or Drained)
         pub fn pop(it: *LevelIterator) Value {
+            const value = it.pop_internal();
+
+            if (constants.verify) {
+                const key = Table.key_from_value(&value);
+                if (it.key_prev) |k| assert(Table.compare_keys(k, key) == .lt);
+                it.key_prev = key;
+            }
+
+            return value;
+        }
+
+        fn pop_internal(it: *LevelIterator) Value {
             if (it.values.pop()) |value| return value;
 
             const table_iterator = &it.tables.head_ptr().?.table_iterator;

package/src/tigerbeetle/src/lsm/manifest.zig

@@ -531,7 +531,7 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
                     .ascending,
                     KeyRange{ .key_min = range.key_min, .key_max = range.key_max },
                 );
-                if (it.next() == null) {
+                if (it.next() != null) {
                     // If the range is being compacted into the last level then this is unreachable,
                     // as the last level has no subsequent levels and must always drop tombstones.
                     assert(level_b != constants.lsm_levels - 1);
@@ -585,5 +585,33 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
             manifest.checkpoint_callback = null;
             callback(manifest);
         }
+
+        pub fn verify(manifest: *Manifest, snapshot: u64) void {
+            for (manifest.levels) |*level| {
+                var key_max_prev: ?Key = null;
+                var table_info_iter = level.iterator(
+                    .visible,
+                    &.{snapshot},
+                    .ascending,
+                    null,
+                );
+                while (table_info_iter.next()) |table_info| {
+                    if (key_max_prev) |k| {
+                        assert(compare_keys(k, table_info.key_min) == .lt);
+                    }
+                    // We could have key_min == key_max if there is only one value.
+                    assert(compare_keys(table_info.key_min, table_info.key_max) != .gt);
+                    key_max_prev = table_info.key_max;
+
+                    Table.verify(
+                        Storage,
+                        manifest.manifest_log.grid.superblock.storage,
+                        table_info.address,
+                        table_info.key_min,
+                        table_info.key_max,
+                    );
+                }
+            }
+        }
     };
 }

package/src/tigerbeetle/src/lsm/manifest_level.zig

@@ -417,6 +417,7 @@ pub fn TestContext(
             std.math.maxInt(Key),
             tombstone,
             tombstone_from_key,
+            .general,
         );
 
         const TableInfo = @import("manifest.zig").TableInfoType(Table);

package/src/tigerbeetle/src/lsm/manifest_log.zig

@@ -404,14 +404,14 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
                 return;
             }
 
-            const block = manifest_log.blocks.head().?;
-            verify_block(block, null, null);
+            const block = manifest_log.blocks.head_ptr().?;
+            verify_block(block.*, null, null);
 
-            const header = mem.bytesAsValue(vsr.Header, block[0..@sizeOf(vsr.Header)]);
-            const address = Block.address(block);
+            const header = mem.bytesAsValue(vsr.Header, block.*[0..@sizeOf(vsr.Header)]);
+            const address = Block.address(block.*);
             assert(address > 0);
 
-            const entry_count = Block.entry_count(block);
+            const entry_count = Block.entry_count(block.*);
 
             if (manifest_log.blocks_closed == 1 and manifest_log.blocks.count == 1) {
                 // This might be the last block of a checkpoint, which can be a partial block.

package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig

@@ -30,8 +30,6 @@ const fuzz = @import("../test/fuzz.zig");
 
 pub const tigerbeetle_config = @import("../config.zig").configs.test_min;
 
-const storage_size_max = data_file_size_min + constants.block_size * 1024;
-
 const entries_max_block = ManifestLog.Block.entry_count_max;
 const entries_max_buffered = entries_max_block *
     std.meta.fieldInfo(ManifestLog, .blocks).field_type.count_max;
@@ -66,20 +64,28 @@ fn run_fuzz(
         .write_latency_mean = 1 + random.uintLessThan(u64, 40),
     };
 
-    var storage = try Storage.init(allocator, storage_size_max, storage_options);
+    var storage = try Storage.init(allocator, constants.storage_size_max, storage_options);
     defer storage.deinit(allocator);
 
-    var storage_verify = try Storage.init(allocator, storage_size_max, storage_options);
+    var storage_verify = try Storage.init(allocator, constants.storage_size_max, storage_options);
     defer storage_verify.deinit(allocator);
 
     // The MessagePool is shared by both superblocks because they will not use it.
     var message_pool = try MessagePool.init(allocator, .replica);
     defer message_pool.deinit(allocator);
 
-    var superblock = try SuperBlock.init(allocator, &storage, &message_pool);
+    var superblock = try SuperBlock.init(allocator, .{
+        .storage = &storage,
+        .storage_size_limit = constants.storage_size_max,
+        .message_pool = &message_pool,
+    });
     defer superblock.deinit(allocator);
 
-    var superblock_verify = try SuperBlock.init(allocator, &storage_verify, &message_pool);
+    var superblock_verify = try SuperBlock.init(allocator, .{
+        .storage = &storage_verify,
+        .storage_size_limit = constants.storage_size_max,
+        .message_pool = &message_pool,
+    });
     defer superblock_verify.deinit(allocator);
 
     var grid = try Grid.init(allocator, &superblock);
@@ -313,7 +319,7 @@ const Environment = struct {
 
     fn wait(env: *Environment, manifest_log: *ManifestLog) void {
         while (env.pending > 0) {
-            manifest_log.grid.tick();
+            // manifest_log.grid.tick();
             manifest_log.superblock.storage.tick();
         }
     }
@@ -324,7 +330,6 @@ const Environment = struct {
         env.manifest_log.superblock.format(format_superblock_callback, &env.superblock_context, .{
             .cluster = 0,
             .replica = 0,
-            .size_max = storage_size_max,
         });
     }
 
@@ -453,8 +458,11 @@ const Environment = struct {
             test_superblock.deinit(env.allocator);
             test_superblock.* = try SuperBlock.init(
                 env.allocator,
-                test_storage,
-                env.manifest_log.superblock.client_table.message_pool,
+                .{
+                    .storage = test_storage,
+                    .storage_size_limit = constants.storage_size_max,
+                    .message_pool = env.manifest_log.superblock.client_table.message_pool,
+                },
             );
 
             test_grid.deinit(env.allocator);
@@ -627,7 +635,7 @@ fn verify_manifest_compaction_set(
     var compact_blocks_checked: u32 = 0;
 
     // This test doesn't include any actual table blocks, so all blocks are manifest blocks.
-    var blocks = superblock.free_set.blocks.iterator(.{ .kind = .unset });
+    var blocks = superblock.free_set.blocks.iterator(.{ .kind = .set });
     while (blocks.next()) |block_index| {
         const block_address = block_index + 1;
         const block = superblock.storage.grid_block(block_address);

package/src/tigerbeetle/src/lsm/merge_iterator.zig

@@ -0,0 +1,106 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const util = @import("../util.zig");
+
+const k_max = 2;
+
+pub fn MergeIteratorType(
+    comptime Table: type,
+    comptime IteratorA: type,
+    comptime IteratorB: type,
+) type {
+    return struct {
+        const Self = @This();
+
+        iterator_a: *IteratorA,
+        iterator_b: *IteratorB,
+
+        empty_a: bool,
+        empty_b: bool,
+
+        previous_key_popped: ?Table.Key,
+
+        pub fn init(
+            iterator_a: *IteratorA,
+            iterator_b: *IteratorB,
+        ) Self {
+            return Self{
+                .iterator_a = iterator_a,
+                .iterator_b = iterator_b,
+                .empty_a = false,
+                .empty_b = false,
+                .previous_key_popped = null,
+            };
+        }
+
+        /// Returns `true` once both `iterator_a` and `iterator_b` have raised `error.Empty`.
+        pub fn empty(it: Self) bool {
+            return it.empty_a and it.empty_b;
+        }
+
+        /// Returns `null` if either `iterator_a` or `iterator_b` return `error.Empty` or `error.Drained`.
+        /// Check `it.empty()` to disambiguate.
+        pub fn pop(it: *Self) ?Table.Value {
+            while (true) {
+                if (it.empty_a and it.empty_b) {
+                    return null;
+                }
+
+                if (it.empty_a) {
+                    _ = it.iterator_b.peek() catch |error_b| switch (error_b) {
+                        error.Drained => return null,
+                        error.Empty => {
+                            it.empty_b = true;
+                            continue;
+                        },
+                    };
+                    return it.iterator_b.pop();
+                }
+
+                if (it.empty_b) {
+                    _ = it.iterator_a.peek() catch |error_a| switch (error_a) {
+                        error.Drained => return null,
+                        error.Empty => {
+                            it.empty_a = true;
+                            continue;
+                        },
+                    };
+                    return it.iterator_a.pop();
+                }
+
+                const key_a = it.iterator_a.peek() catch |error_a| switch (error_a) {
+                    error.Drained => return null,
+                    error.Empty => {
+                        it.empty_a = true;
+                        continue;
+                    },
+                };
+                const key_b = it.iterator_b.peek() catch |error_b| switch (error_b) {
+                    error.Drained => return null,
+                    error.Empty => {
+                        it.empty_b = true;
+                        continue;
+                    },
+                };
+
+                switch (Table.compare_keys(key_a, key_b)) {
+                    .lt => return it.iterator_a.pop(),
+                    .gt => return it.iterator_b.pop(),
+                    .eq => {
+                        const value_a = it.iterator_a.pop();
+                        const value_b = it.iterator_b.pop();
+                        switch (Table.usage) {
+                            .general => return value_a,
+                            .secondary_index => {
+                                // In secondary indexes, puts and removes alternate and can be safely cancelled out.
+                                assert(Table.tombstone(&value_a) != Table.tombstone(&value_b));
+                                continue;
+                            },
+                        }
+                    },
+                }
+            }
+        }
+    };
+}

package/src/tigerbeetle/src/lsm/posted_groove.zig

@@ -67,6 +67,7 @@ pub fn PostedGrooveType(comptime Storage: type) type {
             Value.sentinel_key,
             Value.tombstone,
             Value.tombstone_from_key,
+            .general,
         );
 
         const Tree = TreeType(Table, Storage, "posted_groove");

package/src/tigerbeetle/src/lsm/segmented_array.zig

@@ -1241,6 +1241,7 @@ pub fn run_tests(seed: u64, comptime options: Options) !void {
         Key.sentinel_key,
         Key.tombstone,
         Key.tombstone_from_key,
+        .general,
     ));
 
     const CompareInt = struct {

package/src/tigerbeetle/src/lsm/set_associative_cache.zig

@@ -188,21 +188,26 @@ pub fn SetAssociativeCache(
             mem.set(u64, self.clocks.words, 0);
         }
 
-        /// Returns whether an entry with the given key is cached, 
+        /// Returns whether an entry with the given key is cached,
         /// without modifying the entry's counter.
         pub fn exists(self: *Self, key: Key) bool {
             const set = self.associate(key);
             return self.search(set, key) != null;
         }
 
-        pub fn get(self: *Self, key: Key) ?*align(value_alignment) Value {
+        pub fn get_index(self: *Self, key: Key) ?usize {
             const set = self.associate(key);
             const way = self.search(set, key) orelse return null;
 
             const count = self.counts.get(set.offset + way);
             self.counts.set(set.offset + way, count +| 1);
 
-            return @alignCast(value_alignment, &set.values[way]);
+            return set.offset + way;
+        }
+
+        pub fn get(self: *Self, key: Key) ?*align(value_alignment) Value {
+            const index = self.get_index(key) orelse return null;
+            return @alignCast(value_alignment, &self.values[index]);
         }
 
         /// Remove a key from the set associative cache if present.
@@ -211,6 +216,7 @@ pub fn SetAssociativeCache(
             const way = self.search(set, key) orelse return;
 
             self.counts.set(set.offset + way, 0);
+            set.values[way] = undefined;
         }
 
         /// Hint that the key is less likely to be accessed in the future, without actually removing
@@ -248,39 +254,21 @@ pub fn SetAssociativeCache(
             return @ptrCast(*const Ways, &result).*;
         }
 
-        pub fn insert(self: *Self, key: Key) *align(value_alignment) Value {
-            return self.insert_preserve_locked(
-                void,
-                struct {
-                    inline fn locked(_: void, _: *const Value) bool {
-                        return false;
-                    }
-                }.locked,
-                {},
-                key,
-            );
+        /// Insert a value, evicting an older entry if needed.
+        pub fn insert(self: *Self, value: *const Value) void {
+            _ = self.insert_index(value);
         }
 
-        /// Add a key, evicting an older entry if needed, and return a pointer to the value.
-        /// The key must not already be in the cache.
-        /// Never evicts keys for which locked() returns true.
-        /// The caller must guarantee that locked() returns true for less than layout.ways keys.
-        pub fn insert_preserve_locked(
-            self: *Self,
-            comptime Context: type,
-            comptime locked: fn (
-                Context,
-                *align(value_alignment) const Value,
-            ) callconv(.Inline) bool,
-            context: Context,
-            key: Key,
-        ) *align(value_alignment) Value {
+        /// Insert a value, evicting an older entry if needed.
+        /// Return the index at which the value was inserted.
+        pub fn insert_index(self: *Self, value: *const Value) usize {
+            const key = key_from_value(value);
             const set = self.associate(key);
             if (self.search(set, key)) |way| {
-                // Remove the old entry for this key.
-                // It should be a different value, but since we are returning a value pointer we
-                // can't check against the new one.
-                self.counts.set(set.offset + way, 0);
+                // Overwrite the old entry for this key.
+                self.counts.set(set.offset + way, 1);
+                set.values[way] = value.*;
+                return set.offset + way;
             }
 
             const clock_index = @divExact(set.offset, layout.ways);
@@ -299,11 +287,6 @@ pub fn SetAssociativeCache(
                 safety_count += 1;
                 way +%= 1;
             }) {
-                // We pass a value pointer to the callback here so that a cache miss
-                // can be avoided if the caller is able to determine if the value is
-                // locked by comparing pointers directly.
-                if (locked(context, @alignCast(value_alignment, &set.values[way]))) continue;
-
                 var count = self.counts.get(set.offset + way);
                 if (count == 0) break; // Way is already free.
 
@@ -316,10 +299,11 @@ pub fn SetAssociativeCache(
             assert(self.counts.get(set.offset + way) == 0);
 
             set.tags[way] = set.tag;
+            set.values[way] = value.*;
             self.counts.set(set.offset + way, 1);
             self.clocks.set(clock_index, way +% 1);
 
-            return @alignCast(value_alignment, &set.values[way]);
+            return set.offset + way;
         }
 
         const Set = struct {
@@ -430,7 +414,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.insert(key).* = key;
+                    sac.insert(&key);
                     try expect(sac.counts.get(i) == 1);
                     try expectEqual(key, sac.get(key).?.*);
                     try expect(sac.counts.get(i) == 2);
@@ -443,7 +427,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.insert(key).* = key;
+                sac.insert(&key);
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);
@@ -460,34 +444,6 @@ fn set_associative_cache_test(
 
             if (log) sac.associate(0).inspect(sac);
 
-            // Lock all other slots, causing key layout.ways * sac.sets to be evicted despite having the
-            // highest count.
-            {
-                {
-                    assert(sac.counts.get(0) == 2);
-                    var i: usize = 1;
-                    while (i < layout.ways) : (i += 1) assert(sac.counts.get(i) == 1);
-                }
-
-                const key = (layout.ways + 1) * sac.sets;
-                const expect_evicted = layout.ways * sac.sets;
-
-                sac.insert_preserve_locked(
-                    u64,
-                    struct {
-                        inline fn locked(only_unlocked: u64, value: *const Value) bool {
-                            return value.* != only_unlocked;
-                        }
-                    }.locked,
-                    expect_evicted,
-                    key,
-                ).* = key;
-
-                try expectEqual(@as(?*Value, null), sac.get(expect_evicted));
-            }
-
-            if (log) sac.associate(0).inspect(sac);
-
             // Ensure removal works.
             {
                 const key = 5 * sac.sets;
@@ -512,7 +468,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.insert(key).* = key;
+                    sac.insert(&key);
                     try expect(sac.counts.get(i) == 1);
                     var j: usize = 2;
                     while (j <= math.maxInt(SAC.Count)) : (j += 1) {
@@ -530,7 +486,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.insert(key).* = key;
+                sac.insert(&key);
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);

package/src/tigerbeetle/src/lsm/table.zig

@@ -16,6 +16,17 @@ const snapshot_latest = @import("tree.zig").snapshot_latest;
 const BlockType = @import("grid.zig").BlockType;
 const TableInfoType = @import("manifest.zig").TableInfoType;
 
+pub const TableUsage = enum {
+    /// General purpose table.
+    general,
+    /// If your usage fits this pattern:
+    /// * Only put keys which are not present.
+    /// * Only remove keys which are present.
+    /// * TableKey == TableValue (modulo padding, eg CompositeKey)
+    /// Then we can unlock additional optimizations.
+    secondary_index,
+};
+
 /// A table is a set of blocks:
 ///
 /// * Index block (exactly 1)
@@ -69,6 +80,7 @@ pub fn TableType(
     comptime table_tombstone: fn (*const TableValue) callconv(.Inline) bool,
     /// Returns a tombstone value representation for a key.
     comptime table_tombstone_from_key: fn (TableKey) callconv(.Inline) TableValue,
+    comptime usage: TableUsage,
 ) type {
     return struct {
         const Table = @This();
@@ -81,6 +93,7 @@ pub fn TableType(
         pub const sentinel_key = table_sentinel_key;
         pub const tombstone = table_tombstone;
         pub const tombstone_from_key = table_tombstone_from_key;
+        pub const usage = usage;
 
         // Export hashmap context for Key and Value
         pub const HashMapContextValue = struct {
@@ -616,6 +629,11 @@ pub fn TableType(
                     assert(compare_keys(builder.key_min, builder.key_max) == .lt);
                 }
 
+                if (current > 0) {
+                    const key_max_prev = index_data_keys(builder.index_block)[current - 1];
+                    assert(compare_keys(key_max_prev, key_from_value(&values[0])) == .lt);
+                }
+
                 builder.data_block_count += 1;
                 builder.value = 0;
 
@@ -966,6 +984,43 @@ pub fn TableType(
 
             return null;
         }
+
+        pub fn verify(
+            comptime Storage: type,
+            storage: *Storage,
+            index_address: u64,
+            key_min: ?Key,
+            key_max: ?Key,
+        ) void {
+            if (Storage != @import("../test/storage.zig").Storage)
+                // Too complicated to do async verification
+                return;
+
+            const index_block = storage.grid_block(index_address);
+            const addresses = index_data_addresses(index_block);
+            const data_blocks_used = index_data_blocks_used(index_block);
+            var data_block_index: usize = 0;
+            while (data_block_index < data_blocks_used) : (data_block_index += 1) {
+                const address = addresses[data_block_index];
+                const data_block = storage.grid_block(address);
+                const values = data_block_values_used(data_block);
+                if (values.len > 0) {
+                    if (data_block_index == 0) {
+                        assert(key_min == null or
+                            compare_keys(key_min.?, key_from_value(&values[0])) == .eq);
+                    }
+                    if (data_block_index == data_blocks_used - 1) {
+                        assert(key_max == null or
+                            compare_keys(key_from_value(&values[values.len - 1]), key_max.?) == .eq);
+                    }
+                    var a = &values[0];
+                    for (values[1..]) |*b| {
+                        assert(compare_keys(key_from_value(a), key_from_value(b)) == .lt);
+                        a = b;
+                    }
+                }
+            }
+        }
     };
 }
 
@@ -980,6 +1035,7 @@ test "Table" {
         Key.sentinel_key,
         Key.tombstone,
         Key.tombstone_from_key,
+        .general,
     );
 
     _ = Table;