tigerbeetle-node 0.11.5 → 0.11.7

package/src/tigerbeetle/src/lsm/table_iterator.zig

@@ -52,6 +52,9 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
         /// This field is only used for safety checks, it does not affect the behavior.
         read_pending: bool = false,
 
+        // Used for verifying key order when constants.verify == true.
+        key_prev: ?Table.Key,
+
         pub fn init(allocator: mem.Allocator) !TableIterator {
             const index_block = try allocator.alignedAlloc(
                 u8,
@@ -95,6 +98,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                     },
                 },
                 .value = undefined,
+                .key_prev = null,
             };
         }
 
@@ -132,10 +136,21 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                 .values = .{ .buffer = it.values.buffer },
                 .data_blocks = .{ .buffer = it.data_blocks.buffer },
                 .value = 0,
+                .key_prev = null,
             };
 
             assert(it.values.empty());
             assert(it.data_blocks.empty());
+
+            if (constants.verify) {
+                Table.verify(
+                    Storage,
+                    context.grid.superblock.storage,
+                    context.address,
+                    null,
+                    null,
+                );
+            }
         }
 
         /// Try to buffer at least a full block of values to be peek()'d.
@@ -269,7 +284,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
         /// Returns either:
         /// - the next Key, if available.
         /// - error.Empty when there are no values remaining to iterate.
-        /// - error.Drained when the iterator isn't empty, but some values 
+        /// - error.Drained when the iterator isn't empty, but some values
         ///   still need to be buffered into memory via tick().
         pub fn peek(it: TableIterator) error{ Empty, Drained }!Table.Key {
             assert(!it.read_pending);
@@ -280,7 +295,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
             const block = it.data_blocks.head() orelse {
                 // NOTE: Even if there are no values to peek, some may be unbuffered.
                 // We call buffered_all_values() to distinguish between the iterator
-                // being empty and needing to tic() to refill values.
+                // being empty and needing to tick() to refill values.
                 if (!it.buffered_all_values()) return error.Drained;
                 return error.Empty;
             };
@@ -291,6 +306,18 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
         /// This may only be called after peek() returns a Key (and not Empty or Drained)
         pub fn pop(it: *TableIterator) Table.Value {
+            const value = it.pop_internal();
+
+            if (constants.verify) {
+                const key = Table.key_from_value(&value);
+                if (it.key_prev) |k| assert(Table.compare_keys(k, key) == .lt);
+                it.key_prev = key;
+            }
+
+            return value;
+        }
+
+        fn pop_internal(it: *TableIterator) Table.Value {
             assert(!it.read_pending);
             assert(!it.read_table_index);
 

package/src/tigerbeetle/src/lsm/table_mutable.zig

@@ -14,6 +14,8 @@ pub fn TableMutableType(comptime Table: type) type {
     const compare_keys = Table.compare_keys;
     const key_from_value = Table.key_from_value;
     const tombstone_from_key = Table.tombstone_from_key;
+    const tombstone = Table.tombstone;
+    const usage = Table.usage;
 
     return struct {
         const TableMutable = @This();
@@ -96,26 +98,52 @@ pub fn TableMutableType(comptime Table: type) type {
         }
 
         pub fn put(table: *TableMutable, value: *const Value) void {
-            // If the key is already present in the hash map, the old key will not be overwritten
-            // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
-            // level getOrPut() API and manually overwrite the old key.
-            const upsert = table.values.getOrPutAssumeCapacity(value.*);
-            upsert.key_ptr.* = value.*;
+            switch (usage) {
+                .secondary_index => {
+                    const existing = table.values.fetchRemove(value.*);
+                    if (existing) |kv| {
+                        // If there was a previous operation on this key then it must have been a remove.
+                        // The put and remove cancel out.
+                        assert(tombstone(&kv.key));
+                    } else {
+                        table.values.putAssumeCapacityNoClobber(value.*, {});
+                    }
+                },
+                .general => {
+                    // If the key is already present in the hash map, the old key will not be overwritten
+                    // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
+                    // level getOrPut() API and manually overwrite the old key.
+                    const upsert = table.values.getOrPutAssumeCapacity(value.*);
+                    upsert.key_ptr.* = value.*;
+                },
+            }
 
             // The hash map's load factor may allow for more capacity because of rounding:
             assert(table.values.count() <= table.value_count_max);
-
-            if (table.values_cache) |cache| {
-                cache.insert(key_from_value(value)).* = value.*;
-            }
         }
 
         pub fn remove(table: *TableMutable, value: *const Value) void {
-            // If the key is already present in the hash map, the old key will not be overwritten
-            // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
-            // level getOrPut() API and manually overwrite the old key.
-            const upsert = table.values.getOrPutAssumeCapacity(value.*);
-            upsert.key_ptr.* = tombstone_from_key(key_from_value(value));
+            switch (usage) {
+                .secondary_index => {
+                    const existing = table.values.fetchRemove(value.*);
+                    if (existing) |kv| {
+                        // The previous operation on this key then it must have been a put.
+                        // The put and remove cancel out.
+                        assert(!tombstone(&kv.key));
+                    } else {
+                        // If the put is already on-disk, then we need to follow it with a tombstone.
+                        // The put and the tombstone may cancel each other out later during compaction.
+                        table.values.putAssumeCapacityNoClobber(tombstone_from_key(key_from_value(value)), {});
+                    }
+                },
+                .general => {
+                    // If the key is already present in the hash map, the old key will not be overwritten
+                    // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
+                    // level getOrPut() API and manually overwrite the old key.
+                    const upsert = table.values.getOrPutAssumeCapacity(value.*);
+                    upsert.key_ptr.* = tombstone_from_key(key_from_value(value));
+                },
+            }
 
             assert(table.values.count() <= table.value_count_max);
         }
@@ -154,7 +182,13 @@ pub fn TableMutableType(comptime Table: type) type {
             while (it.next()) |value| : (i += 1) {
                 values_max[i] = value.*;
 
-                if (table.values_cache) |cache| cache.insert(key_from_value(value)).* = value.*;
+                if (table.values_cache) |cache| {
+                    if (tombstone(value)) {
+                        cache.remove(key_from_value(value));
+                    } else {
+                        cache.insert(value);
+                    }
+                }
             }
 
             const values = values_max[0..i];

package/src/tigerbeetle/src/lsm/test.zig

@@ -27,7 +27,7 @@ const SuperBlock = vsr.SuperBlockType(Storage);
 const Environment = struct {
     const cluster = 32;
     const replica = 4;
-    const size_max = vsr.Zone.superblock.size().? +
+    const storage_size_max = vsr.Zone.superblock.size().? +
         vsr.Zone.wal_headers.size().? +
         vsr.Zone.wal_prepares.size().? +
         (512 + 64) * 1024 * 1024;
@@ -35,8 +35,7 @@ const Environment = struct {
     const node_count = 1024;
     const cache_entries_max = 2 * 1024 * 1024;
     const forest_options = StateMachine.forest_options(.{
-        // Ignored by StateMachine.forest_options().
-        .lsm_forest_node_count = undefined,
+        .lsm_forest_node_count = node_count,
         .cache_entries_accounts = cache_entries_max,
         .cache_entries_transfers = cache_entries_max,
         .cache_entries_posted = cache_entries_max,
@@ -73,7 +72,7 @@ const Environment = struct {
         env.dir_fd = try IO.open_dir(dir_path);
         errdefer std.os.close(env.dir_fd);
 
-        env.fd = try IO.open_file(env.dir_fd, "test_forest", size_max, must_create);
+        env.fd = try IO.open_file(env.dir_fd, "test_forest", storage_size_max, must_create);
         errdefer std.os.close(env.fd);
 
         env.io = try IO.init(128, 0);
@@ -85,7 +84,11 @@ const Environment = struct {
         env.message_pool = try MessagePool.init(allocator, .replica);
         errdefer env.message_pool.deinit(allocator);
 
-        env.superblock = try SuperBlock.init(allocator, &env.storage, &env.message_pool);
+        env.superblock = try SuperBlock.init(allocator, .{
+            .storage = &env.storage,
+            .storage_size_limit = constants.storage_size_max,
+            .message_pool = &env.message_pool,
+        });
         env.superblock_context = undefined;
         errdefer env.superblock.deinit(allocator);
 
@@ -117,7 +120,7 @@ const Environment = struct {
     }
 
     fn tick(env: *Environment) !void {
-        env.grid.tick();
+        // env.grid.tick();
         try env.io.tick();
     }
 
@@ -132,7 +135,7 @@ const Environment = struct {
         env.superblock.format(superblock_format_callback, &env.superblock_context, .{
             .cluster = cluster,
             .replica = replica,
-            .size_max = size_max,
+            .storage_size_max = storage_size_max,
         });
 
         while (true) {

package/src/tigerbeetle/src/lsm/tree.zig

@@ -212,13 +212,26 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             var manifest = try Manifest.init(allocator, node_pool, grid, tree_hash);
             errdefer manifest.deinit(allocator);
 
-            var compaction_table_immutable = try CompactionTableImmutable.init(allocator, tree_name);
+            var compaction_table_immutable = try CompactionTableImmutable.init(
+                allocator,
+                std.fmt.comptimePrint("{s}(immutable->0)", .{tree_name}),
+            );
             errdefer compaction_table_immutable.deinit(allocator);
 
             var compaction_table: [@divFloor(constants.lsm_levels, 2)]CompactionTable = undefined;
-            for (compaction_table) |*compaction, i| {
-                errdefer for (compaction_table[0..i]) |*c| c.deinit(allocator);
-                compaction.* = try CompactionTable.init(allocator, tree_name);
+            {
+                comptime var i: usize = 0;
+                inline while (i < compaction_table.len) : (i += 1) {
+                    errdefer for (compaction_table[0..i]) |*c| c.deinit(allocator);
+                    const compaction_name = std.fmt.comptimePrint("{s}({}->{}/{}->{})", .{
+                        tree_name,
+                        2 * i,
+                        2 * i + 1,
+                        2 * i + 1,
+                        2 * i + 2,
+                    });
+                    compaction_table[i] = try CompactionTable.init(allocator, compaction_name);
+                }
             }
             errdefer for (compaction_table) |*c| c.deinit(allocator);
 
@@ -580,10 +593,14 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback == null);
 
+            if (constants.verify) {
+                tree.manifest.verify(tree.compaction_op);
+            }
+
             tracer.start(
                 &tree.tracer_slot,
                 .{ .tree = .{ .tree_name = tree_name } },
-                .{ .tree_compaction_beat = .{ .tree_name = tree_name } },
+                .tree_compaction_beat,
                 @src(),
             );
 
@@ -959,9 +976,13 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             tracer.end(
                 &tree.tracer_slot,
                 .{ .tree = .{ .tree_name = tree_name } },
-                .{ .tree_compaction_beat = .{ .tree_name = tree_name } },
+                .tree_compaction_beat,
             );
 
+            if (constants.verify) {
+                tree.manifest.verify(tree.compaction_op);
+            }
+
             // Invoke the compact() callback after the manifest compacts at the end of the beat.
             const callback = tree.compaction_callback.?;
             tree.compaction_callback = null;