tigerbeetle-node 0.10.0 → 0.11.0

package/src/tigerbeetle/src/lsm/table_immutable.zig

@@ -22,9 +22,12 @@ pub fn TableImmutableType(comptime Table: type) type {
         snapshot_min: u64,
         free: bool,
 
-        pub fn init(allocator: mem.Allocator, commit_count_max: u32) !TableImmutable {
+        /// `commit_entries_max` is the maximum number of Values that can be inserted by a single commit.
+        pub fn init(allocator: mem.Allocator, commit_entries_max: u32) !TableImmutable {
+            assert(commit_entries_max > 0);
+
             // The in-memory immutable table is the same size as the mutable table:
-            const value_count_max = commit_count_max * config.lsm_batch_multiple;
+            const value_count_max = commit_entries_max * config.lsm_batch_multiple;
             const data_block_count = div_ceil(value_count_max, Table.data.value_count_max);
             assert(data_block_count <= Table.data_block_count_max);
 
@@ -45,11 +48,13 @@ pub fn TableImmutableType(comptime Table: type) type {
         }
 
         pub inline fn key_min(table: *const TableImmutable) Key {
+            assert(!table.free);
             assert(table.values.len > 0);
             return key_from_value(&table.values[0]);
         }
 
         pub inline fn key_max(table: *const TableImmutable) Key {
+            assert(!table.free);
             assert(table.values.len > 0);
             return key_from_value(&table.values[table.values.len - 1]);
         }
@@ -96,7 +101,7 @@ pub fn TableImmutableType(comptime Table: type) type {
                     assert(i > 0);
                     const left_key = key_from_value(&sorted_values[i - 1]);
                     const right_key = key_from_value(&sorted_values[i]);
-                    assert(compare_keys(left_key, right_key) != .gt);
+                    assert(compare_keys(left_key, right_key) == .lt);
                 }
             }
 
@@ -112,20 +117,19 @@ pub fn TableImmutableType(comptime Table: type) type {
         pub fn get(table: *const TableImmutable, key: Key) ?*const Value {
             assert(!table.free);
 
-            if (table.values.len > 0) {
-                const result = binary_search.binary_search_values(
-                    Key,
-                    Value,
-                    key_from_value,
-                    compare_keys,
-                    table.values,
-                    key,
-                );
-                if (result.exact) {
-                    const value = &table.values[result.index];
-                    if (config.verify) assert(compare_keys(key, key_from_value(value)) == .eq);
-                    return value;
-                }
+            const result = binary_search.binary_search_values(
+                Key,
+                Value,
+                key_from_value,
+                compare_keys,
+                table.values,
+                key,
+                .{},
+            );
+            if (result.exact) {
+                const value = &table.values[result.index];
+                if (config.verify) assert(compare_keys(key, key_from_value(value)) == .eq);
+                return value;
             }
 
             return null;
@@ -140,7 +144,7 @@ pub fn TableImmutableIteratorType(comptime Table: type, comptime Storage: type)
         const TableImmutableIterator = @This();
         const TableImmutable = TableImmutableType(Table);
 
-        table: *TableImmutable,
+        table: *const TableImmutable,
         values_index: u32,
 
         pub fn init(allocator: mem.Allocator) !TableImmutableIterator {
@@ -158,7 +162,7 @@ pub fn TableImmutableIteratorType(comptime Table: type, comptime Storage: type)
         }
 
         pub const Context = struct {
-            table: *TableImmutable,
+            table: *const TableImmutable,
         };
 
         pub fn start(
@@ -174,21 +178,24 @@ pub fn TableImmutableIteratorType(comptime Table: type, comptime Storage: type)
         }
 
         pub fn tick(it: *const TableImmutableIterator) bool {
-            _ = it;
+            assert(!it.table.free);
             return false; // No I/O is performed as it's all in memory.
         }
 
         pub fn buffered_all_values(it: *const TableImmutableIterator) bool {
-            _ = it;
+            assert(!it.table.free);
             return true; // All values are "buffered" in memory.
         }
 
-        pub fn peek(it: *const TableImmutableIterator) ?Table.Key {
-            if (it.values_index == it.table.values.len) return null;
+        pub fn peek(it: *const TableImmutableIterator) error{Empty, Drained}!Table.Key {
+            // NOTE: This iterator is never Drained as all values are in memory (tick is a no-op).
+            assert(!it.table.free);
+            if (it.values_index == it.table.values.len) return error.Empty;
             return Table.key_from_value(&it.table.values[it.values_index]);
         }
 
         pub fn pop(it: *TableImmutableIterator) Table.Value {
+            assert(!it.table.free);
             defer it.values_index += 1;
             return it.table.values[it.values_index];
         }

package/src/tigerbeetle/src/lsm/table_iterator.zig

@@ -9,6 +9,7 @@ const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
 const ManifestType = @import("manifest.zig").ManifestType;
 const GridType = @import("grid.zig").GridType;
 
+/// A TableIterator iterates a table's values in ascending-key order.
 pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
     return struct {
         const TableIterator = @This();
@@ -38,9 +39,8 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
         /// out of blocks in the blocks ring buffer but haven't buffered a full block of
         /// values in memory. In this case, we copy values from the head of blocks to this
         /// ring buffer to make that block available for reading further values.
-        /// Thus, we guarantee that iterators will always have at least a block's worth of
-        /// values buffered. This simplifies the peek() interface as null always means that
-        /// iteration is complete.
+        /// Thus, we guarantee that iterators will always have at least a block's worth
+        /// of values buffered.
         values: ValuesRingBuffer,
 
         data_blocks: RingBuffer(Grid.BlockPtr, 2, .array),
@@ -100,8 +100,8 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
         pub const Context = struct {
             grid: *Grid,
-            address: u64,
-            checksum: u128,
+            address: u64, // Table index block address.
+            checksum: u128, // Table index block checksum.
             index_block_callback: ?IndexBlockCallback = null,
         };
 
@@ -148,6 +148,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                     &it.read,
                     it.address,
                     it.checksum,
+                    .index,
                 );
                 return true;
             }
@@ -171,29 +172,31 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
             assert(!it.read_pending);
             it.read_pending = true;
-            it.grid.read_block(on_read, &it.read, address, checksum);
+            it.grid.read_block(on_read, &it.read, address, checksum, .data);
         }
 
         fn on_read_table_index(read: *Grid.Read, block: Grid.BlockPtrConst) void {
             const it = @fieldParentPtr(TableIterator, "read", read);
             assert(it.read_pending);
+            assert(it.data_block_index == 0);
             it.read_pending = false;
 
             assert(it.read_table_index);
             it.read_table_index = false;
 
+            // Copy the bytes read into a buffer owned by the iterator since the Grid
+            // only guarantees the provided pointer to be valid in this callback.
+            mem.copy(u8, it.index_block, block);
+
             if (it.index_block_callback) |callback| {
                 it.index_block_callback = null;
                 callback(it, block);
             }
 
-            // Copy the bytes read into a buffer owned by the iterator since the Grid
-            // only guarantees the provided pointer to be valid in this callback.
-            mem.copy(u8, it.index_block, block);
-
             const read_pending = it.tick();
             // After reading the table index, we always read at least one data block.
             assert(read_pending);
+            assert(it.read_pending);
         }
 
         fn on_read(read: *Grid.Read, block: Grid.BlockPtrConst) void {
@@ -256,22 +259,30 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                 it.buffered_value_count() >= Table.data.value_count_max;
         }
 
-        pub fn peek(it: TableIterator) ?Table.Key {
+        /// Returns either:
+        /// - the next Key, if available.
+        /// - error.Empty when there are no values remaining to iterate.
+        /// - error.Drained when the iterator isn't empty, but some values 
+        ///   still need to be buffered into memory via tick().
+        pub fn peek(it: TableIterator) error{Empty, Drained}!Table.Key {
             assert(!it.read_pending);
             assert(!it.read_table_index);
 
             if (it.values.head_ptr_const()) |value| return Table.key_from_value(value);
 
             const block = it.data_blocks.head() orelse {
-                assert(it.data_block_index == Table.index_data_blocks_used(it.index_block));
-                return null;
+                // NOTE: Even if there are no values to peek, some may be unbuffered.
+                // We call buffered_all_values() to distinguish between the iterator
+                // being empty and needing to tic() to refill values.
+                if (!it.buffered_all_values()) return error.Drained;
+                return error.Empty;
             };
 
             const values = Table.data_block_values_used(block);
             return Table.key_from_value(&values[it.value]);
         }
 
-        /// This is only safe to call after peek() has returned non-null.
+        /// This may only be called after peek() returns a Key (and not Empty or Drained)
         pub fn pop(it: *TableIterator) Table.Value {
             assert(!it.read_pending);
             assert(!it.read_table_index);

package/src/tigerbeetle/src/lsm/table_mutable.zig

@@ -5,6 +5,7 @@ const assert = std.debug.assert;
 
 const config = @import("../config.zig");
 const div_ceil = @import("../util.zig").div_ceil;
+const SetAssociativeCache = @import("set_associative_cache.zig").SetAssociativeCache;
 
 /// Range queries are not supported on the TableMutable, it must first be made immutable.
 pub fn TableMutableType(comptime Table: type) type {
@@ -20,15 +21,51 @@ pub fn TableMutableType(comptime Table: type) type {
         const load_factor = 50;
         const Values = std.HashMapUnmanaged(Value, void, Table.HashMapContextValue, load_factor);
 
+        pub const ValuesCache = SetAssociativeCache(
+            Key,
+            Value,
+            Table.key_from_value,
+            struct {
+                inline fn hash(key: Key) u64 {
+                    return std.hash.Wyhash.hash(0, mem.asBytes(&key));
+                }
+            }.hash,
+            struct {
+                inline fn equal(a: Key, b: Key) bool {
+                    return compare_keys(a, b) == .eq;
+                }
+            }.equal,
+            .{},
+        );
+
         value_count_max: u32,
         values: Values = .{},
-        dirty: bool = false,
 
-        pub fn init(allocator: mem.Allocator, commit_count_max: u32) !TableMutable {
+        /// This is used to accelerate point lookups and is not used for range queries.
+        /// Secondary index trees used only for range queries can therefore set this to null.
+        ///
+        /// The values cache is only used for the latest snapshot for simplicity.
+        /// Earlier snapshots will still be able to utilize the block cache.
+        ///
+        /// The values cache is updated (in bulk) when the mutable table is sorted and frozen,
+        /// rather than updating on every `put()`/`remove()`.
+        /// This amortizes cache inserts for hot keys in the mutable table, and avoids redundantly
+        /// storing duplicate values in both the mutable table and values cache.
+        // TODO Share cache between trees of different grooves:
+        // "A set associative cache of values shared by trees with the same key/value sizes.
+        // The value type will be []u8 and this will be shared by trees with the same value size."
+        values_cache: ?*ValuesCache,
+
+        /// `commit_entries_max` is the maximum number of Values that can be inserted by a single commit.
+        pub fn init(
+            allocator: mem.Allocator,
+            values_cache: ?*ValuesCache,
+            commit_entries_max: u32,
+        ) !TableMutable {
             comptime assert(config.lsm_batch_multiple > 0);
-            assert(commit_count_max > 0);
+            assert(commit_entries_max > 0);
 
-            const value_count_max = commit_count_max * config.lsm_batch_multiple;
+            const value_count_max = commit_entries_max * config.lsm_batch_multiple;
             const data_block_count = div_ceil(value_count_max, Table.data.value_count_max);
             assert(data_block_count <= Table.data_block_count_max);
 
@@ -39,6 +76,7 @@ pub fn TableMutableType(comptime Table: type) type {
             return TableMutable{
                 .value_count_max = value_count_max,
                 .values = values,
+                .values_cache = values_cache,
             };
         }
 
@@ -47,31 +85,43 @@ pub fn TableMutableType(comptime Table: type) type {
         }
 
         pub fn get(table: *const TableMutable, key: Key) ?*const Value {
-            return table.values.getKeyPtr(tombstone_from_key(key));
+            if (table.values.getKeyPtr(tombstone_from_key(key))) |value| {
+                return value;
+            }
+            if (table.values_cache) |cache| {
+                // Check the cache after the mutable table (see `values_cache` for explanation).
+                if (cache.get(key)) |value| return value;
+            }
+            return null;
         }
 
         pub fn put(table: *TableMutable, value: *const Value) void {
             // If the key is already present in the hash map, the old key will not be overwritten
             // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
             // level getOrPut() API and manually overwrite the old key.
-            const gop = table.values.getOrPutAssumeCapacity(value.*);
-            gop.key_ptr.* = value.*;
+            const upsert = table.values.getOrPutAssumeCapacity(value.*);
+            upsert.key_ptr.* = value.*;
 
             // The hash map's load factor may allow for more capacity because of rounding:
             assert(table.values.count() <= table.value_count_max);
-            table.dirty = true;
+
+            if (table.values_cache) |cache| {
+                cache.insert(key_from_value(value)).* = value.*;
+            }
         }
 
         pub fn remove(table: *TableMutable, value: *const Value) void {
             // If the key is already present in the hash map, the old key will not be overwritten
             // by the new one if using e.g. putAssumeCapacity(). Instead we must use the lower
             // level getOrPut() API and manually overwrite the old key.
-            const tombstone = tombstone_from_key(key_from_value(value));
-            const gop = table.values.getOrPutAssumeCapacity(tombstone);
-            gop.key_ptr.* = tombstone;
+            const upsert = table.values.getOrPutAssumeCapacity(value.*);
+            upsert.key_ptr.* = tombstone_from_key(key_from_value(value));
+
             assert(table.values.count() <= table.value_count_max);
         }
 
+        /// This may return `false` even when committing would succeed — it pessimistically
+        /// assumes that none of the batch's keys are already in `table.values`.
         pub fn can_commit_batch(table: *TableMutable, batch_count: u32) bool {
             assert(batch_count <= table.value_count_max);
             return (table.count() + batch_count) <= table.value_count_max;
@@ -81,7 +131,6 @@ pub fn TableMutableType(comptime Table: type) type {
             assert(table.values.count() > 0);
             table.values.clearRetainingCapacity();
             assert(table.values.count() == 0);
-            table.dirty = false;
         }
 
         pub fn count(table: *const TableMutable) u32 {
@@ -104,6 +153,8 @@ pub fn TableMutableType(comptime Table: type) type {
             var it = table.values.keyIterator();
             while (it.next()) |value| : (i += 1) {
                 values_max[i] = value.*;
+
+                if (table.values_cache) |cache| cache.insert(key_from_value(value)).* = value.*;
             }
 
             const values = values_max[0..i];

package/src/tigerbeetle/src/lsm/test.zig

@@ -13,49 +13,31 @@ const Transfer = @import("../tigerbeetle.zig").Transfer;
 const Account = @import("../tigerbeetle.zig").Account;
 const Storage = @import("../storage.zig").Storage;
 const IO = @import("../io.zig").IO;
+const StateMachine = @import("../state_machine.zig").StateMachineType(Storage, .{
+    .message_body_size_max = config.message_body_size_max,
+});
 
 const GridType = @import("grid.zig").GridType;
 const GrooveType = @import("groove.zig").GrooveType;
-const ForestType = @import("forest.zig").ForestType;
+const Forest = StateMachine.Forest;
 
 const Grid = GridType(Storage);
 const SuperBlock = vsr.SuperBlockType(Storage);
-const Forest = ForestType(Storage, .{
-    .accounts = GrooveType(
-        Storage,
-        Account,
-        .{
-            .ignored = &[_][]const u8{ "reserved", "flags" },
-            .derived = .{},
-        },
-    ),
-    .transfers = GrooveType(
-        Storage,
-        Transfer,
-        .{
-            .ignored = &[_][]const u8{ "reserved", "flags" },
-            .derived = .{},
-        },
-    ),
-});
 
 const Environment = struct {
     const cluster = 32;
     const replica = 4;
-    const size_max = (512 + 64) * 1024 * 1024;
+    const size_max = vsr.Zone.superblock.size().? + vsr.Zone.wal.size().? + (512 + 64) * 1024 * 1024;
 
     const node_count = 1024;
-    const cache_size = 2 * 1024 * 1024;
-    const forest_config = .{
-        .transfers = .{
-            .cache_size = cache_size,
-            .commit_count_max = 8191 * 2,
-        },
-        .accounts = .{
-            .cache_size = cache_size,
-            .commit_count_max = 8191,
-        },
-    };
+    const cache_entries_max = 2 * 1024 * 1024;
+    const forest_options = StateMachine.forest_options(.{
+        // Ignored by StateMachine.forest_options().
+        .lsm_forest_node_count = undefined,
+        .cache_entries_accounts = cache_entries_max,
+        .cache_entries_transfers = cache_entries_max,
+        .cache_entries_posted = cache_entries_max,
+    });
 
     const State = enum {
         uninit,
@@ -78,6 +60,8 @@ const Environment = struct {
     superblock_context: SuperBlock.Context,
     grid: Grid,
     forest: Forest,
+    // We need @fieldParentPtr() of forest, so we can't use an optional Forest.
+    forest_exists: bool = false,
 
     fn init(env: *Environment, must_create: bool) !void {
         env.state = .uninit;
@@ -105,8 +89,9 @@ const Environment = struct {
         env.grid = try Grid.init(allocator, &env.superblock);
         errdefer env.grid.deinit(allocator);
 
-        env.forest = try Forest.init(allocator, &env.grid, node_count, forest_config);
-        errdefer env.forest.deinit(allocator);
+        // Forest must be initialized with an open superblock.
+        env.forest = undefined;
+        env.forest_exists = false;
 
         env.state = .init;
     }
@@ -115,7 +100,10 @@ const Environment = struct {
         assert(env.state != .uninit);
         defer env.state = .uninit;
 
-        env.forest.deinit(allocator);
+        if (env.forest_exists) {
+            env.forest.deinit(allocator);
+            env.forest_exists = false;
+        }
         env.grid.deinit(allocator);
         env.superblock.deinit(allocator);
         env.message_pool.deinit(allocator);
@@ -176,6 +164,9 @@ const Environment = struct {
         const env = @fieldParentPtr(@This(), "superblock_context", superblock_context);
         assert(env.state == .init);
         env.state = .superblock_open;
+
+        env.forest = Forest.init(allocator, &env.grid, node_count, forest_options) catch unreachable;
+        env.forest_exists = true;
         env.forest.open(forest_open_callback);
     }
 
@@ -247,7 +238,7 @@ const Environment = struct {
         comptime visibility: Visibility,
         groove: anytype,
         objects: anytype,
-        comptime commit_count_max: u32,
+        comptime commit_entries_max: u32,
     ) !void {
         const Groove = @TypeOf(groove.*);
         const Object = @TypeOf(objects[0]);
@@ -260,9 +251,10 @@ const Environment = struct {
 
             fn verify(assertion: *@This()) void {
                 assert(assertion.verify_count == 0);
-                assertion.verify_count = std.math.min(commit_count_max, assertion.objects.len);
+                assertion.verify_count = std.math.min(commit_entries_max, assertion.objects.len);
                 if (assertion.verify_count == 0) return;
 
+                assertion.groove.prefetch_setup(null);
                 for (assertion.objects[0..assertion.verify_count]) |*object| {
                     assertion.groove.prefetch_enqueue(object.id);
                 }
@@ -275,9 +267,8 @@ const Environment = struct {
                 assert(assertion.verify_count > 0);
 
                 {
-                    defer assertion.groove.prefetch_clear();
                     for (assertion.objects[0..assertion.verify_count]) |*object| {
-                        log.debug("verifying {} for id={}", .{visibility, object.id});
+                        log.debug("verifying {} for id={}", .{ visibility, object.id });
                         const result = assertion.groove.get(object.id);
 
                         switch (visibility) {
@@ -285,7 +276,7 @@ const Environment = struct {
                             .visible => {
                                 assert(result != null);
                                 assert(std.mem.eql(u8, std.mem.asBytes(result.?), std.mem.asBytes(object)));
-                            }
+                            },
                         }
                     }
                 }
@@ -319,18 +310,18 @@ const Environment = struct {
         var inserted = std.ArrayList(Account).init(allocator);
         defer inserted.deinit();
 
-        const accounts_to_insert_per_op = 1; // forest_config.accounts.commit_count_max;
+        const accounts_to_insert_per_op = 1; // forest_options.accounts.commit_entries_max;
         const iterations = 4;
 
-        var op: u64 = 0;
-        var id: u128 = 0;
+        var op: u64 = 1;
+        var id: u128 = 1;
         var timestamp: u64 = 42;
         var crash_probability = std.rand.DefaultPrng.init(1337);
 
         var iter: usize = 0;
         while (iter < (accounts_to_insert_per_op * config.lsm_batch_multiple * iterations)) : (iter += 1) {
             // Insert a bunch of accounts
-            
+
             var i: u32 = 0;
             while (i < accounts_to_insert_per_op) : (i += 1) {
                 defer id += 1;
@@ -349,7 +340,7 @@ const Environment = struct {
                     .credits_posted = 42,
                 };
 
-                 // Insert an account ...
+                // Insert an account ...
                 const groove = &env.forest.grooves.accounts;
                 groove.put(&account);
 
@@ -357,8 +348,8 @@ const Environment = struct {
                 try env.assert_visibility(
                     .visible,
                     &env.forest.grooves.accounts,
-                    @as([]const Account, &.{ account }),
-                    forest_config.accounts.commit_count_max,
+                    @as([]const Account, &.{account}),
+                    forest_options.accounts.tree_options_object.commit_entries_max,
                 );
 
                 // Record the successfull insertion.
@@ -369,15 +360,20 @@ const Environment = struct {
             try env.compact(op);
             defer op += 1;
 
-            // checkpoint the records when the forest is likely finished compaction.
+            // Checkpoint when the forest finishes compaction.
+            // Don't repeat a checkpoint (commit_min must always advance).
             const checkpoint_op = op -| config.lsm_batch_multiple;
-            if (checkpoint_op % config.lsm_batch_multiple == config.lsm_batch_multiple - 1) {
+            if (checkpoint_op % config.lsm_batch_multiple == config.lsm_batch_multiple - 1 and
+                checkpoint_op != env.superblock.staging.vsr_state.commit_min)
+            {
                 // Checkpoint the forest then superblock
+                env.superblock.staging.vsr_state.commit_min = checkpoint_op;
+                env.superblock.staging.vsr_state.commit_max = checkpoint_op;
                 try env.checkpoint();
 
-                const checkpointed = inserted.items[0..((checkpoint_op + 1) * accounts_to_insert_per_op)];
+                const checkpointed = inserted.items[0 .. checkpoint_op * accounts_to_insert_per_op];
                 const uncommitted = inserted.items[checkpointed.len..];
-                log.debug("checkpointed={d} uncommitted={d}", .{checkpointed.len, uncommitted.len});
+                log.debug("checkpointed={d} uncommitted={d}", .{ checkpointed.len, uncommitted.len });
                 assert(uncommitted.len == config.lsm_batch_multiple * accounts_to_insert_per_op);
 
                 // Randomly initiate a crash
@@ -397,7 +393,7 @@ const Environment = struct {
                             .invisible,
                             &env.forest.grooves.accounts,
                             uncommitted,
-                            forest_config.accounts.commit_count_max,
+                            forest_options.accounts.tree_options_object.commit_entries_max,
                         );
 
                         // Reset everything to after checkpoint
@@ -407,14 +403,14 @@ const Environment = struct {
                         timestamp = checkpointed[checkpointed.len - 1].timestamp + 1;
                     }
                     crashing = false;
-                } 
+                }
 
                 // Double check the forest contains the checkpointed values (positive space)
                 try env.assert_visibility(
                     .visible,
                     &env.forest.grooves.accounts,
                     checkpointed,
-                    forest_config.accounts.commit_count_max,
+                    forest_options.accounts.tree_options_object.commit_entries_max,
                 );
             }
         }
@@ -425,5 +421,3 @@ pub fn main() !void {
     try Environment.format(); // NOTE: this can be commented out after first run to speed up testing.
     try Environment.run(); //try do_simple();
 }
-
-