tigerbeetle-node 0.10.0 → 0.11.0

package/src/tigerbeetle/src/lsm/groove.zig

@@ -13,6 +13,7 @@ const CompositeKey = @import("composite_key.zig").CompositeKey;
 const NodePool = @import("node_pool.zig").NodePool(config.lsm_manifest_node_size, 16);
 
 const snapshot_latest = @import("tree.zig").snapshot_latest;
+const compaction_snapshot_for_op = @import("tree.zig").compaction_snapshot_for_op;
 
 fn ObjectTreeHelpers(comptime Object: type) type {
     assert(@hasField(Object, "id"));
@@ -49,6 +50,12 @@ const IdTreeValue = extern struct {
     timestamp: u64,
     padding: u64 = 0,
 
+    comptime {
+        // Assert that there is no implicit padding.
+        assert(@sizeOf(IdTreeValue) == 32);
+        assert(@bitSizeOf(IdTreeValue) == 32 * 8);
+    }
+
     inline fn compare_keys(a: u128, b: u128) std.math.Order {
         return std.math.order(a, b);
     }
@@ -78,7 +85,7 @@ fn IndexCompositeKeyType(comptime Field: type) type {
         .Enum => |e| {
             return switch (@bitSizeOf(e.tag_type)) {
                 0...@bitSizeOf(u64) => u64,
-                @bitSizeOf(u64)...@bitSizeOf(u128) => u128,
+                @bitSizeOf(u65)...@bitSizeOf(u128) => u128,
                 else => @compileError("Unsupported enum tag for index: " ++ @typeName(e.tag_type)),
             };
         },
@@ -88,7 +95,7 @@ fn IndexCompositeKeyType(comptime Field: type) type {
             }
             return switch (@bitSizeOf(Field)) {
                 0...@bitSizeOf(u64) => u64,
-                @bitSizeOf(u64)...@bitSizeOf(u128) => u128,
+                @bitSizeOf(u65)...@bitSizeOf(u128) => u128,
                 else => @compileError("Unsupported int type for index: " ++ @typeName(Field)),
             };
         },
@@ -131,6 +138,9 @@ fn IndexTreeType(
 
 /// A Groove is a collection of LSM trees auto generated for fields on a struct type
 /// as well as custom derived fields from said struct type.
+///
+/// Invariants:
+/// - Between beats, all of a groove's trees share the same lookup_snapshot_max.
 pub fn GrooveType(
     comptime Storage: type,
     comptime Object: type,
@@ -142,7 +152,7 @@ pub fn GrooveType(
     /// - derived: { .field = fn (*const Object) ?DerivedType }:
     ///     An anonymous struct which contain fields that don't exist on the Object
     ///     but can be derived from an Object instance using the field's corresponding function.
-    comptime options: anytype,
+    comptime groove_options: anytype,
 ) type {
     @setEvalBranchQuota(64000);
 
@@ -154,13 +164,13 @@ pub fn GrooveType(
     comptime var index_fields: []const std.builtin.TypeInfo.StructField = &.{};
 
     // Generate index LSM trees from the struct fields.
-    inline for (std.meta.fields(Object)) |field| {
+    for (std.meta.fields(Object)) |field| {
         // See if we should ignore this field from the options.
         //
         // By default, we ignore the "timestamp" field since it's a special identifier.
-        // Since the "timestamp" is ignored by default, it shouldn't be provided in options.ignored.
+        // Since the "timestamp" is ignored by default, it shouldn't be provided in groove_options.ignored.
         comptime var ignored = mem.eql(u8, field.name, "timestamp") or mem.eql(u8, field.name, "id");
-        inline for (options.ignored) |ignored_field_name| {
+        for (groove_options.ignored) |ignored_field_name| {
             comptime assert(!std.mem.eql(u8, ignored_field_name, "timestamp"));
             comptime assert(!std.mem.eql(u8, ignored_field_name, "id"));
             ignored = ignored or std.mem.eql(u8, field.name, ignored_field_name);
@@ -181,11 +191,11 @@ pub fn GrooveType(
         }
     }
 
-    // Generiate IndexTrees for fields derived from the Value in options.
-    const derived_fields = std.meta.fields(@TypeOf(options.derived));
-    inline for (derived_fields) |field| {
+    // Generate IndexTrees for fields derived from the Value in groove_options.
+    const derived_fields = std.meta.fields(@TypeOf(groove_options.derived));
+    for (derived_fields) |field| {
         // Get the function info for the derived field.
-        const derive_func = @field(options.derived, field.name);
+        const derive_func = @field(groove_options.derived, field.name);
         const derive_func_info = @typeInfo(@TypeOf(derive_func)).Fn;
 
         // Make sure it has only one argument.
@@ -221,6 +231,20 @@ pub fn GrooveType(
         };
     }
 
+    comptime var index_options_fields: []const std.builtin.TypeInfo.StructField = &.{};
+    for (index_fields) |index_field| {
+        const IndexTree = index_field.field_type;
+        index_options_fields = index_options_fields ++ [_]std.builtin.TypeInfo.StructField{
+            .{
+                .name = index_field.name,
+                .field_type = IndexTree.Options,
+                .default_value = null,
+                .is_comptime = false,
+                .alignment = @alignOf(IndexTree.Options),
+            },
+        };
+    }
+
     const ObjectTree = blk: {
         const Table = TableType(
             u64, // key = timestamp
@@ -259,6 +283,14 @@ pub fn GrooveType(
             .is_tuple = false,
         },
     });
+    const IndexTreeOptions = @Type(.{
+        .Struct = .{
+            .layout = .Auto,
+            .fields = index_options_fields,
+            .decls = &.{},
+            .is_tuple = false,
+        },
+    });
 
     // Verify no hash collisions between all the trees:
     comptime var hashes: []const u128 = &.{ObjectTree.hash};
@@ -267,19 +299,20 @@ pub fn GrooveType(
         const IndexTree = @TypeOf(@field(@as(IndexTrees, undefined), field.name));
         const hash: []const u128 = &.{IndexTree.hash};
 
-        assert(std.mem.containsAtLeast(u128, hashes, 0, hash));
+        assert(std.mem.indexOf(u128, hashes, hash) == null);
         hashes = hashes ++ hash;
     }
 
     // Verify groove index count:
     const indexes_count_actual = std.meta.fields(IndexTrees).len;
     const indexes_count_expect = std.meta.fields(Object).len -
-        options.ignored.len -
+        groove_options.ignored.len -
         // The id/timestamp field is implicitly ignored since it's the primary key for ObjectTree:
         2 +
-        std.meta.fields(@TypeOf(options.derived)).len;
+        std.meta.fields(@TypeOf(groove_options.derived)).len;
 
     assert(indexes_count_actual == indexes_count_expect);
+    assert(indexes_count_actual == std.meta.fields(IndexTreeOptions).len);
 
     // Generate a helper function for interacting with an Index field type.
     const IndexTreeFieldHelperType = struct {
@@ -298,7 +331,7 @@ pub fn GrooveType(
                 return @TypeOf(@field(@as(Object, undefined), field_name));
             }
 
-            const derived_fn = @TypeOf(@field(options.derived, field_name));
+            const derived_fn = @TypeOf(@field(groove_options.derived, field_name));
             return @typeInfo(derived_fn).Fn.return_type.?.Optional.child;
         }
 
@@ -309,7 +342,7 @@ pub fn GrooveType(
                 /// Try to extract an index from the object, deriving it when necessary.
                 pub fn derive_index(object: *const Object) ?Index {
                     if (comptime is_derived(field_name)) {
-                        return @field(options.derived, field_name)(object);
+                        return @field(groove_options.derived, field_name)(object);
                     } else {
                         return @field(object, field_name);
                     }
@@ -371,10 +404,10 @@ pub fn GrooveType(
         join_pending: usize = 0,
         join_callback: ?Callback = null,
 
-        objects_cache: *ObjectTree.ValueCache,
+        objects_cache: *ObjectTree.TableMutable.ValuesCache,
         objects: ObjectTree,
 
-        ids_cache: *IdTree.ValueCache,
+        ids_cache: *IdTree.TableMutable.ValuesCache,
         ids: IdTree,
 
         indexes: IndexTrees,
@@ -391,41 +424,36 @@ pub fn GrooveType(
         /// sufficient to query this hashmap alone to know the state of the LSM trees.
         prefetch_objects: PrefetchObjects,
 
+        /// The snapshot to prefetch from.
+        prefetch_snapshot: ?u64,
+
+        pub const Options = struct {
+            /// TODO Improve unit in this name to make more clear what should be passed.
+            /// For example, is this a size in bytes or a count in objects? It's a count in objects,
+            /// but the name poorly reflects this.
+            cache_entries_max: u32,
+            /// The maximum number of objects that might be prefetched by a batch.
+            prefetch_entries_max: u32,
+
+            tree_options_object: ObjectTree.Options,
+            tree_options_id: IdTree.Options,
+            tree_options_index: IndexTreeOptions,
+        };
+
         pub fn init(
             allocator: mem.Allocator,
             node_pool: *NodePool,
             grid: *Grid,
-            // The cache size is meant to be computed based on the left over available memory
-            // that tigerbeetle was given to allocate from CLI arguments.
-            // TODO Improve unit in this name to make more clear what should be passed.
-            // For example, is this a size in bytes or a count in objects? It's a count in objects,
-            // but the name poorly reflects this.
-            cache_size: u32,
-            // In general, the commit count max for a field, depends on the field's object,
-            // how many objects might be changed by a batch:
-            //   (config.message_size_max - sizeOf(vsr.header))
-            // For example, there are at most 8191 transfers in a batch.
-            // So commit_count_max=8191 for transfer objects and indexes.
-            //
-            // However, if a transfer is ever mutated, then this will double commit_count_max
-            // since the old index might need to be removed, and the new index inserted.
-            //
-            // A way to see this is by looking at the state machine. If a transfer is inserted,
-            // how many accounts and transfer put/removes will be generated?
-            //
-            // This also means looking at the state machine operation that will generate the
-            // most put/removes in the worst case.
-            // For example, create_accounts will put at most 8191 accounts.
-            // However, create_transfers will put 2 accounts (8191 * 2) for every transfer, and
-            // some of these accounts may exist, requiring a remove/put to update the index.
-            commit_count_max: u32,
+            options: Options,
         ) !Groove {
-            // Cache is dynamically allocated to pass a pointer into the Object tree.
-            const objects_cache = try allocator.create(ObjectTree.ValueCache);
+            // Cache is heap-allocated to pass a pointer into the Object tree.
+            const objects_cache = try allocator.create(ObjectTree.TableMutable.ValuesCache);
             errdefer allocator.destroy(objects_cache);
 
-            objects_cache.* = .{};
-            try objects_cache.ensureTotalCapacity(allocator, cache_size);
+            objects_cache.* = try ObjectTree.TableMutable.ValuesCache.init(
+                allocator,
+                options.cache_entries_max,
+            );
             errdefer objects_cache.deinit(allocator);
 
             // Intialize the object LSM tree.
@@ -434,18 +462,15 @@ pub fn GrooveType(
                 node_pool,
                 grid,
                 objects_cache,
-                .{
-                    .commit_count_max = commit_count_max,
-                },
+                options.tree_options_object,
             );
             errdefer object_tree.deinit(allocator);
 
-            // Cache is dynamically allocated to pass a pointer into the ID tree.
-            const ids_cache = try allocator.create(IdTree.ValueCache);
+            // Cache is heap-allocated to pass a pointer into the ID tree.
+            const ids_cache = try allocator.create(IdTree.TableMutable.ValuesCache);
             errdefer allocator.destroy(ids_cache);
 
-            ids_cache.* = .{};
-            try ids_cache.ensureTotalCapacity(allocator, cache_size);
+            ids_cache.* = try IdTree.TableMutable.ValuesCache.init(allocator, options.cache_entries_max);
             errdefer ids_cache.deinit(allocator);
 
             var id_tree = try IdTree.init(
@@ -453,9 +478,7 @@ pub fn GrooveType(
                 node_pool,
                 grid,
                 ids_cache,
-                .{
-                    .commit_count_max = commit_count_max,
-                },
+                options.tree_options_id,
             );
             errdefer id_tree.deinit(allocator);
 
@@ -475,23 +498,18 @@ pub fn GrooveType(
                     allocator,
                     node_pool,
                     grid,
-                    null, // No value cache for index trees.
-                    .{
-                        .commit_count_max = commit_count_max,
-                    },
+                    null, // No value cache for index trees, since they only do range queries.
+                    @field(options.tree_options_index, field.name),
                 );
                 index_trees_initialized += 1;
             }
 
-            // TODO: document why this is twice the commit count max.
-            const prefetch_count_max = commit_count_max * 2;
-
             var prefetch_ids = PrefetchIDs{};
-            try prefetch_ids.ensureTotalCapacity(allocator, prefetch_count_max);
+            try prefetch_ids.ensureTotalCapacity(allocator, options.prefetch_entries_max);
             errdefer prefetch_ids.deinit(allocator);
 
             var prefetch_objects = PrefetchObjects{};
-            try prefetch_objects.ensureTotalCapacity(allocator, prefetch_count_max);
+            try prefetch_objects.ensureTotalCapacity(allocator, options.prefetch_entries_max);
             errdefer prefetch_objects.deinit(allocator);
 
             return Groove{
@@ -505,14 +523,11 @@ pub fn GrooveType(
 
                 .prefetch_ids = prefetch_ids,
                 .prefetch_objects = prefetch_objects,
+                .prefetch_snapshot = null,
             };
         }
 
         pub fn deinit(groove: *Groove, allocator: mem.Allocator) void {
-            assert(groove.join_op == null);
-            assert(groove.join_pending == 0);
-            assert(groove.join_callback == null);
-
             inline for (std.meta.fields(IndexTrees)) |field| {
                 @field(groove.indexes, field.name).deinit(allocator);
             }
@@ -535,10 +550,26 @@ pub fn GrooveType(
             return groove.prefetch_objects.getKeyPtrAdapted(id, PrefetchObjectsAdapter{});
         }
 
-        /// Must be called directly after the state machine commit is finished and prefetch results
-        /// are no longer needed.
-        pub fn prefetch_clear(groove: *Groove) void {
-            groove.prefetch_objects.clearRetainingCapacity();
+        /// Must be called directly before the state machine begins queuing ids for prefetch.
+        /// When `snapshot` is null, prefetch from the current snapshot.
+        pub fn prefetch_setup(groove: *Groove, snapshot: ?u64) void {
+            // We may query the input tables of an ongoing compaction, but must not query the
+            // output tables until the compaction is complete. (Until then, the output tables may
+            // be in the manifest but not yet on disk).
+            const snapshot_max = groove.objects.lookup_snapshot_max;
+            assert(snapshot_max == groove.ids.lookup_snapshot_max);
+
+            const snapshot_target = snapshot orelse snapshot_max;
+            assert(snapshot_target <= snapshot_max);
+
+            if (groove.prefetch_snapshot == null) {
+                groove.prefetch_objects.clearRetainingCapacity();
+            } else {
+                // If there is a snapshot already set from the previous prefetch_setup(), then its
+                // prefetch() was never called, so there must already be no queued objects or ids.
+            }
+
+            groove.prefetch_snapshot = snapshot_target;
             assert(groove.prefetch_objects.count() == 0);
             assert(groove.prefetch_ids.count() == 0);
         }
@@ -547,14 +578,21 @@ pub fn GrooveType(
         /// We tolerate duplicate IDs enqueued by the state machine.
         /// For example, if all unique operations require the same two dependencies.
         pub fn prefetch_enqueue(groove: *Groove, id: u128) void {
-            if (groove.ids.get_cached(id)) |id_tree_value| {
-                if (!id_tree_value.tombstone()) {
-                    const object = groove.objects.get_cached(id_tree_value.timestamp).?;
-                    assert(!ObjectTreeHelpers(Object).tombstone(object));
-                    groove.prefetch_objects.putAssumeCapacity(object.*, {});
+            if (groove.ids.lookup_from_memory(groove.prefetch_snapshot.?, id)) |id_tree_value| {
+                if (id_tree_value.tombstone()) {
+                    // Do nothing; an explicit ID tombstone indicates that the object was deleted.
                 } else {
-                    // Do nothing, a prefetched ID not present in prefetch_objects indicates
-                    // that the object has either been deleted or never existed.
+                    if (groove.objects.lookup_from_memory(
+                        groove.prefetch_snapshot.?,
+                        id_tree_value.timestamp,
+                    )) |object| {
+                        assert(!ObjectTreeHelpers(Object).tombstone(object));
+                        groove.prefetch_objects.putAssumeCapacity(object.*, {});
+                    } else {
+                        // The id was in the IdTree's value cache, but not in the ObjectTree's
+                        // value cache.
+                        groove.prefetch_ids.putAssumeCapacity(id, {});
+                    }
                 }
             } else {
                 groove.prefetch_ids.putAssumeCapacity(id, {});
@@ -562,8 +600,7 @@ pub fn GrooveType(
         }
 
         /// Ensure the objects corresponding to all ids enqueued with prefetch_enqueue() are
-        /// in memory, either in the value cache of the object tree or in the prefetch_objects
-        /// backup hash map.
+        /// available in `prefetch_objects`.
         pub fn prefetch(
             groove: *Groove,
             callback: fn (*PrefetchContext) void,
@@ -572,14 +609,17 @@ pub fn GrooveType(
             context.* = .{
                 .groove = groove,
                 .callback = callback,
+                .snapshot = groove.prefetch_snapshot.?,
                 .id_iterator = groove.prefetch_ids.keyIterator(),
             };
+            groove.prefetch_snapshot = null;
             context.start_workers();
         }
 
         pub const PrefetchContext = struct {
             groove: *Groove,
             callback: fn (*PrefetchContext) void,
+            snapshot: u64,
 
             id_iterator: PrefetchIDs.KeyIterator,
 
@@ -595,17 +635,16 @@ pub fn GrooveType(
 
                 // Track an extra "worker" that will finish after the loop.
                 //
-                // This prevents `context.finish()` from being called within the loop body when every
-                // worker finishes synchronously. `context.finish()` sets the `context` to undefined,
-                // but `context` is required for the last loop condition check.
+                // This prevents `context.finish()` from being called within the loop body when
+                // every worker finishes synchronously. `context.finish()` calls the user-provided
+                // callback which may re-use the memory of this `PrefetchContext`. However, we
+                // rely on `context` being well-defined for the loop condition.
                 context.workers_busy += 1;
 
-                // -1 to ignore the extra worker.
-                while (context.workers_busy - 1 < context.workers.len) {
-                    const worker = &context.workers[context.workers_busy - 1];
+                for (context.workers) |*worker| {
                     worker.* = .{ .context = context };
                     context.workers_busy += 1;
-                    if (!worker.lookup_start()) break;
+                    worker.lookup_start_next();
                 }
 
                 assert(context.workers_busy >= 1);
@@ -619,12 +658,12 @@ pub fn GrooveType(
 
             fn finish(context: *PrefetchContext) void {
                 assert(context.workers_busy == 0);
-                assert(context.groove.prefetch_ids.count() == 0);
+
                 assert(context.id_iterator.next() == null);
+                context.groove.prefetch_ids.clearRetainingCapacity();
+                assert(context.groove.prefetch_ids.count() == 0);
 
-                const callback = context.callback;
-                context.* = undefined;
-                callback(context);
+                context.callback(context);
             }
         };
 
@@ -636,34 +675,37 @@ pub fn GrooveType(
             lookup_id: IdTree.LookupContext = undefined,
             lookup_object: ObjectTree.LookupContext = undefined,
 
-            /// Returns true if asynchronous I/O has been started.
-            /// Returns false if there are no more IDs to prefetch.
-            fn lookup_start(worker: *PrefetchWorker) bool {
-                const groove = worker.context.groove;
-
+            fn lookup_start_next(worker: *PrefetchWorker) void {
                 const id = worker.context.id_iterator.next() orelse {
-                    groove.prefetch_ids.clearRetainingCapacity();
-                    assert(groove.prefetch_ids.count() == 0);
                     worker.context.worker_finished();
-                    return false;
+                    return;
                 };
 
-                if (config.verify) {
-                    // This is checked in prefetch_enqueue()
-                    assert(groove.ids.get_cached(id.*) == null);
-                }
-
-                // If not in the LSM tree's cache, the object must be read from disk and added
-                // to the auxiliary prefetch_objects hash map.
-                // TODO: this LSM tree function needlessly checks the LSM tree's cache a
-                // second time. Adding API to the LSM tree to avoid this may be worthwhile.
-                groove.ids.lookup(
-                    lookup_id_callback,
-                    &worker.lookup_id,
-                    snapshot_latest,
+                if (worker.context.groove.ids.lookup_from_memory(
+                    worker.context.snapshot,
                     id.*,
-                );
-                return true;
+                )) |id_tree_value| {
+                    assert(!id_tree_value.tombstone());
+                    lookup_id_callback(&worker.lookup_id, id_tree_value);
+
+                    if (config.verify) {
+                        // If the id is cached, then we must be prefetching it because the object
+                        // was not also cached.
+                        assert(worker.context.groove.objects.lookup_from_memory(
+                            worker.context.snapshot,
+                            id_tree_value.timestamp,
+                        ) == null);
+                    }
+                } else {
+                    // If not in the LSM tree's cache, the object must be read from disk and added
+                    // to the auxiliary prefetch_objects hash map.
+                    worker.context.groove.ids.lookup_from_levels(
+                        lookup_id_callback,
+                        &worker.lookup_id,
+                        worker.context.snapshot,
+                        id.*,
+                    );
+                }
             }
 
             fn lookup_id_callback(
@@ -673,18 +715,45 @@ pub fn GrooveType(
                 const worker = @fieldParentPtr(PrefetchWorker, "lookup_id", completion);
 
                 if (result) |id_tree_value| {
-                    if (!id_tree_value.tombstone()) {
-                        worker.context.groove.objects.lookup(
-                            lookup_object_callback,
-                            &worker.lookup_object,
-                            snapshot_latest,
-                            id_tree_value.timestamp,
+                    if (config.verify) {
+                        // This was checked in prefetch_enqueue().
+                        assert(
+                            worker.context.groove.ids.lookup_from_memory(
+                                worker.context.snapshot,
+                                worker.lookup_id.key,
+                            ) == null or
+                                worker.context.groove.objects.lookup_from_memory(
+                                worker.context.snapshot,
+                                id_tree_value.timestamp,
+                            ) == null,
                         );
-                    } else {
-                        worker.lookup_finish();
                     }
+
+                    if (id_tree_value.tombstone()) {
+                        worker.lookup_start_next();
+                        return;
+                    }
+
+                    if (worker.context.groove.objects.lookup_from_memory(
+                        worker.context.snapshot,
+                        id_tree_value.timestamp,
+                    )) |object| {
+                        // The object is not a tombstone; the ID and Object trees are in sync.
+                        assert(!ObjectTreeHelpers(Object).tombstone(object));
+
+                        worker.context.groove.prefetch_objects.putAssumeCapacityNoClobber(object.*, {});
+                        worker.lookup_start_next();
+                        return;
+                    }
+
+                    worker.context.groove.objects.lookup_from_levels(
+                        lookup_object_callback,
+                        &worker.lookup_object,
+                        worker.context.snapshot,
+                        id_tree_value.timestamp,
+                    );
                 } else {
-                    worker.lookup_finish();
+                    worker.lookup_start_next();
                 }
             }
 
@@ -699,13 +768,7 @@ pub fn GrooveType(
                 assert(!ObjectTreeHelpers(Object).tombstone(object));
 
                 worker.context.groove.prefetch_objects.putAssumeCapacityNoClobber(object.*, {});
-                worker.lookup_finish();
-            }
-
-            fn lookup_finish(worker: *PrefetchWorker) void {
-                if (!worker.lookup_start()) {
-                    worker.* = undefined;
-                }
+                worker.lookup_start_next();
             }
         };
 
@@ -794,7 +857,7 @@ pub fn GrooveType(
             assert(groove.prefetch_objects.removeAdapted(object.id, PrefetchObjectsAdapter{}));
         }
 
-        /// Maximum number of pending sync callbacks (ObjecTree + IdTree + IndexTrees).
+        /// Maximum number of pending sync callbacks (ObjectTree + IdTree + IndexTrees).
         const join_pending_max = 2 + std.meta.fields(IndexTrees).len;
 
         fn JoinType(comptime join_op: JoinOp) type {
@@ -932,7 +995,7 @@ test "Groove" {
 
     _ = Groove.prefetch_enqueue;
     _ = Groove.prefetch;
-    _ = Groove.prefetch_clear;
+    _ = Groove.prefetch_setup;
 
     std.testing.refAllDecls(Groove.PrefetchWorker);
     std.testing.refAllDecls(Groove.PrefetchContext);