tigerbeetle-node 0.10.0 → 0.11.1

package/src/tigerbeetle/src/lsm/compaction.zig

@@ -1,8 +1,41 @@
+//! Compaction moves or merges a table's values into the next level.
+//!
+//! Each Compaction is paced to run in one half-bar.
+//!
+//!
+//! Compaction overview:
+//!
+//! 1. Given:
+//!
+//!   - levels A and B, where A+1=B
+//!   - a single table in level A ("table A")
+//!   - all tables from level B which intersect table A's key range ("tables B")
+//!     (This can include anything between 0 tables and all of level B's tables.)
+//!
+//! 2. If table A's key range is disjoint from the keys in level B, move table A into level B.
+//!    All done! (But if the key ranges intersect, jump to step 3).
+//!
+//! 3. Create an iterator from the sort-merge of table A and the concatenation of tables B.
+//!    If the same key exists in level A and B, take A's and discard B's. †
+//!
+//! 4. Write the sort-merge iterator into a sequence of new tables on disk.
+//!
+//! 5. Update the input tables in the Manifest with their new `snapshot_max` so that they become
+//!    invisible to subsequent read transactions.
+//!
+//! 6. Insert the new level-B tables into the Manifest.
+//!
+//! † When A's value is a tombstone, there is a special case for garbage collection. When either:
+//! * level B is the final level, or
+//! * A's key does not exist in B or any deeper level,
+//! then the tombstone is omitted from the compacted output (see: `compaction_must_drop_tombstones`).
+//!
 const std = @import("std");
 const mem = std.mem;
 const math = std.math;
 const assert = std.debug.assert;
 
+const log = std.log.scoped(.compaction);
 const config = @import("../config.zig");
 
 const GridType = @import("grid.zig").GridType;
@@ -14,84 +47,113 @@ const LevelIteratorType = @import("level_iterator.zig").LevelIteratorType;
 pub fn CompactionType(
     comptime Table: type,
     comptime Storage: type,
-    comptime IteratorAType: anytype, // fn (Table: type, Storage: type) type
+    comptime IteratorAType: anytype,
 ) type {
     const Key = Table.Key;
     const Value = Table.Value;
     const tombstone = Table.tombstone;
-    const compare_keys = Table.compare_keys;
 
     return struct {
         const Compaction = @This();
 
         const Grid = GridType(Storage);
         const BlockPtr = Grid.BlockPtr;
+        const BlockPtrConst = Grid.BlockPtrConst;
+        const BlockWrite = struct {
+            write: Grid.Write = undefined,
+            block: BlockPtr = undefined,
+            writable: bool = false,
+        };
+
         const Manifest = ManifestType(Table, Storage);
         const TableInfo = Manifest.TableInfo;
 
         const IteratorA = IteratorAType(Table, Storage);
         const IteratorB = LevelIteratorType(Table, Storage);
 
-        pub const Callback = fn (it: *Compaction) void;
-
         const k = 2;
         const MergeIterator = KWayMergeIterator(
             Compaction,
-            Key,
-            Value,
+            Table.Key,
+            Table.Value,
             Table.key_from_value,
             Table.compare_keys,
             k,
-            stream_peek,
-            stream_pop,
-            stream_precedence,
+            MergeStreamSelector.peek,
+            MergeStreamSelector.pop,
+            MergeStreamSelector.precedence,
         );
 
-        const Status = enum {
-            idle,
-            compacting,
-            done,
-        };
+        const MergeStreamSelector = struct {
+            fn peek(compaction: *const Compaction, stream_id: u32) error{ Empty, Drained }!Key {
+                return switch (stream_id) {
+                    0 => compaction.iterator_a.peek(),
+                    1 => compaction.iterator_b.peek(),
+                    else => unreachable,
+                };
+            }
 
-        const BlockPtrConst = *align(config.sector_size) const [config.block_size]u8;
-        const BlockWrite = struct {
-            block: BlockPtr,
-            write: Grid.Write = undefined,
-            ready: bool = false,
+            fn pop(compaction: *Compaction, stream_id: u32) Value {
+                return switch (stream_id) {
+                    0 => compaction.iterator_a.pop(),
+                    1 => compaction.iterator_b.pop(),
+                    else => unreachable,
+                };
+            }
+
+            /// Returns true if stream A has higher precedence than stream B.
+            /// This is used to deduplicate values across streams.
+            fn precedence(compaction: *const Compaction, stream_a: u32, stream_b: u32) bool {
+                _ = compaction;
+                assert(stream_a + stream_b == 1);
+
+                // All tables in iterator_a (stream=0) have a higher precedence.
+                return stream_a == 0;
+            }
         };
 
-        const TableInfoBuffer = @import("manifest.zig").TableInfoBufferType(Table, .ascending);
+        pub const Callback = fn (it: *Compaction) void;
 
-        status: Status,
+        const Status = enum {
+            idle,
+            processing,
+            done,
+        };
 
         grid: *Grid,
-        manifest: *Manifest,
-        level_b: u8,
+        grid_reservation: Grid.Reservation,
         range: Manifest.CompactionRange,
-        snapshot: u64,
+
+        /// `op_min` is the first op/beat of this compaction's half-bar.
+        /// `op_min` is used as a snapshot — the compaction's input tables must be visible
+        /// to `op_min`.
+        ///
+        /// After this compaction finishes:
+        /// - `op_min + half_bar_beat_count - 1` will be the input tables' snapshot_max.
+        /// - `op_min + half_bar_beat_count` will be the output tables' snapshot_min.
+        op_min: u64,
         drop_tombstones: bool,
 
+        status: Status,
         callback: ?Callback = null,
-        ticks: u32 = 0,
         io_pending: u32 = 0,
 
         iterator_a: IteratorA,
         iterator_b: IteratorB,
 
-        /// Private:
-        /// The caller must use the Callback's `done` argument to know when compaction is done,
-        /// because a write I/O may yet follow even after the merge is done.
-        merge_done: bool = false,
-        merge_iterator: MergeIterator,
-        table_builder: Table.Builder,
+        merge_done: bool,
+        merge_iterator: ?MergeIterator,
 
+        table_builder: Table.Builder,
         index: BlockWrite,
         filter: BlockWrite,
         data: BlockWrite,
 
-        remove_level_a: ?*const TableInfo = null,
-        update_level_b: TableInfoBuffer,
-        insert_level_b: TableInfoBuffer,
+        manifest: *Manifest,
+        level_b: u8,
+        level_a_input: ?TableInfo,
+
+        tables_output_count: usize = 0,
 
         pub fn init(allocator: mem.Allocator) !Compaction {
             var iterator_a = try IteratorA.init(allocator);
@@ -103,162 +165,137 @@ pub fn CompactionType(
             var table_builder = try Table.Builder.init(allocator);
             errdefer table_builder.deinit(allocator);
 
-            const index = BlockWrite{ .block = try allocate_block(allocator) };
-            errdefer allocator.free(index.block);
-
-            const filter = BlockWrite{ .block = try allocate_block(allocator) };
-            errdefer allocator.free(filter.block);
-
-            const data = BlockWrite{ .block = try allocate_block(allocator) };
-            errdefer allocator.free(data.block);
-
-            // The average number of tables involved in a compaction is the 1 table from level A,
-            // plus the growth_factor number of tables from level B, plus 1 on either side,
-            // since the overlap may not be perfectly aligned to table boundaries.
-            // However, the worst case number of tables may approach all tables in level B,
-            // since key ranges may be skewed and not evenly distributed across a level.
-            const table_buffer_count_max = 1 + config.lsm_growth_factor + 2;
-
-            var update_level_b = try TableInfoBuffer.init(allocator, table_buffer_count_max);
-            errdefer update_level_b.deinit(allocator);
-
-            var insert_level_b = try TableInfoBuffer.init(allocator, table_buffer_count_max);
-            errdefer insert_level_b.deinit(allocator);
-
             return Compaction{
-                .status = .idle,
-
-                // Assigned by start():
+                // Assigned by start()
                 .grid = undefined,
-                .manifest = undefined,
-                .level_b = undefined,
+                .grid_reservation = undefined,
                 .range = undefined,
-                .snapshot = undefined,
+                .op_min = undefined,
                 .drop_tombstones = undefined,
 
+                .status = .idle,
                 .iterator_a = iterator_a,
                 .iterator_b = iterator_b,
 
-                .merge_iterator = undefined, // This can only be initialized on tick 1.
-                .table_builder = table_builder,
+                .merge_done = false,
+                .merge_iterator = null,
 
-                .index = index,
-                .filter = filter,
-                .data = data,
+                .table_builder = table_builder,
+                .index = .{},
+                .filter = .{},
+                .data = .{},
 
-                .update_level_b = update_level_b,
-                .insert_level_b = insert_level_b,
+                // Assigned by start()
+                .manifest = undefined,
+                .level_b = undefined,
+                .level_a_input = null,
             };
         }
 
-        fn allocate_block(allocator: mem.Allocator) !BlockPtr {
-            const block = try allocator.alignedAlloc(u8, config.sector_size, config.block_size);
-            return block[0..config.block_size];
-        }
-
         pub fn deinit(compaction: *Compaction, allocator: mem.Allocator) void {
-            compaction.iterator_a.deinit(allocator);
-            compaction.iterator_b.deinit(allocator);
             compaction.table_builder.deinit(allocator);
-            compaction.update_level_b.deinit(allocator);
-            compaction.insert_level_b.deinit(allocator);
 
-            allocator.free(compaction.index.block);
-            allocator.free(compaction.filter.block);
-            allocator.free(compaction.data.block);
+            compaction.iterator_b.deinit(allocator);
+            compaction.iterator_a.deinit(allocator);
         }
 
+        /// The compaction's input tables are:
+        /// * table_a (which is null when level B is 0), and
+        /// * any level-B tables visible to `op_min` within `range`.
         pub fn start(
             compaction: *Compaction,
             grid: *Grid,
             manifest: *Manifest,
-            // TODO level_a_table: ?TableInfo,
-            level_b: u8,
+            op_min: u64,
             range: Manifest.CompactionRange,
-            snapshot: u64,
+            table_a: ?*const TableInfo,
+            level_b: u8,
             iterator_a_context: IteratorA.Context,
         ) void {
             assert(compaction.status == .idle);
             assert(compaction.callback == null);
             assert(compaction.io_pending == 0);
-            assert(level_b < config.lsm_levels);
+            assert(!compaction.merge_done and compaction.merge_iterator == null);
+
+            assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
             assert(range.table_count > 0);
+            if (table_a) |t| assert(t.visible(op_min));
+
+            assert(level_b < config.lsm_levels);
+            assert((level_b == 0) == (table_a == null));
 
+            // Levels may choose to drop tombstones if keys aren't included in the lower levels.
+            // This invariant is always true for the last level as it doesn't have any lower ones.
             const drop_tombstones = manifest.compaction_must_drop_tombstones(level_b, range);
             assert(drop_tombstones or level_b < config.lsm_levels - 1);
 
             compaction.* = .{
-                .status = .compacting,
-
                 .grid = grid,
-                .manifest = manifest,
-                .level_b = level_b,
+                // Reserve enough blocks to write our output tables in the worst case, where:
+                // - no tombstones are dropped,
+                // - no values are overwritten,
+                // - and all tables are full.
+                //
+                // We must reserve before doing any async work so that the block acquisition order
+                // is deterministic (relative to other concurrent compactions).
+                // TODO The replica must stop accepting requests if it runs out of blocks/capacity,
+                // rather than panicking here.
+                // TODO(Compaction Pacing): Reserve smaller increments, at the start of each beat.
+                // (And likewise release the reservation at the end of each beat, instead of at the
+                // end of each half-bar).
+                // TODO(Move Table) Don't reserve these when we just move the table to the next level.
+                .grid_reservation = grid.reserve(range.table_count * Table.block_count_max).?,
                 .range = range,
-                .snapshot = snapshot,
+                .op_min = op_min,
                 .drop_tombstones = drop_tombstones,
 
+                .status = .processing,
                 .iterator_a = compaction.iterator_a,
                 .iterator_b = compaction.iterator_b,
 
-                .merge_iterator = undefined,
-                .table_builder = compaction.table_builder,
+                .merge_done = false,
+                .merge_iterator = null,
 
+                .table_builder = compaction.table_builder,
                 .index = compaction.index,
                 .filter = compaction.filter,
                 .data = compaction.data,
 
-                .update_level_b = compaction.update_level_b,
-                .insert_level_b = compaction.insert_level_b,
+                .manifest = manifest,
+                .level_b = level_b,
+                .level_a_input = if (table_a) |table| table.* else null,
             };
 
-            assert(!compaction.data.ready);
-            assert(!compaction.filter.ready);
-            assert(!compaction.index.ready);
-
-            // TODO Reset builder.
-
-            // TODO: Enable when move_table() can fetch TableInfo from address/checksum.
-            //
-            // Perform a "compaction move" to the next level inline if certain factors allow:
-            // - Can only do the specialization if there's a single table to compact.
-            // - Must be compacting from a table iterator which has an address and checksum.
-            // - Cannot drop tombstones as then we have to go through the normal compaction path.
-            // - Cannot be performing the immutable table -> level 0 compaction
-            //   as it requires the table being moved to reside on disk (tracked by manifest).
-            if (false and IteratorA.Context == TableIteratorType(Table, Storage)) {
-                if (!drop_tombstones and range.table_count == 1) {
-                    assert(compaction.level_b != 0);
-                    assert(compaction.status == .compacting);
-
-                    const level_a = level_b - 1;
-                    assert(level_a < config.lsm_levels - 1);
-
-                    compaction.manifest.move_table(
-                        level_a,
-                        level_b,
-                        snapshot,
-                        iterator_a_context.address,
-                        iterator_a_context.checksum,
-                    );
-
-                    compaction.status = .done;
-                    return;
-                }
-            }
+            assert(!compaction.index.writable);
+            assert(!compaction.filter.writable);
+            assert(!compaction.data.writable);
+
+            // TODO Implement manifest.move_table() optimization if there's only range.table_count == 1.
+            // This would do update_tables + insert_tables inline without going through the iterators.
 
             const iterator_b_context = .{
                 .grid = grid,
                 .manifest = manifest,
                 .level = level_b,
-                .snapshot = snapshot,
+                .snapshot = op_min,
                 .key_min = range.key_min,
                 .key_max = range.key_max,
                 .direction = .ascending,
-                .table_info_callback = iterator_b_table_info_callback, // TODO
+                .table_info_callback = iterator_b_table_info_callback,
             };
 
-            compaction.iterator_a.start(iterator_a_context, iterator_a_callback);
-            compaction.iterator_b.start(iterator_b_context, iterator_b_callback);
+            compaction.iterator_a.start(iterator_a_context, iterator_a_io_callback);
+            compaction.iterator_b.start(iterator_b_context, iterator_b_io_callback);
+        }
+
+        fn iterator_a_io_callback(iterator_a: *IteratorA) void {
+            const compaction = @fieldParentPtr(Compaction, "iterator_a", iterator_a);
+            compaction.io_finish();
+        }
+
+        fn iterator_b_io_callback(iterator_b: *IteratorB) void {
+            const compaction = @fieldParentPtr(Compaction, "iterator_b", iterator_b);
+            compaction.io_finish();
         }
 
         fn iterator_b_table_info_callback(
@@ -267,337 +304,279 @@ pub fn CompactionType(
             index_block: BlockPtrConst,
         ) void {
             const compaction = @fieldParentPtr(Compaction, "iterator_b", iterator_b);
-            compaction.queue_manifest_update(&compaction.update_level_b, table);
-
-            // Release the table's block addresses if it's invisible to the compaction.
-            if (table.invisible(&.{compaction.snapshot})) {
-                compaction.grid.release(Table.index_block_address(index_block));
-
-                for (Table.index_filter_addresses_used(index_block)) |address| {
-                    compaction.grid.release(address);
-                }
-
-                for (Table.index_data_addresses_used(index_block)) |address| {
-                    compaction.grid.release(address);
-                }
-            }
-        }
-
-        fn queue_manifest_update(
-            compaction: *Compaction,
-            buffer: *TableInfoBuffer,
-            table: *const TableInfo,
-        ) void {
-            assert(buffer == &compaction.update_level_b or buffer == &compaction.insert_level_b);
-            if (buffer.full()) compaction.update_manifest(buffer);
-            buffer.push(table);
-        }
-
-        fn update_manifest(compaction: *Compaction, buffer: *TableInfoBuffer) void {
-            assert(buffer == &compaction.update_level_b or buffer == &compaction.insert_level_b);
-
-            const tables: []const TableInfo = buffer.drain();
-            if (tables.len == 0) return;
-
-            for (tables) |table| {
-                assert(compare_keys(table.key_min, compaction.range.key_min) != .lt);
-                assert(compare_keys(table.key_max, compaction.range.key_max) != .gt);
+            assert(compaction.status == .processing);
+            assert(compaction.callback != null);
+            assert(!compaction.merge_done);
+            assert(table.visible(compaction.op_min));
+
+            // Tables discovered by iterator_b that are visible at the start of compaction.
+            var table_copy = table.*;
+            compaction.manifest.update_table(
+                compaction.level_b,
+                snapshot_max_for_table_input(compaction.op_min),
+                &table_copy,
+            );
+
+            // Release the table's block addresses in the Grid as it will be made invisible.
+            // This is safe; iterator_b makes a copy of the block before calling us.
+            const grid = compaction.grid;
+            for (Table.index_data_addresses_used(index_block)) |address| {
+                grid.release(address);
             }
-
-            if (buffer == &compaction.update_level_b) {
-                compaction.manifest.update_tables(compaction.level_b, compaction.snapshot, tables);
-            } else {
-                compaction.manifest.insert_tables(compaction.level_b, tables);
+            for (Table.index_filter_addresses_used(index_block)) |address| {
+                grid.release(address);
             }
+            grid.release(Table.index_block_address(index_block));
         }
 
-        /// Compaction.ticks stages at which each action may be performed.
-        const pipeline_tick_read = 0;
-        const pipeline_tick_merge = 1;
-        const pipeline_tick_write = 2;
-
-        /// Submits all read/write I/O before starting the CPU-intensive k-way merge.
-        /// This allows the I/O to happen in parallel with the merge.
-        ///
-        /// The caller must call:
-        ///
-        /// 1. tick_io() across all trees,
-        /// 2. IO.tick() to submit these I/O operations to the kernel,
-        /// 3. tick_cpu() across all trees.
-        pub fn tick_io(compaction: *Compaction, callback: Callback) void {
-            assert(compaction.status == .compacting);
+        pub fn compact_tick(compaction: *Compaction, callback: Callback) void {
+            assert(compaction.status == .processing);
             assert(compaction.callback == null);
             assert(compaction.io_pending == 0);
             assert(!compaction.merge_done);
 
             compaction.callback = callback;
 
-            if (compaction.ticks >= pipeline_tick_read) compaction.tick_io_read();
-            if (compaction.ticks >= pipeline_tick_write) compaction.tick_io_write();
-
-            // All values may be eclipsed by tombstones, with no write I/O pending here.
+            // Generate fake IO to make sure io_pending doesn't reach zero multiple times from
+            // IO being completed inline down below.
+            // The fake IO is immediately resolved and triggers the cpu_merge_start if all
+            // IO completes inline or if no IO was started.
+            compaction.io_start();
+            defer compaction.io_finish();
+
+            // Start reading blocks from the iterators to merge them.
+            if (compaction.iterator_a.tick()) compaction.io_start();
+            if (compaction.iterator_b.tick()) compaction.io_start();
+
+            // Start writing blocks prepared by the merge iterator from a previous compact_tick().
+            compaction.io_write_start(.data);
+            compaction.io_write_start(.filter);
+            compaction.io_write_start(.index);
         }
 
-        pub fn tick_cpu(compaction: *Compaction) void {
-            assert(compaction.status == .compacting);
-            assert(compaction.callback != null);
-            assert(compaction.io_pending >= 0);
-            assert(!compaction.merge_done);
+        const BlockWriteField = enum { data, filter, index };
 
-            if (compaction.ticks == pipeline_tick_merge) {
-                // We cannot initialize the merge until we can peek() a value from each stream,
-                // which depends on tick 0 (to read blocks) having happened.
-                compaction.merge_iterator = MergeIterator.init(compaction, k, .ascending);
-            }
-
-            if (compaction.ticks >= pipeline_tick_merge) {
-                if (compaction.merge_iterator.empty()) {
-                    assert(!compaction.merge_done);
+        fn io_write_start(compaction: *Compaction, comptime field: BlockWriteField) void {
+            const write_callback = struct {
+                fn callback(write: *Grid.Write) void {
+                    const block_write = @fieldParentPtr(BlockWrite, "write", write);
+                    block_write.block = undefined;
 
-                    // We must distinguish between merge_iterator.empty() and merge_done.
-                    // The former cannot be accessed before MergeIterator.init() on tick 1.
-                    compaction.merge_done = true;
-                } else {
-                    compaction.tick_cpu_merge();
+                    const _compaction = @fieldParentPtr(Compaction, @tagName(field), block_write);
+                    _compaction.io_finish();
                 }
-            }
+            }.callback;
 
-            compaction.ticks += 1;
+            const block_write: *BlockWrite = &@field(compaction, @tagName(field));
+            if (block_write.writable) {
+                block_write.writable = false;
 
-            // Normally, a tick completes only after a read/write I/O.
-            // However, the compaction may drop only tombstones, resulting in no write I/O.
-            if (compaction.io_pending == 0) compaction.tick_done();
+                compaction.io_start();
+                compaction.grid.write_block(
+                    write_callback,
+                    &block_write.write,
+                    block_write.block,
+                    Table.block_address(block_write.block),
+                );
+            }
         }
 
-        fn tick_done(compaction: *Compaction) void {
-            assert(compaction.status == .compacting);
+        fn io_start(compaction: *Compaction) void {
+            assert(compaction.status == .processing);
             assert(compaction.callback != null);
-            assert(compaction.io_pending == 0);
-
-            // Consume the callback and invoke it one finished updating state below.
-            const callback = compaction.callback.?;
-            compaction.callback = null;
-            defer callback(compaction);
-
-            // Once merge completes, the compaction is now officially over.
-            if (compaction.merge_done) {
-                compaction.status = .done;
+            assert(!compaction.merge_done);
 
-                // Flush updates to the table infos discovered during compaction
-                // TODO Handle compaction.remove_level_a
-                compaction.update_manifest(&compaction.update_level_b);
-                compaction.update_manifest(&compaction.insert_level_b);
-            }
+            compaction.io_pending += 1;
         }
 
-        pub fn reset(compaction: *Compaction) void {
-            assert(compaction.callback == null);
-            assert(compaction.io_pending == 0);
-
-            assert(compaction.status == .done);
-            compaction.status = .idle;
+        fn io_finish(compaction: *Compaction) void {
+            assert(compaction.status == .processing);
+            assert(compaction.callback != null);
+            assert(compaction.io_pending > 0);
+            assert(!compaction.merge_done);
 
-            assert(compaction.update_level_b.drain().len == 0);
-            assert(compaction.insert_level_b.drain().len == 0);
+            compaction.io_pending -= 1;
+            if (compaction.io_pending == 0) compaction.cpu_merge_start();
         }
 
-        fn tick_io_read(compaction: *Compaction) void {
+        fn cpu_merge_start(compaction: *Compaction) void {
+            assert(compaction.status == .processing);
             assert(compaction.callback != null);
+            assert(compaction.io_pending == 0);
+            assert(!compaction.merge_done);
 
-            if (compaction.iterator_a.tick()) compaction.io_pending += 1;
-            if (compaction.iterator_b.tick()) compaction.io_pending += 1;
+            // Create the merge iterator only when we can peek() from the read iterators.
+            // This happens after IO for the first reads complete.
+            if (compaction.merge_iterator == null) {
+                compaction.merge_iterator = MergeIterator.init(compaction, k, .ascending);
+                assert(!compaction.merge_iterator.?.empty());
+            }
 
-            if (compaction.merge_done) assert(compaction.io_pending == 0);
-        }
+            assert(!compaction.data.writable);
+            assert(!compaction.filter.writable);
+            assert(!compaction.index.writable);
 
-        fn tick_io_write(compaction: *Compaction) void {
-            assert(compaction.callback != null);
-            assert(compaction.ticks >= pipeline_tick_write);
-            // There may be no data block to write if all values are eclipsed by tombstones.
-            assert(compaction.data.ready or !compaction.data.ready);
+            if (!compaction.merge_iterator.?.empty()) {
+                compaction.cpu_merge();
+            } else {
+                compaction.cpu_merge_finish();
+            }
 
-            compaction.write_block_if_ready(&compaction.data, write_block_callback("data"));
-            compaction.write_block_if_ready(&compaction.filter, write_block_callback("filter"));
-            compaction.write_block_if_ready(&compaction.index, write_block_callback("index"));
+            // TODO Implement pacing here by deciding if we should do another compact_tick()
+            // instead of invoking the callback, using compaction.range.table_count as the heuristic.
 
-            assert(!compaction.data.ready);
-            assert(!compaction.filter.ready);
-            assert(!compaction.index.ready);
+            const callback = compaction.callback.?;
+            compaction.callback = null;
+            callback(compaction);
         }
 
-        fn tick_cpu_merge(compaction: *Compaction) void {
+        fn cpu_merge(compaction: *Compaction) void {
+            // Ensure this is the result of a compact_tick() call that finished processing IO.
+            assert(compaction.status == .processing);
             assert(compaction.callback != null);
-            assert(compaction.ticks >= pipeline_tick_merge);
+            assert(compaction.io_pending == 0);
             assert(!compaction.merge_done);
-            assert(!compaction.merge_iterator.empty());
 
-            assert(!compaction.data.ready);
-            assert(!compaction.filter.ready);
-            assert(!compaction.index.ready);
+            // Ensure there are values to merge and that is it safe to do so.
+            const merge_iterator = &compaction.merge_iterator.?;
+            assert(!merge_iterator.empty());
+            assert(!compaction.data.writable);
+            assert(!compaction.filter.writable);
+            assert(!compaction.index.writable);
 
-            var tombstones_dropped: u32 = 0;
+            // Build up a data block with values merged from the read iterators.
+            // This skips tombstone values if compaction was started with the intent to drop them.
             while (!compaction.table_builder.data_block_full()) {
-                const value = compaction.merge_iterator.pop() orelse {
-                    compaction.assert_read_iterators_empty();
-                    break;
-                };
-                if (compaction.drop_tombstones and tombstone(&value)) {
-                    tombstones_dropped += 1;
-                } else {
-                    compaction.table_builder.data_block_append(&value);
-                }
+                const value = merge_iterator.pop() orelse break;
+                if (compaction.drop_tombstones and tombstone(&value)) continue;
+                compaction.table_builder.data_block_append(&value);
             }
 
-            if (compaction.table_builder.data_block_empty()) {
-                assert(compaction.drop_tombstones);
-                assert(tombstones_dropped > 0);
-            } else {
+            // Finalize the data block if it's full or if it contains pending values when there's
+            // no more left to merge.
+            if (compaction.table_builder.data_block_full() or
+                (merge_iterator.empty() and !compaction.table_builder.data_block_empty()))
+            {
                 compaction.table_builder.data_block_finish(.{
                     .cluster = compaction.grid.superblock.working.cluster,
-                    .address = compaction.grid.acquire(),
+                    .address = compaction.grid.acquire(compaction.grid_reservation),
                 });
-                swap_buffers(&compaction.data, &compaction.table_builder.data_block);
-                assert(compaction.data.ready);
 
-                if (!compaction.merge_iterator.empty()) {
-                    // Ensure that the block was filled completely.
-                    const values_used = Table.data_block_values_used(compaction.data.block).len;
-                    assert(values_used == Table.data.value_count_max);
-                }
+                // Mark the finished data block as writable for the next compact_tick() call.
+                compaction.data.block = compaction.table_builder.data_block;
+                assert(!compaction.data.writable);
+                compaction.data.writable = true;
             }
 
+            // Finalize the filter block if it's full or if it contains pending data blocks
+            // when there's no more merged values to fill them.
             if (compaction.table_builder.filter_block_full() or
-                compaction.table_builder.index_block_full() or
-                compaction.merge_iterator.empty())
+                (merge_iterator.empty() and !compaction.table_builder.filter_block_empty()))
             {
-                if (compaction.table_builder.filter_block_empty()) {
-                    assert(compaction.drop_tombstones);
-                    assert(tombstones_dropped > 0);
-                } else {
-                    compaction.table_builder.filter_block_finish(.{
-                        .cluster = compaction.grid.superblock.working.cluster,
-                        .address = compaction.grid.acquire(),
-                    });
-                    swap_buffers(&compaction.filter, &compaction.table_builder.filter_block);
-                    assert(compaction.filter.ready);
-                }
+                compaction.table_builder.filter_block_finish(.{
+                    .cluster = compaction.grid.superblock.working.cluster,
+                    .address = compaction.grid.acquire(compaction.grid_reservation),
+                });
+
+                // Mark the finished filter block as writable for the next compact_tick() call.
+                compaction.filter.block = compaction.table_builder.filter_block;
+                assert(!compaction.filter.writable);
+                compaction.filter.writable = true;
             }
 
+            // Finalize the index block if it's full or if it contains pending data blocks
+            // when there's no more merged values to fill them.
             if (compaction.table_builder.index_block_full() or
-                compaction.merge_iterator.empty())
+                (merge_iterator.empty() and !compaction.table_builder.index_block_empty()))
             {
-                if (compaction.table_builder.index_block_empty()) {
-                    assert(compaction.drop_tombstones);
-                    assert(tombstones_dropped > 0);
-                } else {
-                    const snapshot_min = compaction.snapshot;
-                    const table = compaction.table_builder.index_block_finish(.{
-                        .cluster = compaction.grid.superblock.working.cluster,
-                        .address = compaction.grid.acquire(),
-                        .snapshot_min = snapshot_min,
-                    });
-                    compaction.queue_manifest_update(&compaction.insert_level_b, &table);
-
-                    swap_buffers(&compaction.index, &compaction.table_builder.index_block);
-                    assert(compaction.index.ready);
-                }
-            }
-        }
-
-        fn iterator_a_callback(iterator_a: *IteratorA) void {
-            const compaction = @fieldParentPtr(Compaction, "iterator_a", iterator_a);
-            compaction.io_callback();
-        }
-
-        fn iterator_b_callback(iterator_b: *IteratorB) void {
-            const compaction = @fieldParentPtr(Compaction, "iterator_b", iterator_b);
-            compaction.io_callback();
-        }
-
-        fn io_callback(compaction: *Compaction) void {
-            compaction.io_pending -= 1;
-            if (compaction.io_pending == 0) compaction.tick_done();
-        }
+                const table = compaction.table_builder.index_block_finish(.{
+                    .cluster = compaction.grid.superblock.working.cluster,
+                    .address = compaction.grid.acquire(compaction.grid_reservation),
+                    .snapshot_min = snapshot_min_for_table_output(compaction.op_min),
+                    // TODO(Persistent Snapshots) set snapshot_max to the minimum snapshot_max of
+                    // all the (original) input tables.
+                });
+                compaction.manifest.insert_table(compaction.level_b, &table);
 
-        fn write_block_if_ready(
-            compaction: *Compaction,
-            block_write: *BlockWrite,
-            callback: fn (*Grid.Write) void,
-        ) void {
-            if (block_write.ready) {
-                block_write.ready = false;
+                // Mark the finished index block as writable for the next compact_tick() call.
+                compaction.index.block = compaction.table_builder.index_block;
+                assert(!compaction.index.writable);
+                compaction.index.writable = true;
 
-                compaction.io_pending += 1;
-                compaction.grid.write_block(
-                    callback,
-                    &block_write.write,
-                    block_write.block,
-                    Table.block_address(block_write.block),
-                );
+                compaction.tables_output_count += 1;
+                assert(compaction.tables_output_count <= compaction.range.table_count);
             }
         }
 
-        fn write_block_callback(comptime field: []const u8) fn (*Grid.Write) void {
-            return struct {
-                fn callback(write: *Grid.Write) void {
-                    const block_write = @fieldParentPtr(BlockWrite, "write", write);
-                    const compaction = @fieldParentPtr(Compaction, field, block_write);
+        fn cpu_merge_finish(compaction: *Compaction) void {
+            // Ensure this is the result of a compact_tick() call that finished processing IO.
+            assert(compaction.status == .processing);
+            assert(compaction.callback != null);
+            assert(compaction.io_pending == 0);
+            assert(!compaction.merge_done);
 
-                    io_callback(compaction);
+            // Ensure merging is truly finished.
+            assert(compaction.merge_iterator.?.empty());
+            assert(!compaction.data.writable);
+            assert(!compaction.filter.writable);
+            assert(!compaction.index.writable);
+
+            // Double check the iterators are finished as well.
+            const stream_empty = struct {
+                fn empty(it: anytype) bool {
+                    _ = it.peek() catch |err| switch (err) {
+                        error.Drained => {},
+                        error.Empty => {
+                            assert(it.buffered_all_values());
+                            return true;
+                        },
+                    };
+                    return false;
                 }
-            }.callback;
-        }
-
-        fn swap_buffers(block_write: *BlockWrite, block_ready: *BlockPtr) void {
-            mem.swap(BlockPtr, &block_write.block, block_ready);
-
-            assert(!block_write.ready);
-            block_write.ready = true;
-        }
-
-        fn assert_read_iterators_empty(compaction: Compaction) void {
-            assert(compaction.iterator_a.buffered_all_values());
-            assert(compaction.iterator_a.peek() == null);
-
-            assert(compaction.iterator_b.buffered_all_values());
-            assert(compaction.iterator_b.peek() == null);
-        }
-
-        fn stream_peek(compaction: *Compaction, stream_id: u32) ?Key {
-            assert(stream_id <= 1);
-
-            if (stream_id == 0) {
-                return compaction.iterator_a.peek();
+            }.empty;
+            assert(stream_empty(&compaction.iterator_a));
+            assert(stream_empty(&compaction.iterator_b));
+
+            // Mark the level_a table as invisible if it was provided;
+            // it has been merged into level_b.
+            // TODO: Release the grid blocks associated with level_a as well
+            if (compaction.level_a_input) |*level_a_table| {
+                const level_a = compaction.level_b - 1;
+                const snapshot_max = snapshot_max_for_table_input(compaction.op_min);
+                compaction.manifest.update_table(level_a, snapshot_max, level_a_table);
+                assert(level_a_table.snapshot_max == snapshot_max);
             } else {
-                return compaction.iterator_b.peek();
+                assert(compaction.level_b == 0);
             }
+
+            // Finally, mark Compaction as officially complete and ready to be reset().
+            compaction.merge_iterator = null;
+            compaction.merge_done = true;
+            compaction.status = .done;
         }
 
-        fn stream_pop(compaction: *Compaction, stream_id: u32) Value {
-            assert(stream_id <= 1);
+        pub fn reset(compaction: *Compaction) void {
+            assert(compaction.status == .done);
+            assert(compaction.callback == null);
+            assert(compaction.io_pending == 0);
+            assert(compaction.merge_done);
 
-            if (stream_id == 0) {
-                return compaction.iterator_a.pop();
-            } else {
-                return compaction.iterator_b.pop();
-            }
-        }
+            // TODO(Beat Pacing) This should really be where the compaction callback is invoked,
+            // but currently that can occur multiple times per beat.
+            compaction.grid.forfeit(compaction.grid_reservation);
 
-        /// Returns true if stream A has higher precedence than stream B.
-        /// This is used to deduplicate values across streams.
-        ///
-        /// This assumes that all overlapping tables in level A at the time the compaction was
-        /// started are included in the compaction. If this is not the case, the older table
-        /// in a pair of overlapping tables could be left in level A and shadow the newer table
-        /// in level B, resulting in data loss/invalid data.
-        fn stream_precedence(compaction: *Compaction, a: u32, b: u32) bool {
-            _ = compaction;
-
-            assert(a + b == 1);
-
-            // A stream_id of 0 indicates the level A iterator.
-            // All tables in level A have higher precedence.
-            return a == 0;
+            compaction.status = .idle;
+            compaction.merge_done = false;
         }
     };
 }
+
+fn snapshot_max_for_table_input(op_min: u64) u64 {
+    assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
+    return op_min + @divExact(config.lsm_batch_multiple, 2) - 1;
+}
+
+fn snapshot_min_for_table_output(op_min: u64) u64 {
+    assert(op_min % @divExact(config.lsm_batch_multiple, 2) == 0);
+    return op_min + @divExact(config.lsm_batch_multiple, 2);
+}