tigerbeetle-node 0.10.0 → 0.11.0

package/src/tigerbeetle/src/lsm/tree.zig

@@ -1,3 +1,4 @@
+//! An LSM tree.
 const std = @import("std");
 const builtin = @import("builtin");
 const assert = std.debug.assert;
@@ -11,13 +12,11 @@ const config = @import("../config.zig");
 const div_ceil = @import("../util.zig").div_ceil;
 const eytzinger = @import("eytzinger.zig").eytzinger;
 const vsr = @import("../vsr.zig");
-const binary_search = @import("binary_search.zig");
 const bloom_filter = @import("bloom_filter.zig");
 
 const CompositeKey = @import("composite_key.zig").CompositeKey;
 const NodePool = @import("node_pool.zig").NodePool(config.lsm_manifest_node_size, 16);
 const RingBuffer = @import("../ring_buffer.zig").RingBuffer;
-const SuperBlockType = vsr.SuperBlockType;
 
 /// We reserve maxInt(u64) to indicate that a table has not been deleted.
 /// Tables that have not been deleted have snapshot_max of maxInt(u64).
@@ -26,6 +25,8 @@ const SuperBlockType = vsr.SuperBlockType;
 /// to query all non-deleted tables.
 pub const snapshot_latest: u64 = math.maxInt(u64) - 1;
 
+const half_bar_beat_count = @divExact(config.lsm_batch_multiple, 2);
+
 // StateMachine:
 //
 // /// state machine will pass this on to all object stores
@@ -35,7 +36,7 @@ pub const snapshot_latest: u64 = math.maxInt(u64) - 1;
 // /// write the ops in batch to the memtable/objcache, previously called commit()
 // pub fn write(batch) void
 //
-// /// Flush in memory state to disk, preform merges, etc
+// /// Flush in memory state to disk, perform merges, etc
 // /// Only function that triggers Write I/O in LSMs, as well as some Read
 // /// Make as incremental as possible, don't block the main thread, avoid high latency/spikes
 // pub fn flush(callback) void
@@ -47,14 +48,14 @@ pub const snapshot_latest: u64 = math.maxInt(u64) - 1;
 // pub fn decode_superblock(buffer) void
 //
 
+/// The maximum number of tables for a single tree.
 pub const table_count_max = table_count_max_for_tree(config.lsm_growth_factor, config.lsm_levels);
 
-pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name: []const u8) type {
-    const Key = Table.Key;
-    const Value = Table.Value;
-    const compare_keys = Table.compare_keys;
-    const tombstone = Table.tombstone;
-    const tombstone_from_key = Table.tombstone_from_key;
+pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_name: []const u8) type {
+    const Key = TreeTable.Key;
+    const Value = TreeTable.Value;
+    const compare_keys = TreeTable.compare_keys;
+    const tombstone = TreeTable.tombstone;
 
     const tree_hash = blk: {
         // Blake3 hash does alot at comptime..
@@ -69,34 +70,25 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
         const Tree = @This();
 
         // Expose the Table & hash for the Groove.
-        pub const TableType = Table;
+        pub const Table = TreeTable;
         pub const name = tree_name;
         pub const hash = tree_hash;
 
         const Grid = @import("grid.zig").GridType(Storage);
         const Manifest = @import("manifest.zig").ManifestType(Table, Storage);
-        const TableMutable = @import("table_mutable.zig").TableMutableType(Table);
+        pub const TableMutable = @import("table_mutable.zig").TableMutableType(Table);
         const TableImmutable = @import("table_immutable.zig").TableImmutableType(Table);
 
         const CompactionType = @import("compaction.zig").CompactionType;
         const TableIteratorType = @import("table_iterator.zig").TableIteratorType;
         const TableImmutableIteratorType = @import("table_immutable.zig").TableImmutableIteratorType;
 
-        pub const ValueCache = std.HashMapUnmanaged(Value, void, Table.HashMapContextValue, 70);
-
         const CompactionTable = CompactionType(Table, Storage, TableIteratorType);
         const CompactionTableImmutable = CompactionType(Table, Storage, TableImmutableIteratorType);
 
         grid: *Grid,
         options: Options,
 
-        /// TODO(ifreund) Replace this with SetAssociativeCache:
-        /// A set associative cache of values shared by trees with the same key/value sizes.
-        /// This is used to accelerate point lookups and is not used for range queries.
-        /// Secondary index trees used only for range queries can therefore set this to null.
-        /// The value type will be []u8 and this will be shared by trees with the same value size.
-        value_cache: ?*ValueCache,
-
         table_mutable: TableMutable,
         table_immutable: TableImmutable,
 
@@ -111,7 +103,26 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
         /// This means, that for odd lsm_levels, the last CompactionTable is unused.
         compaction_table: [@divFloor(config.lsm_levels, 2)]CompactionTable,
 
+        /// While a compaction is running, this is the op of the last compact().
+        /// While no compaction is running, this is the op of the last compact() to complete.
+        /// (When recovering from a checkpoint, compaction_op starts at op_checkpoint).
         compaction_op: u64,
+
+        /// The maximum snapshot which is safe to prefetch from.
+        /// The minimum snapshot which can see the mutable table.
+        ///
+        /// This field ensures that the tree never queries the output tables of a running
+        /// compaction; they are incomplete.
+        ///
+        /// See lookup_snapshot_max_for_checkpoint().
+        ///
+        /// Invariants:
+        /// * `lookup_snapshot_max = compaction_op` while any compaction beat is in progress.
+        /// * `lookup_snapshot_max = compaction_op + 1` after a compaction beat finishes.
+        /// * `lookup_snapshot_max ≥ op_checkpoint + 1 + lsm_batch_multiple`
+        ///    when `op_checkpoint ≠ 0`.
+        lookup_snapshot_max: u64,
+
         compaction_io_pending: usize,
         compaction_callback: ?fn (*Tree) void,
 
@@ -120,22 +131,45 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
 
         pub const Options = struct {
             /// The maximum number of keys that may be committed per batch.
-            commit_count_max: u32,
+            ///
+            /// In general, the commit count max for a field depends on the field's object —
+            /// how many objects might be inserted/updated/removed by a batch:
+            ///   (config.message_size_max - sizeOf(vsr.header))
+            /// For example, there are at most 8191 transfers in a batch.
+            /// So commit_entries_max=8191 for transfer objects and indexes.
+            ///
+            /// However, if a transfer is ever mutated, then this will double commit_entries_max
+            /// since the old index might need to be removed, and the new index inserted.
+            ///
+            /// A way to see this is by looking at the state machine. If a transfer is inserted,
+            /// how many accounts and transfer put/removes will be generated?
+            ///
+            /// This also means looking at the state machine operation that will generate the
+            /// most put/removes in the worst case.
+            /// For example, create_accounts will put at most 8191 accounts.
+            /// However, create_transfers will put 2 accounts (8191 * 2) for every transfer, and
+            /// some of these accounts may exist, requiring a remove/put to update the index.
+            commit_entries_max: u32,
         };
 
         pub fn init(
             allocator: mem.Allocator,
             node_pool: *NodePool,
             grid: *Grid,
-            value_cache: ?*ValueCache,
+            values_cache: ?*TableMutable.ValuesCache,
             options: Options,
         ) !Tree {
-            var table_mutable = try TableMutable.init(allocator, options.commit_count_max);
+            assert(options.commit_entries_max > 0);
+            assert(grid.superblock.opened);
+
+            var table_mutable = try TableMutable.init(allocator, values_cache, options.commit_entries_max);
             errdefer table_mutable.deinit(allocator);
 
-            var table_immutable = try TableImmutable.init(allocator, options.commit_count_max);
+            var table_immutable = try TableImmutable.init(allocator, options.commit_entries_max);
             errdefer table_immutable.deinit(allocator);
 
+            assert(table_immutable.value_count_max == table_mutable.value_count_max);
+
             var manifest = try Manifest.init(allocator, node_pool, grid, tree_hash);
             errdefer manifest.deinit(allocator);
 
@@ -149,16 +183,21 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             }
             errdefer for (compaction_table) |*c| c.deinit(allocator);
 
+            // Compaction is one bar ahead of superblock's commit_min.
+            const op_checkpoint = grid.superblock.working.vsr_state.commit_min;
+            const lookup_snapshot_max = lookup_snapshot_max_for_checkpoint(op_checkpoint);
+            const compaction_op = op_checkpoint;
+
             return Tree{
                 .grid = grid,
                 .options = options,
-                .value_cache = value_cache,
                 .table_mutable = table_mutable,
                 .table_immutable = table_immutable,
                 .manifest = manifest,
                 .compaction_table_immutable = compaction_table_immutable,
                 .compaction_table = compaction_table,
-                .compaction_op = 0,
+                .compaction_op = compaction_op,
+                .lookup_snapshot_max = lookup_snapshot_max,
                 .compaction_io_pending = 0,
                 .compaction_callback = null,
                 .checkpoint_callback = null,
@@ -176,15 +215,6 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             tree.manifest.deinit(allocator);
         }
 
-        /// Get a cached value/tombstone for the given key.
-        /// Returns null if no value/tombstone for the given key is cached.
-        pub fn get_cached(tree: *const Tree, key: Key) ?*const Value {
-            const value = tree.table_mutable.get(key) orelse
-                tree.value_cache.?.getKeyPtr(tombstone_from_key(key));
-
-            return value;
-        }
-
         pub fn put(tree: *Tree, value: *const Value) void {
             tree.table_mutable.put(value);
         }
@@ -193,33 +223,41 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             tree.table_mutable.remove(value);
         }
 
-        pub fn lookup(
+        /// Returns the value from the mutable or immutable table (possibly a tombstone),
+        /// if one is available for the specified snapshot.
+        pub fn lookup_from_memory(tree: *Tree, snapshot: u64, key: Key) ?*const Value {
+            assert(tree.lookup_snapshot_max >= snapshot);
+
+            if (tree.lookup_snapshot_max == snapshot) {
+                if (tree.table_mutable.get(key)) |value| return value;
+            } else {
+                // The mutable table is converted to an immutable table when a snapshot is created.
+                // This means that a past snapshot will never be able to see the mutable table.
+                // This simplifies the mutable table and eliminates compaction for duplicate puts.
+            }
+
+            if (!tree.table_immutable.free and tree.table_immutable.snapshot_min <= snapshot) {
+                if (tree.table_immutable.get(key)) |value| return value;
+            } else {
+                // If the immutable table is invisible, then the mutable table is also invisible.
+                assert(tree.table_immutable.free or snapshot != tree.lookup_snapshot_max);
+            }
+
+            return null;
+        }
+
+        /// Call this function only after checking `lookup_from_memory()`.
+        pub fn lookup_from_levels(
             tree: *Tree,
             callback: fn (*LookupContext, ?*const Value) void,
             context: *LookupContext,
             snapshot: u64,
             key: Key,
         ) void {
-            assert(snapshot <= snapshot_latest);
-            if (snapshot == snapshot_latest) {
-                // The mutable table is converted to an immutable table when a snapshot is created.
-                // This means that a snapshot will never be able to see the mutable table.
-                // This simplifies the mutable table and eliminates compaction for duplicate puts.
-                // The value cache is only used for the latest snapshot for simplicity.
-                // Earlier snapshots will still be able to utilize the block cache.
-                if (tree.table_mutable.get(key) orelse
-                    tree.value_cache.?.getKeyPtr(tombstone_from_key(key))) |value|
-                {
-                    callback(context, unwrap_tombstone(value));
-                    return;
-                }
-            }
-
-            if (!tree.table_immutable.free and tree.table_immutable.snapshot_min < snapshot) {
-                if (tree.table_immutable.get(key)) |value| {
-                    callback(context, unwrap_tombstone(value));
-                    return;
-                }
+            assert(tree.lookup_snapshot_max >= snapshot);
+            if (config.verify) {
+                // The caller is responsible for checking the mutable table.
+                assert(tree.lookup_from_memory(snapshot, key) == null);
             }
 
             var index_block_count: u8 = 0;
@@ -285,12 +323,6 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
 
             callback: fn (*Tree.LookupContext, ?*const Value) void,
 
-            fn finish(context: *LookupContext, value: ?*const Value) void {
-                const callback = context.callback;
-                context.* = undefined;
-                callback(context, value);
-            }
-
             fn read_index_block(context: *LookupContext) void {
                 assert(context.data_block == null);
                 assert(context.index_block < context.index_block_count);
@@ -302,6 +334,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
                     &context.completion,
                     context.index_block_addresses[context.index_block],
                     context.index_block_checksums[context.index_block],
+                    .index,
                 );
             }
 
@@ -324,6 +357,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
                     completion,
                     blocks.filter_block_address,
                     blocks.filter_block_checksum,
+                    .filter,
                 );
             }
 
@@ -341,6 +375,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
                         completion,
                         context.data_block.?.address,
                         context.data_block.?.checksum,
+                        .data,
                     );
                 } else {
                     // The key is not present in this table, check the next level.
@@ -356,7 +391,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
                 assert(context.index_block_count <= config.lsm_levels);
 
                 if (Table.data_block_search(data_block, context.key)) |value| {
-                    context.finish(unwrap_tombstone(value));
+                    context.callback(context, unwrap_tombstone(value));
                 } else {
                     // The key is not present in this table, check the next level.
                     context.advance_to_next_level();
@@ -371,7 +406,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
 
                 context.index_block += 1;
                 if (context.index_block == context.index_block_count) {
-                    context.finish(null);
+                    context.callback(context, null);
                     return;
                 }
                 assert(context.index_block < context.index_block_count);
@@ -384,7 +419,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
         /// Returns null if the value is null or a tombstone, otherwise returns the value.
         /// We use tombstone values internally, but expose them as null to the user.
         /// This distinction enables us to cache a null result as a tombstone in our hash maps.
-        inline fn unwrap_tombstone(value: ?*const Value) ?*const Value {
+        pub inline fn unwrap_tombstone(value: ?*const Value) ?*const Value {
             return if (value == null or tombstone(value.?)) null else value.?;
         }
 
@@ -404,53 +439,6 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             callback(tree);
         }
 
-        // Tree compaction runs to the sound of music!
-        //
-        // Compacting LSM trees involves merging and moving tables into the next levels as needed.
-        // To avoid write amplification stalls and bound latency, compaction is done incrementally.
-        //
-        // A full compaction phase is denoted as a bar or measure, using terms from music notation.
-        // Each measure consists of `lsm_batch_multiple` beats or "compaction ticks" of work.
-        // A compaction beat is started asynchronously with `compact_io` which takes a callback.
-        // After `compact_io` is called, `compact_cpu` should be called to enable pipelining.
-        // The compaction beat completes when the `compact_io` callback is invoked.
-        //
-        // A measure is split in half according to the "first" down beat and "middle" down beat.
-        // The first half of the measure compacts even levels while the latter compacts odd levels.
-        // Mutable table changes are sorted and compacted into the immutable table.
-        // The immutable table is compacted into level 0 during the odd level half of the measure.
-        //
-        // At any given point, there's only levels/2 max compactions happening concurrently.
-        // The source level is denoted as `level_a` with the target level being `level_b`.
-        // The last level in the LSM tree has no target level so it's not compaction-from.
-        //
-        // Assuming a measure/`lsm_batch_multiple` of 4, the invariants can be described as follows:
-        //  * assert: at the end of every beat, there's space in mutable table for the next beat.
-        //  * manifest info for the tables compacted are updating during the compaction.
-        //  * manifest is compacted at the end of every beat.
-        //
-        //  - (first) down beat of the measure:
-        //      * assert: no compactions are currently running.
-        //      * compact immutable table if contains any sorted values (could be empty).
-        //      * allow level visible table counts to overflow if needed.
-        //      * start even level compactions if there's any tables to compact.
-        //
-        //  - (second) up beat of the measure:
-        //      * finish ticking running even-level compactions.
-        //      * assert: on callback completion, all compactions should be completed.
-        //
-        //  - (third) down beat of the measure:
-        //      * assert: no compactions are currently running.
-        //      * start odd level and immutable table compactions.
-        //
-        //  - (fourth) last beat of the measure:
-        //      * finish ticking running odd-level and immutable table compactions.
-        //      * assert: on callback completion, all compactions should be completed.
-        //      * assert: on callback completion, all level visible table counts shouldn't overflow.
-        //      * flush, clear, and sort mutable table values into immutable table for next measure.
-
-        const half_measure_beat_count = @divExact(config.lsm_batch_multiple, 2);
-
         const CompactionTableContext = struct {
             compaction: *CompactionTable,
             level_a: u8,
@@ -465,7 +453,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
                 assert(it.tree.compaction_callback != null);
 
                 const compaction_beat = it.tree.compaction_op % config.lsm_batch_multiple;
-                const even_levels = compaction_beat < half_measure_beat_count;
+                const even_levels = compaction_beat < half_bar_beat_count;
                 const level_a = (it.index * 2) + @boolToInt(!even_levels);
                 const level_b = level_a + 1;
 
@@ -489,77 +477,93 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
         /// This order (even levels, then odd levels) is significant, since it reduces the number of
         /// level 0 tables that overlap with the immutable table, reducing write amplification.
         ///
-        /// We therefore take the measure, during which all compactions run, and divide by two,
-        /// running the compactions from even levels in the first half measure, and then the odd.
+        /// We therefore take the bar, during which all compactions run, and divide by two,
+        /// running the compactions from even levels in the first half bar, and then the odd.
         ///
-        /// Compactions start on the down beat of a half measure, using 0-based beats.
-        /// For example, if there are 4 beats in a measure, start on beat 0 or beat 2.
+        /// Compactions start on the down beat of a half bar, using 0-based beats.
+        /// For example, if there are 4 beats in a bar, start on beat 0 or beat 2.
         pub fn compact(tree: *Tree, callback: fn (*Tree) void, op: u64) void {
-            tree.compact_start(callback, op);
-            tree.compact_drive();
-        }
+            assert(tree.compaction_callback == null);
+            assert(op != 0);
+            assert(op == tree.compaction_op + 1);
+            assert(op > tree.grid.superblock.working.vsr_state.commit_min);
 
-        fn compact_drive(tree: *Tree) void {
-            tree.compact_io();
-            // tree.manifest.manifest_log.superblock.storage.tick();
-            tree.compact_cpu();
+            tree.compaction_op = op;
+
+            if (tree.grid.superblock.working.vsr_state.op_compacted(op)) {
+                // We recovered from a checkpoint, and must avoid replaying one bar of
+                // compactions that were applied before the checkpoint. Repeating these ops'
+                // compactions would actually perform different compactions than before,
+                // causing the storage state of the replica to diverge from the cluster.
+                // See also: lookup_snapshot_max_for_checkpoint().
+
+                if (tree.compaction_op + 1 == tree.lookup_snapshot_max) {
+                    // This is the last op of the skipped compaction bar.
+                    // Prepare the immutable table for the next bar — since this state is
+                    // in-memory, it cannot be skipped.
+                    tree.compact_mutable_table_into_immutable();
+                }
+
+                // TODO Defer this callback until tick() to avoid stack growth.
+                callback(tree);
+                return;
+            }
+            assert(op == tree.lookup_snapshot_max);
+
+            tree.compact_start(callback);
+            tree.compact_drive();
         }
 
-        fn compact_start(tree: *Tree, callback: fn (*Tree) void, op: u64) void {
+        fn compact_start(tree: *Tree, callback: fn (*Tree) void) void {
             assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback == null);
-            
-            if (op > 0) assert(op > tree.compaction_op);
-            tree.compaction_op = op;
+
             tree.compaction_callback = callback;
 
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
             const start = (compaction_beat == 0) or
-                (compaction_beat == half_measure_beat_count);
+                (compaction_beat == half_bar_beat_count);
 
-            // The target snapshot of a compaction is actually the previous batch minus one.
-            //
-            // At the start of the current batch, mutable table inserts from the previous batch
-            // would be in the immutable table. This means the current batch compaction will
-            // actually be flushing to disk (levels) mutable table updates from the previous batch.
-            //
-            // -1 as the ops are zero based so the "last" op from previous batch is reflected.
-            const snapshot = std.mem.alignBackward(op, config.lsm_batch_multiple) -| 1;
-            assert(snapshot != snapshot_latest);
+            const op_min = compaction_op_min(tree.compaction_op);
+            assert(op_min < snapshot_latest);
+            assert(op_min % half_bar_beat_count == 0);
 
-            log.debug(tree_name ++ ": compact_start: op={d} snapshot={d} beat={d}/{d}", .{
-                op,
-                snapshot,
+            log.debug(tree_name ++ ": compact_start: op={d} op_min={d} beat={d}/{d}", .{
+                tree.compaction_op,
+                op_min,
                 compaction_beat + 1,
                 config.lsm_batch_multiple,
             });
 
             // Try to start compacting the immutable table.
-            const even_levels = compaction_beat < half_measure_beat_count;
+            const even_levels = compaction_beat < half_bar_beat_count;
             if (even_levels) {
                 assert(tree.compaction_table_immutable.status == .idle);
             } else {
-                if (start) tree.compact_io_start_table_immutable(snapshot);
+                if (start) tree.compact_start_table_immutable(op_min);
             }
 
             // Try to start compacting the other levels.
             var it = CompactionTableIterator{ .tree = tree };
             while (it.next()) |context| {
-                if (start) tree.compact_io_start_table(snapshot, context);
+                if (start) tree.compact_start_table(op_min, context);
             }
         }
 
-        fn compact_io_start_table_immutable(tree: *Tree, snapshot: u64) void {
+        fn compact_start_table_immutable(tree: *Tree, op_min: u64) void {
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            assert(compaction_beat == half_measure_beat_count);
+            assert(compaction_beat == half_bar_beat_count);
 
             // Do not start compaction if the immutable table does not require compaction.
             if (tree.table_immutable.free) return;
 
+            assert(tree.table_immutable.snapshot_min % half_bar_beat_count == 0);
+
             const values_count = tree.table_immutable.values.len;
             assert(values_count > 0);
 
             const level_b: u8 = 0;
+            const table_a: ?*const Manifest.TableInfo = null;
             const range = tree.manifest.compaction_range(
                 level_b,
                 tree.table_immutable.key_min(),
@@ -572,24 +576,25 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
 
             log.debug(tree_name ++
                 ": compacting immutable table to level 0 " ++
-                "(values.len={d} snapshot_min={d} compaction.snapshot={d} table_count={d})", .{
+                "(values.len={d} snapshot_min={d} compaction.op_min={d} table_count={d})", .{
                 tree.table_immutable.values.len,
                 tree.table_immutable.snapshot_min,
-                snapshot,
+                op_min,
                 range.table_count,
             });
 
             tree.compaction_table_immutable.start(
                 tree.grid,
                 &tree.manifest,
-                level_b,
+                op_min,
                 range,
-                snapshot,
+                table_a,
+                level_b,
                 .{ .table = &tree.table_immutable },
             );
         }
 
-        fn compact_io_start_table(tree: *Tree, snapshot: u64, context: CompactionTableContext) void {
+        fn compact_start_table(tree: *Tree, op_min: u64, context: CompactionTableContext) void {
             assert(context.level_a < config.lsm_levels);
             assert(context.level_b < config.lsm_levels);
             assert(context.level_a + 1 == context.level_b);
@@ -613,9 +618,10 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             context.compaction.start(
                 tree.grid,
                 &tree.manifest,
-                context.level_b,
+                op_min,
                 table_range.range,
-                snapshot,
+                table_range.table,
+                context.level_b,
                 .{
                     .grid = tree.grid,
                     .address = table.address,
@@ -624,72 +630,70 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             );
         }
 
-        fn compact_io(tree: *Tree) void {
+        fn compact_drive(tree: *Tree) void {
             assert(tree.compaction_io_pending <= 2 + tree.compaction_table.len);
             assert(tree.compaction_callback != null);
 
-            // Try to tick the cpu portion of the immutable table compaction:
+            // Always start one fake io_pending that is resolved right after
+            // to handle the case where this compaction tick triggers no IO.
+            // (For example, ticking the immutable table, or level B is already done).
+            tree.compaction_io_pending += 1;
+            defer tree.compact_tick_done();
+
+            // Try to tick the immutable table compaction:
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            const even_levels = compaction_beat < half_measure_beat_count;
+            const even_levels = compaction_beat < half_bar_beat_count;
             if (even_levels) {
                 assert(tree.compaction_table_immutable.status == .idle);
             } else {
-                if (tree.compaction_table_immutable.status == .compacting) {
-                    tree.compact_io_tick(&tree.compaction_table_immutable);
-                }
+                tree.compact_tick(&tree.compaction_table_immutable);
             }
 
-            // Try to tick the cpu portion of the level compactions:
+            // Try to tick the compaction for each level:
             var it = CompactionTableIterator{ .tree = tree };
             while (it.next()) |context| {
-                if (context.compaction.status == .compacting) {
-                    assert(context.compaction.level_b == context.level_b);
-                    tree.compact_io_tick(context.compaction);
-                }
+                tree.compact_tick(context.compaction);
             }
-
-            // Always start one io_pending that is resolved in compact_cpu()
-            // to handle the case of no level or immutable table being selected for compaction
-            tree.compaction_io_pending += 1;
-            assert(tree.compaction_io_pending <= 2 + tree.compaction_table.len);
         }
 
-        fn compact_io_tick(tree: *Tree, compaction: anytype) void {
+        fn compact_tick(tree: *Tree, compaction: anytype) void {
+            if (compaction.status != .processing) return;
             tree.compaction_io_pending += 1;
 
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            const even_levels = compaction_beat < half_measure_beat_count;
+            const even_levels = compaction_beat < half_bar_beat_count;
             assert(compaction.level_b < config.lsm_levels);
             assert(compaction.level_b % 2 == @boolToInt(even_levels));
 
             if (@TypeOf(compaction.*) == CompactionTableImmutable) {
                 assert(compaction.level_b == 0);
-                compaction.tick_io(Tree.compact_io_tick_callback_table_immutable);
-                log.debug(tree_name ++ ": queued compaction for immutable table to level 0", .{});
+                log.debug(tree_name ++ ": compact_tick() for immutable table to level 0", .{});
+                compaction.compact_tick(Tree.compact_tick_callback_table_immutable);
             } else {
-                compaction.tick_io(Tree.compact_io_tick_callback_table);
-                log.debug(tree_name ++ ": queued compaction for level {d} to level {d}", .{
+                assert(@TypeOf(compaction.*) == CompactionTable);
+                log.debug(tree_name ++ ": compact_tick() for level {d} to level {d}", .{
                     compaction.level_b - 1,
                     compaction.level_b,
                 });
+                compaction.compact_tick(Tree.compact_tick_callback_table);
             }
         }
 
-        fn compact_io_tick_callback_table_immutable(compaction: *CompactionTableImmutable) void {
-            assert(compaction.status == .compacting or compaction.status == .done);
+        fn compact_tick_callback_table_immutable(compaction: *CompactionTableImmutable) void {
+            assert(compaction.status == .processing or compaction.status == .done);
             assert(compaction.level_b < config.lsm_levels);
             assert(compaction.level_b == 0);
 
             const tree = @fieldParentPtr(Tree, "compaction_table_immutable", compaction);
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            assert(compaction_beat >= half_measure_beat_count);
+            assert(compaction_beat >= half_bar_beat_count);
 
-            log.debug(tree_name ++ ": compact_io complete for immutable table to level 0", .{});
-            tree.compact_io_tick_done();
+            log.debug(tree_name ++ ": compact_tick() complete for immutable table to level 0", .{});
+            tree.compact_tick_done();
         }
 
-        fn compact_io_tick_callback_table(compaction: *CompactionTable) void {
-            assert(compaction.status == .compacting or compaction.status == .done);
+        fn compact_tick_callback_table(compaction: *CompactionTable) void {
+            assert(compaction.status == .processing or compaction.status == .done);
             assert(compaction.level_b < config.lsm_levels);
             assert(compaction.level_b > 0);
 
@@ -699,182 +703,201 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
             const table_size = @divFloor(config.lsm_levels, 2);
             const table: *[table_size]CompactionTable = table_ptr[0..table_size];
 
-            const tree = @fieldParentPtr(Tree, "compaction_table", table);
-            log.debug(tree_name ++ ": compact_io complete for level {d} to level {d}", .{
+            log.debug(tree_name ++ ": compact_tick() complete for level {d} to level {d}", .{
                 compaction.level_b - 1,
                 compaction.level_b,
             });
 
-            tree.compact_io_tick_done();
+            const tree = @fieldParentPtr(Tree, "compaction_table", table);
+            tree.compact_tick_done();
         }
 
-        fn compact_io_tick_done(tree: *Tree) void {
+        fn compact_tick_done(tree: *Tree) void {
             assert(tree.compaction_io_pending <= 2 + tree.compaction_table.len);
             assert(tree.compaction_callback != null);
 
-            // compact_done() is called after all compact_io_tick()'s complete.
-            // This function can be triggered asynchronously or by compact_cpu() below.
+            // compact_done() is called after all compact_tick()'s complete.
             tree.compaction_io_pending -= 1;
             if (tree.compaction_io_pending == 0) tree.compact_done();
         }
 
-        fn compact_cpu(tree: *Tree) void {
-            assert(tree.compaction_io_pending <= 2 + tree.compaction_table.len);
+        /// Called at the end of each compaction tick.
+        fn compact_done(tree: *Tree) void {
+            assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback != null);
+            assert(tree.compaction_op == tree.lookup_snapshot_max);
 
-            // Try to tick the cpu portion of the immutable table compaction:
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            const even_levels = compaction_beat < half_measure_beat_count;
-            if (even_levels) {
-                assert(tree.compaction_table_immutable.status == .idle);
-            } else {
-                if (tree.compaction_table_immutable.status == .compacting) {
-                    tree.compaction_table_immutable.tick_cpu();
-                }
+            const even_levels = compaction_beat < half_bar_beat_count;
+            const compacted_levels_even = compaction_beat == half_bar_beat_count - 1;
+            const compacted_levels_odd = compaction_beat == config.lsm_batch_multiple - 1;
+            if (!compacted_levels_even and !compacted_levels_odd) {
+                // TODO(Deterministic Beats): Remove this when compact_done() is called exactly
+                // once when the beat finishes.
+                tree.lookup_snapshot_max = tree.compaction_op + 1;
+
+                tree.compact_finish();
+                return;
             }
 
-            // Try to tick the cpu portion of the level compactions:
-            var it = CompactionTableIterator{ .tree = tree };
-            while (it.next()) |context| {
-                if (context.compaction.status == .compacting) {
-                    assert(context.compaction.level_b == context.level_b);
-                    context.compaction.tick_cpu();
+            // At the end of the second and fourth beat:
+            // 1. Tick the Compactions until all have completed.
+            // 2. Remove invisible tables from the manifest.
+            // 3. Compact the manifest.
+            // Then at the end of the fourth beat, freeze the mutable table.
+            assert(compacted_levels_even or compacted_levels_odd);
+            assert(compacted_levels_even != compacted_levels_odd);
+
+            const still_compacting = blk: {
+                if (even_levels) {
+                    assert(tree.compaction_table_immutable.status == .idle);
+                } else {
+                    if (tree.compaction_table_immutable.status == .processing) break :blk true;
                 }
-            }
 
-            // Resolve the io_pending added by compact_io(). This may trigger compact_done().
-            tree.compact_io_tick_done();
-        }
-
-        fn compact_done(tree: *Tree) void {
-            assert(tree.compaction_io_pending == 0);
-            assert(tree.compaction_callback != null);
+                var it = CompactionTableIterator{ .tree = tree };
+                while (it.next()) |context| {
+                    if (context.compaction.status == .processing) break :blk true;
+                }
+                break :blk false;
+            };
 
-            var still_compacting = false;
+            if (still_compacting) {
+                // We are at the end of a half-bar, but the compactions have not finished.
+                // We keep ticking them until they finish.
+                log.debug(tree_name ++ ": compact_done: driving outstanding compactions", .{});
+                tree.compact_drive();
+                return;
+            }
 
-            // Mark immutable compaction that reported done in their callback as "completed".
-            const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            const even_levels = compaction_beat < half_measure_beat_count;
-            if (even_levels) {
-                assert(tree.compaction_table_immutable.status == .idle);
-            } else {
-                if (tree.compaction_table_immutable.status == .done) {
-                    tree.compaction_table_immutable.reset();
-                    tree.table_immutable.clear();
-                } else if (tree.compaction_table_immutable.status == .compacting) {
-                    still_compacting = true;
+            // TODO(Deterministic Beats): Move this to the top of the function when compact_done()
+            // is called exactly once when the beat finishes.
+            tree.lookup_snapshot_max = tree.compaction_op + 1;
+
+            // All compactions have finished for the current half-bar.
+            // We couldn't remove the (invisible) input tables until now because prefetch()
+            // needs a complete set of tables for lookups to avoid missing data.
+
+            // Reset the immutable table Compaction.
+            // Also clear any tables made invisible by the compaction.
+            if (!even_levels) {
+                switch (tree.compaction_table_immutable.status) {
+                    // The compaction wasn't started for this half bar.
+                    .idle => assert(tree.table_immutable.free),
+                    .processing => unreachable,
+                    .done => {
+                        tree.compaction_table_immutable.reset();
+                        tree.table_immutable.clear();
+                        tree.manifest.remove_invisible_tables(
+                            tree.compaction_table_immutable.level_b,
+                            tree.lookup_snapshot_max,
+                            tree.compaction_table_immutable.range.key_min,
+                            tree.compaction_table_immutable.range.key_max,
+                        );
+                    },
                 }
             }
 
-            // Mark compactions that reported done in their callback as "completed" (done = null).
+            // Reset all the other Compactions.
+            // Also clear any tables made invisible by the compactions.
             var it = CompactionTableIterator{ .tree = tree };
             while (it.next()) |context| {
-                if (context.compaction.status == .done) {
-                    assert(context.compaction.level_b == context.level_b);
-                    context.compaction.reset();
-                } else if (context.compaction.status == .compacting) {
-                    still_compacting = true;
+                switch (context.compaction.status) {
+                    .idle => {}, // The compaction wasn't started for this half bar.
+                    .processing => unreachable,
+                    .done => {
+                        context.compaction.reset();
+                        tree.manifest.remove_invisible_tables(
+                            context.compaction.level_b,
+                            tree.lookup_snapshot_max,
+                            context.compaction.range.key_min,
+                            context.compaction.range.key_max,
+                        );
+                        if (context.compaction.level_b > 0) {
+                            tree.manifest.remove_invisible_tables(
+                                context.compaction.level_b - 1,
+                                tree.lookup_snapshot_max,
+                                context.compaction.range.key_min,
+                                context.compaction.range.key_max,
+                            );
+                        }
+                    },
                 }
             }
 
-            // At the end of every beat, ensure mutable table can be flushed to immutable table.
-            assert(tree.table_mutable.can_commit_batch(tree.options.commit_count_max));
-
-            // At end of second/half measure:
-            // - assert: even compactions from previous tick are finished.
-            // - remove tables made invisible during compaction of even levels.
-            if (compaction_beat == half_measure_beat_count - 1) {
-                if (still_compacting) {
-                    log.debug(tree_name ++ ": compact_done: driving outstanding compactions", .{});
-                    return tree.compact_drive();
-                }
-
-                log.debug(tree_name ++ ": compact_done: compacted even levels", .{});
-
-                it = CompactionTableIterator{ .tree = tree };
-                while (it.next()) |context| {
-                    assert(context.compaction.status == .idle);
-                    tree.manifest.remove_invisible_tables(
-                        context.level_a,
-                        context.compaction.snapshot,
-                        context.compaction.range.key_min,
-                        context.compaction.range.key_max,
-                    );
-                }
+            assert(tree.compaction_table_immutable.status == .idle);
+            it = CompactionTableIterator{ .tree = tree };
+            while (it.next()) |context| {
+                assert(context.compaction.status == .idle);
             }
 
-            // At end of fourth/last measure:
-            // - assert: immutable table and odd level compactions from previous tick are finished.
-            // - remove tables made invisible during compaction of odd levels.
-            // - assert: all visible levels haven't overflowed their max.
-            // - convert mutable table to immutable tables for next measure.
-            if (compaction_beat == config.lsm_batch_multiple - 1) {
-                if (still_compacting) {
-                    log.debug(tree_name ++ ": compact_done: driving outstanding compactions", .{});
-                    return tree.compact_drive();
-                }
-
-                // TODO Make log message more accurate according to what was compacted.
-                log.debug(tree_name ++ ": compact_done: compacted immutable table and odd levels", .{});
-
-                assert(tree.compaction_table_immutable.status == .idle);
-                it = CompactionTableIterator{ .tree = tree };
-                while (it.next()) |context| {
-                    assert(context.compaction.status == .idle);
-                    tree.manifest.remove_invisible_tables(
-                        context.level_a,
-                        context.compaction.snapshot,
-                        context.compaction.range.key_min,
-                        context.compaction.range.key_max,
-                    );
-                }
-
+            // At the end of the fourth/last beat:
+            // - Assert all visible tables haven't overflowed their max per level.
+            // - Convert mutable table to immutable table for next bar.
+            if (compacted_levels_odd) {
                 tree.manifest.assert_level_table_counts();
                 tree.compact_mutable_table_into_immutable();
             }
 
-            // At the end of every beat, call manifest.compact before invoking the compact callback.
+            // At the end of the second/fourth beat:
+            // - Compact the manifest before invoking the compact() callback.
             tree.manifest.compact(compact_manifest_callback);
         }
 
-        fn compact_manifest_callback(manifest: *Manifest) void {
-            const tree = @fieldParentPtr(Tree, "manifest", manifest);
-            assert(tree.compaction_io_pending == 0);
-            assert(tree.compaction_callback != null);
-
-            // Invoke the compact_io() callback after the manifest compacts at the end of the beat.
-            const callback = tree.compaction_callback.?;
-            tree.compaction_callback = null;
-            callback(tree);
-        }
-
+        /// Called after the last beat of a full compaction bar.
         fn compact_mutable_table_into_immutable(tree: *Tree) void {
-            // Ensure mutable table can be flushed into immutable table.
-            if (tree.table_mutable.count() == 0) return;
             assert(tree.table_immutable.free);
+            assert((tree.compaction_op + 1) % config.lsm_batch_multiple == 0);
+            assert(tree.compaction_op + 1 == tree.lookup_snapshot_max);
+
+            if (tree.table_mutable.count() == 0) return;
 
             // Sort the mutable table values directly into the immutable table's array.
             const values_max = tree.table_immutable.values_max();
             const values = tree.table_mutable.sort_into_values_and_clear(values_max);
             assert(values.ptr == values_max.ptr);
 
-            // Take a manifest snapshot and setup the immutable table with the sorted values.
-            const snapshot_min = tree.compaction_table_immutable.snapshot;
-            tree.table_immutable.reset_with_sorted_values(snapshot_min, values);
+            // The immutable table must be visible to the next bar — setting its snapshot_min to
+            // lookup_snapshot_max guarantees.
+            //
+            // In addition, the immutable table is conceptually an output table of this compaction
+            // bar, and now its snapshot_min matches the snapshot_min of the Compactions' output
+            // tables.
+            tree.table_immutable.reset_with_sorted_values(tree.lookup_snapshot_max, values);
 
             assert(tree.table_mutable.count() == 0);
             assert(!tree.table_immutable.free);
         }
 
+        fn compact_manifest_callback(manifest: *Manifest) void {
+            const tree = @fieldParentPtr(Tree, "manifest", manifest);
+            assert(tree.compaction_io_pending == 0);
+            assert(tree.compaction_callback != null);
+            tree.compact_finish();
+        }
+
+        /// Called at the end of each compaction beat.
+        fn compact_finish(tree: *Tree) void {
+            assert(tree.compaction_io_pending == 0);
+            assert(tree.table_mutable.can_commit_batch(tree.options.commit_entries_max));
+
+            // Invoke the compact() callback after the manifest compacts at the end of the beat.
+            const callback = tree.compaction_callback.?;
+            tree.compaction_callback = null;
+            callback(tree);
+        }
+
         pub fn checkpoint(tree: *Tree, callback: fn (*Tree) void) void {
-            // Assert no outstanding compact_io() work..
+            // Assert no outstanding compact_tick() work..
             assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback == null);
+            assert(tree.compaction_op > 0);
+            assert(tree.compaction_op + 1 == tree.lookup_snapshot_max);
 
-            // Avoid checkpointing if this is not the last beat in the compaction measure.
+            // Assert that this is the last beat in the compaction bar.
             const compaction_beat = tree.compaction_op % config.lsm_batch_multiple;
-            const last_beat_in_measure = config.lsm_batch_multiple - 1;
-            if (compaction_beat != last_beat_in_measure) return callback(tree);
+            const last_beat_in_bar = config.lsm_batch_multiple - 1;
+            assert(last_beat_in_bar == compaction_beat);
 
             // Assert no outstanding compactions.
             assert(tree.compaction_table_immutable.status == .idle);
@@ -887,7 +910,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
 
             // Assert that we're checkpointing only after invisible tables have been removed.
             if (config.verify) {
-                tree.manifest.assert_no_invisible_tables(tree.compaction_op);
+                tree.manifest.assert_no_invisible_tables(compaction_op_min(tree.compaction_op));
             }
 
             // Start an asynchronous checkpoint on the manifest.
@@ -932,7 +955,7 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
         pub fn range_query(
             tree: *Tree,
             /// The snapshot timestamp, if any
-            snapshot: u64,
+            snapshot: ?u64,
             query: RangeQuery,
         ) RangeQueryIterator {
             _ = tree;
@@ -942,6 +965,87 @@ pub fn TreeType(comptime Table: type, comptime Storage: type, comptime tree_name
     };
 }
 
+/// Returns the first op of the compaction (Compaction.op_min) for a given op/beat.
+///
+/// After this compaction finishes:
+/// - `op_min + half_bar_beat_count - 1` will be the input tables' snapshot_max.
+/// - `op_min + half_bar_beat_count` will be the output tables' snapshot_min.
+///
+/// Each half-bar has a separate op_min (for deriving the output snapshot_min) instead of each full
+/// bar because this allows the output tables of the first half-bar's compaction to be prefetched
+/// against earlier — hopefully while they are still warm in the cache from being written.
+pub fn compaction_op_min(op: u64) u64 {
+    return op - op % half_bar_beat_count;
+}
+
+/// These charts depict the commit/compact ops and `lookup_snapshot_max` over a series of
+/// commits and compactions (with lsm_batch_multiple=8).
+///
+/// Legend:
+///
+///   ┼   full bar (first half-bar start)
+///   ┬   half bar (second half-bar start)
+///   $   lookup_snapshot_max (prefetch reads from the current snapshot)
+///       This is incremented at the end of each compact().
+///   .   op is in mutable table (in memory)
+///   ,   op is in immutable table (in memory)
+///   #   op is on disk
+///   ✓   checkpoint() may follow compact()
+///
+///   0 2 4 6 8 0 2 4 6
+///   ┼───┬───┼───┬───┼
+///   .$      ╷       ╷     init(superblock.commit_min=0)⎤ Compaction is effectively a noop for the
+///   .$      ╷       ╷     commit;compact( 1) start/end ⎥ first bar because there are no tables on
+///   ..$     ╷       ╷     commit;compact( 2) start/end ⎥ disk yet, and no immutable table to
+///   ...$    ╷       ╷     commit;compact( 3) start/end ⎥ flush.
+///   ....$   ╷       ╷     commit;compact( 4) start/end ⎥
+///   .....$  ╷       ╷     commit;compact( 5) start/end ⎥ This applies:
+///   ......$ ╷       ╷     commit;compact( 6) start/end ⎥ - when the LSM is starting on a freshly
+///   .......$╷       ╷     commit;compact( 7) start    ⎤⎥   formatted data file, and also
+///   ,,,,,,,,$       ╷  ✓         compact( 7)       end⎦⎦ - when the LSM is recovering from a crash
+///   ,,,,,,,,$       ╷     commit;compact( 8) start/end     (see below).
+///   ,,,,,,,,.$      ╷     commit;compact( 9) start/end
+///   ,,,,,,,,..$     ╷     commit;compact(10) start/end
+///   ,,,,,,,,...$    ╷     commit;compact(11) start/end
+///   ,,,,,,,,....$   ╷     commit;compact(12) start/end
+///   ,,,,,,,,.....$  ╷     commit;compact(13) start/end
+///   ,,,,,,,,......$ ╷     commit;compact(14) start/end
+///   ,,,,,,,,.......$╷     commit;compact(15) start    ⎤
+///   ########,,,,,,,,$  ✓         compact(15)       end⎦
+///   ########,,,,,,,,$     commit;compact(16) start/end
+///   ┼───┬───┼───┬───┼
+///   0 2 4 6 8 0 2 4 6
+///   ┼───┬───┼───┬───┼                                    Recover with a checkpoint taken at op 15.
+///   ########        $     init(superblock.commit_min=7)  At op 15, ops 8…15 are in memory, so they
+///   ########.       $     commit        ( 8) start/end ⎤ were dropped by the crash.
+///   ########..      $     commit        ( 9) start/end ⎥
+///   ########...     $     commit        (10) start/end ⎥ But compaction is not run for ops 8…15
+///   ########....    $     commit        (11) start/end ⎥ because it was already performed
+///   ########.....   $     commit        (12) start/end ⎥ before the checkpoint.
+///   ########......  $     commit        (13) start/end ⎥
+///   ########....... $     commit        (14) start/end ⎥ We can begin to compact again at op 16,
+///   ########........$     commit        (15) start    ⎤⎥ because those compactions (if previously
+///   ########,,,,,,,,$  ✓                (15)       end⎦⎦ performed) are not included in the
+///   ########,,,,,,,,$     commit;compact(16) start/end   checkpoint.
+///   ┼───┬───┼───┬───┼
+///   0 2 4 6 8 0 2 4 6
+///
+/// Notice how in the checkpoint recovery example above, we are careful not to `compact(op)` twice
+/// for any op (even if we crash/recover), since that could lead to differences between replicas'
+/// storage. The last bar of `commit()`s is always only in memory, so it is safe to repeat.
+///
+/// Additionally, while skipping compactions during recovery, we use a `lookup_snapshot_max`
+/// different than the original compactions — the old tables may have been removed during the
+/// checkpoint.
+fn lookup_snapshot_max_for_checkpoint(op_checkpoint: u64) u64 {
+    if (op_checkpoint == 0) {
+        // Start from 1 because we never commit op 0.
+        return 1;
+    } else {
+        return op_checkpoint + config.lsm_batch_multiple + 1;
+    }
+}
+
 /// The total number of tables that can be supported by the tree across so many levels.
 pub fn table_count_max_for_tree(growth_factor: u32, levels_count: u32) u32 {
     assert(growth_factor >= 4);
@@ -984,102 +1088,3 @@ test "table_count_max_for_level/tree" {
     try expectEqual(@as(u32, 37448 + 262144), table_count_max_for_tree(8, 6));
     try expectEqual(@as(u32, 299592 + 2097152), table_count_max_for_tree(8, 7));
 }
-
-pub fn main() !void {
-    const testing = std.testing;
-    const allocator = testing.allocator;
-
-    const IO = @import("../io.zig").IO;
-    const Storage = @import("../storage.zig").Storage;
-    const Grid = @import("grid.zig").GridType(Storage);
-
-    const data_file_size_min = @import("../vsr/superblock.zig").data_file_size_min;
-
-    const dir_fd = try IO.open_dir(".");
-    const storage_fd = try IO.open_file(dir_fd, "test_tree", data_file_size_min, true);
-    defer std.fs.cwd().deleteFile("test_tree") catch {};
-
-    var io = try IO.init(128, 0);
-    defer io.deinit();
-
-    var storage = try Storage.init(&io, storage_fd);
-    defer storage.deinit();
-
-    const Key = CompositeKey(u128);
-    const Table = @import("table.zig").TableType(
-        Key,
-        Key.Value,
-        Key.compare_keys,
-        Key.key_from_value,
-        Key.sentinel_key,
-        Key.tombstone,
-        Key.tombstone_from_key,
-    );
-
-    const Tree = TreeType(Table, Storage, @typeName(Table) ++ "_test");
-
-    // Check out our spreadsheet to see how we calculate node_count for a forest of trees.
-    const node_count = 1024;
-    var node_pool = try NodePool.init(allocator, node_count);
-    defer node_pool.deinit(allocator);
-
-    var value_cache = Tree.ValueCache{};
-    try value_cache.ensureTotalCapacity(allocator, 10000);
-    defer value_cache.deinit(allocator);
-
-    const batch_size_max = config.message_size_max - @sizeOf(vsr.Header);
-    const commit_count_max = @divFloor(batch_size_max, 128);
-
-    var sort_buffer = try allocator.allocAdvanced(
-        u8,
-        16,
-        // This must be the greatest commit_count_max and value_size across trees:
-        commit_count_max * config.lsm_batch_multiple * 128,
-        .exact,
-    );
-    defer allocator.free(sort_buffer);
-
-    // TODO Initialize SuperBlock:
-    var superblock: SuperBlockType(Storage) = undefined;
-
-    var grid = try Grid.init(allocator, &superblock);
-    defer grid.deinit(allocator);
-
-    var tree = try Tree.init(
-        allocator,
-        &node_pool,
-        &grid,
-        &value_cache,
-        .{
-            .prefetch_count_max = commit_count_max * 2,
-            .commit_count_max = commit_count_max,
-        },
-    );
-    defer tree.deinit(allocator);
-
-    testing.refAllDecls(@This());
-
-    // TODO: more references
-    _ = Table;
-    _ = Table.Builder.data_block_finish;
-
-    // TODO: more references
-    _ = Tree.CompactionTable;
-
-    _ = tree.prefetch_enqueue;
-    _ = tree.prefetch;
-    _ = tree.prefetch_key;
-    _ = tree.get;
-    _ = tree.put;
-    _ = tree.remove;
-    _ = tree.lookup;
-    _ = tree.compact_io;
-    _ = tree.compact_cpu;
-
-    _ = Tree.Manifest.LookupIterator.next;
-    _ = tree.manifest;
-    _ = tree.manifest.lookup;
-    _ = tree.manifest.insert_tables;
-
-    std.debug.print("table_count_max={}\n", .{table_count_max});
-}