tigerbeetle-node 0.10.0 → 0.11.0

package/src/tigerbeetle/src/lsm/segmented_array_benchmark.zig

@@ -0,0 +1,148 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const config = @import("../config.zig");
+const NodePoolType = @import("node_pool.zig").NodePool;
+const table_count_max_for_level = @import("tree.zig").table_count_max_for_level;
+const table_count_max_for_tree = @import("tree.zig").table_count_max_for_tree;
+const SortedSegmentedArray = @import("segmented_array.zig").SortedSegmentedArray;
+
+const samples = 5_000_000;
+
+const Options = struct {
+    Key: type,
+    value_size: u32,
+    value_count: u32,
+    node_size: u32,
+};
+
+// Benchmark 112B values to match `@sizeOf(TableInfo)`, which is either 112B or 80B depending on
+// the Key type.
+const configs = [_]Options{
+    Options{ .Key = u64, .value_size = 112, .value_count = 33, .node_size = 256 },
+    Options{ .Key = u64, .value_size = 112, .value_count = 34, .node_size = 256 },
+    Options{ .Key = u64, .value_size = 112, .value_count = 1024, .node_size = 256 },
+    Options{ .Key = u64, .value_size = 112, .value_count = 1024, .node_size = 512 },
+
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 1),
+        .node_size = config.lsm_manifest_node_size,
+    },
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 2),
+        .node_size = config.lsm_manifest_node_size,
+    },
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 3),
+        .node_size = config.lsm_manifest_node_size,
+    },
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 4),
+        .node_size = config.lsm_manifest_node_size,
+    },
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 5),
+        .node_size = config.lsm_manifest_node_size,
+    },
+    Options{
+        .Key = u64,
+        .value_size = 112,
+        .value_count = table_count_max_for_level(config.lsm_growth_factor, 6),
+        .node_size = config.lsm_manifest_node_size,
+    },
+};
+
+pub fn main() !void {
+    const stdout = std.io.getStdOut().writer();
+    var prng = std.rand.DefaultPrng.init(42);
+
+    inline for (configs) |options| {
+        const Key = options.Key;
+        const Value = struct {
+            key: Key,
+            padding: [options.value_size - @sizeOf(Key)]u8,
+        };
+
+        const NodePool = NodePoolType(options.node_size, @alignOf(Value));
+        const SegmentedArray = SortedSegmentedArray(
+            Value,
+            NodePool,
+            // Must be max of both to avoid hitting SegmentedArray's assertion:
+            //   assert(element_count_max > node_capacity);
+            comptime std.math.max(
+                options.value_count,
+                @divFloor(options.node_size, @sizeOf(Key)) + 1,
+            ),
+            Key,
+            struct {
+                inline fn key_from_value(value: *const Value) Key {
+                    return value.key;
+                }
+            }.key_from_value,
+            struct {
+                inline fn compare_keys(a: Key, b: Key) std.math.Order {
+                    return std.math.order(a, b);
+                }
+            }.compare_keys,
+            .{ .verify = false },
+        );
+
+        var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+        defer arena.deinit();
+        const allocator = arena.allocator();
+
+        var node_pool = try NodePool.init(allocator, SegmentedArray.node_count_max);
+        defer node_pool.deinit(allocator);
+
+        var array = try SegmentedArray.init(allocator);
+        defer array.deinit(allocator, &node_pool);
+
+        var i: usize = 0;
+        while (i < options.value_count) : (i += 1) {
+            _ = array.insert_element(&node_pool, .{
+                .key = prng.random().uintLessThanBiased(u64, options.value_count),
+                .padding = [_]u8{0} ** (options.value_size - @sizeOf(Key)),
+            });
+        }
+
+        const queries = try alloc_shuffled_index(allocator, options.value_count, prng.random());
+        defer allocator.free(queries);
+
+        const timer = try std.time.Timer.start();
+        const repetitions = std.math.max(1, @divFloor(samples, queries.len));
+        var j: usize = 0;
+        while (j < repetitions) : (j += 1) {
+            for (queries) |query| {
+                std.mem.doNotOptimizeAway(array.absolute_index_for_cursor(array.search(query)));
+            }
+        }
+        const time = timer.read() / repetitions / queries.len;
+
+        try stdout.print("KeyType={} ValueCount={:_>7} ValueSize={:_>2}B NodeSize={:_>6}B LookupTime={:_>6}ns\n", .{
+            options.Key,
+            options.value_count,
+            options.value_size,
+            options.node_size,
+            time,
+        });
+    }
+}
+
+// shuffle([0,1,…,n-1])
+fn alloc_shuffled_index(allocator: std.mem.Allocator, n: usize, rand: std.rand.Random) ![]usize {
+    // Allocate on the heap; the array may be too large to fit on the stack.
+    var indices = try allocator.alloc(usize, n);
+    for (indices) |*i, j| i.* = j;
+    rand.shuffle(usize, indices[0..]);
+    return indices;
+}

package/src/tigerbeetle/src/lsm/segmented_array_fuzz.zig

@@ -0,0 +1,9 @@
+const std = @import("std");
+
+const fuzz = @import("../test/fuzz.zig");
+const segmented_array = @import("segmented_array.zig");
+
+pub fn main() !void {
+    const fuzz_args = try fuzz.parse_fuzz_args(std.testing.allocator);
+    try segmented_array.run_tests(fuzz_args.seed, .{ .verify = true });
+}

package/src/tigerbeetle/src/lsm/set_associative_cache.zig

@@ -8,6 +8,7 @@ const meta = std.meta;
 const Vector = meta.Vector;
 
 const config = @import("../config.zig");
+const div_ceil = @import("../util.zig").div_ceil;
 const verify = config.verify;
 
 pub const Layout = struct {
@@ -19,6 +20,7 @@ pub const Layout = struct {
     value_alignment: ?u29 = null,
 };
 
+/// Each Key is associated with a set of n consecutive ways (or slots) that may contain the Value.
 pub fn SetAssociativeCache(
     comptime Key: type,
     comptime Value: type,
@@ -86,9 +88,37 @@ pub fn SetAssociativeCache(
         const Clock = meta.Int(.unsigned, clock_hand_bits);
 
         sets: u64,
+
+        /// A short, partial hash of a Key, corresponding to a Value.
+        /// Because the tag is small, collisions are possible:
+        /// `tag(v₁) = tag(v₂)` does not imply `v₁ = v₂`.
+        /// However, most of the time, where the tag differs, a full key comparison can be avoided.
+        /// Since tags are 16-32x smaller than keys, they can also be kept hot in cache.
         tags: []Tag,
+
+        /// When the corresponding Count is zero, the Value is absent.
         values: []align(value_alignment) Value,
+
+        /// Each value has a Count, which tracks the number of recent reads.
+        ///
+        /// * A Count is incremented when the value is accessed by `get`.
+        /// * A Count is decremented when a cache write to the value's Set misses.
+        /// * The value is evicted when its Count reaches zero.
+        ///
         counts: PackedUnsignedIntegerArray(Count),
+
+        /// Each set has a Clock: a counter that cycles between each of the set's ways (i.e. slots).
+        ///
+        /// On cache write, entries are checked for occupancy (or eviction) beginning from the
+        /// clock's position, wrapping around.
+        ///
+        /// The algorithm implemented is "CLOCK Nth-Chance" — each way has more than one bit,
+        /// to give ways more than one chance before eviction.
+        ///
+        /// * A similar algorithm called "RRIParoo" is described in
+        ///   "Kangaroo: Caching Billions of Tiny Objects on Flash".
+        /// * For more general information on CLOCK algorithms, see:
+        ///   https://en.wikipedia.org/wiki/Page_replacement_algorithm.
         clocks: PackedUnsignedIntegerArray(Clock),
 
         pub fn init(allocator: mem.Allocator, value_count_max: u64) !Self {
@@ -158,6 +188,13 @@ pub fn SetAssociativeCache(
             mem.set(u64, self.clocks.words, 0);
         }
 
+        /// Returns whether an entry with the given key is cached, 
+        /// without modifying the entry's counter.
+        pub fn exists(self: *Self, key: Key) bool {
+            const set = self.associate(key);
+            return self.search(set, key) != null;
+        }
+
         pub fn get(self: *Self, key: Key) ?*align(value_alignment) Value {
             const set = self.associate(key);
             const way = self.search(set, key) orelse return null;
@@ -176,8 +213,17 @@ pub fn SetAssociativeCache(
             self.counts.set(set.offset + way, 0);
         }
 
+        /// Hint that the key is less likely to be accessed in the future, without actually removing
+        /// it from the cache.
+        pub fn demote(self: *Self, key: Key) void {
+            const set = self.associate(key);
+            const way = self.search(set, key) orelse return;
+
+            self.counts.set(set.offset + way, 1);
+        }
+
         /// If the key is present in the set, returns the way. Otherwise returns null.
-        inline fn search(self: *Self, set: Set, key: Key) ?usize {
+        inline fn search(self: *const Self, set: Set, key: Key) ?usize {
             const ways = search_tags(set.tags, set.tag);
 
             var it = BitIterator(Ways){ .bits = ways };
@@ -194,7 +240,7 @@ pub fn SetAssociativeCache(
         /// Where each set bit represents the index of a way that has the same tag.
         const Ways = meta.Int(.unsigned, layout.ways);
 
-        inline fn search_tags(tags: *[layout.ways]Tag, tag: Tag) Ways {
+        inline fn search_tags(tags: *const [layout.ways]Tag, tag: Tag) Ways {
             const x: Vector(layout.ways, Tag) = tags.*;
             const y: Vector(layout.ways, Tag) = @splat(layout.ways, tag);
 
@@ -202,8 +248,8 @@ pub fn SetAssociativeCache(
             return @ptrCast(*const Ways, &result).*;
         }
 
-        pub fn put_no_clobber(self: *Self, key: Key) *align(value_alignment) Value {
-            return self.put_no_clobber_preserve_locked(
+        pub fn insert(self: *Self, key: Key) *align(value_alignment) Value {
+            return self.insert_preserve_locked(
                 void,
                 struct {
                     inline fn locked(_: void, _: *const Value) bool {
@@ -219,7 +265,7 @@ pub fn SetAssociativeCache(
         /// The key must not already be in the cache.
         /// Never evicts keys for which locked() returns true.
         /// The caller must guarantee that locked() returns true for less than layout.ways keys.
-        pub fn put_no_clobber_preserve_locked(
+        pub fn insert_preserve_locked(
             self: *Self,
             comptime Context: type,
             comptime locked: fn (
@@ -230,8 +276,12 @@ pub fn SetAssociativeCache(
             key: Key,
         ) *align(value_alignment) Value {
             const set = self.associate(key);
-
-            if (verify) assert(self.search(set, key) == null);
+            if (self.search(set, key)) |way| {
+                // Remove the old entry for this key.
+                // It should be a different value, but since we are returning a value pointer we
+                // can't check against the new one.
+                self.counts.set(set.offset + way, 0);
+            }
 
             const clock_index = @divExact(set.offset, layout.ways);
 
@@ -380,7 +430,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.put_no_clobber(key).* = key;
+                    sac.insert(key).* = key;
                     try expect(sac.counts.get(i) == 1);
                     try expectEqual(key, sac.get(key).?.*);
                     try expect(sac.counts.get(i) == 2);
@@ -393,7 +443,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.put_no_clobber(key).* = key;
+                sac.insert(key).* = key;
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);
@@ -422,7 +472,7 @@ fn set_associative_cache_test(
                 const key = (layout.ways + 1) * sac.sets;
                 const expect_evicted = layout.ways * sac.sets;
 
-                sac.put_no_clobber_preserve_locked(
+                sac.insert_preserve_locked(
                     u64,
                     struct {
                         inline fn locked(only_unlocked: u64, value: *const Value) bool {
@@ -462,7 +512,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.put_no_clobber(key).* = key;
+                    sac.insert(key).* = key;
                     try expect(sac.counts.get(i) == 1);
                     var j: usize = 2;
                     while (j <= math.maxInt(SAC.Count)) : (j += 1) {
@@ -480,7 +530,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.put_no_clobber(key).* = key;
+                sac.insert(key).* = key;
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);

package/src/tigerbeetle/src/lsm/table.zig

@@ -12,8 +12,48 @@ const div_ceil = @import("../util.zig").div_ceil;
 const eytzinger = @import("eytzinger.zig").eytzinger;
 const snapshot_latest = @import("tree.zig").snapshot_latest;
 
+const BlockType = @import("grid.zig").BlockType;
 const TableInfoType = @import("manifest.zig").TableInfoType;
 
+/// A table is a set of blocks:
+///
+/// * Index block (exactly 1)
+/// * Filter blocks (at least one, at most `filter_block_count_max`)
+///   Each filter block summarizes the keys for several adjacent (in terms of key) data blocks.
+/// * Data blocks (at least one, at most `data_block_count_max`)
+///   Store the actual keys/values, along with a small index of the keys to optimize lookups.
+///
+///
+/// Every block begins with a `vsr.Header` that includes:
+///
+/// * `checksum`, `checksum_body` verify the data integrity.
+/// * `cluster` is the cluster id.
+/// * `command` is `.block`.
+/// * `op` is the block address.
+/// * `size` is the block size excluding padding.
+///
+/// Index block schema:
+/// │ vsr.Header                   │ operation=BlockType.index
+/// │ vsr.Header                   │ commit=filter_block_count,
+/// │                              │ request=data_block_count,
+/// │                              │ timestamp=snapshot_min
+/// │ [filter_block_count_max]u128 │ checksums of filter blocks
+/// │ [data_block_count_max]u128   │ checksums of data blocks
+/// │ [data_block_count_max]Key    │ the maximum/last key in the respective data block
+/// │ [filter_block_count_max]u64  │ addresses of filter blocks
+/// │ [data_block_count_max]u64    │ addresses of data blocks
+/// │ […]u8{0}                     │ padding (to end of block)
+///
+/// Filter block schema:
+/// │ vsr.Header │ operation=BlockType.filter
+/// │ […]u8      │ A split-block Bloom filter, "containing" every key from as many as
+/// │            │   `filter_data_block_count_max` data blocks.
+///
+/// Data block schema:
+/// │ vsr.Header               │ operation=BlockType.data
+/// │ [block_key_count + 1]Key │ Eytzinger-layout keys from a subset of the values.
+/// │ [≤value_count_max]Value  │ At least one value (no empty tables).
+/// │ […]u8{0}                 │ padding (to end of block)
 pub fn TableType(
     comptime TableKey: type,
     comptime TableValue: type,
@@ -84,7 +124,7 @@ pub fn TableType(
         const table_block_count_max = @divExact(table_size_max, block_size);
         const block_body_size = block_size - @sizeOf(vsr.Header);
 
-        pub const layout = blk: {
+        pub const layout = layout: {
             assert(block_size % config.sector_size == 0);
             assert(math.isPowerOfTwo(table_size_max));
             assert(math.isPowerOfTwo(block_size));
@@ -158,15 +198,18 @@ pub fn TableType(
                 block_value_count_max * filter_bytes_per_key,
             );
 
+            // Compute the number of data and filter blocks by solving the constraints:
+            // * the filter and data blocks' metadata must fix in the index block
+            // * the filter blocks must index all data blocks
+            // * minimize the number of filter blocks
+            // * maximize the number of data blocks
             var data_blocks = table_block_count_max - index_block_count;
-            var data_index_size = 0;
             var filter_blocks = 0;
-            var filter_index_size = 0;
             while (true) : (data_blocks -= 1) {
-                data_index_size = data_index_entry_size * data_blocks;
-
                 filter_blocks = div_ceil(data_blocks, filter_data_block_count_max);
-                filter_index_size = filter_index_entry_size * filter_blocks;
+
+                const data_index_size = data_index_entry_size * data_blocks;
+                const filter_index_size = filter_index_entry_size * filter_blocks;
 
                 const index_size = @sizeOf(vsr.Header) + data_index_size + filter_index_size;
                 const table_block_count = index_block_count + filter_blocks + data_blocks;
@@ -178,14 +221,18 @@ pub fn TableType(
             const table_block_count = index_block_count + filter_blocks + data_blocks;
             assert(table_block_count <= table_block_count_max);
 
-            break :blk .{
+            break :layout .{
+                // The number of keys in the Eytzinger layout per data block.
                 .block_key_count = block_key_count,
+                // The number of bytes used by the keys in the data block.
                 .block_key_layout_size = block_key_layout_size,
+                // The maximum number of values in a data block.
                 .block_value_count_max = block_value_count_max,
 
                 .data_block_count_max = data_blocks,
                 .filter_block_count_max = filter_blocks,
 
+                // The number of data blocks covered by a single filter block.
                 .filter_data_block_count_max = filter_data_block_count_max,
             };
         };
@@ -397,21 +444,11 @@ pub fn TableType(
             }
         }
 
-        fn blocks_used(table: *Table) u32 {
-            assert(!table.free);
-            return Table.index_blocks_used(&table.blocks[0]);
-        }
-
-        fn filter_blocks_used(table: *Table) u32 {
-            assert(!table.free);
-            return Table.index_filter_blocks_used(&table.blocks[0]);
-        }
-
         pub const Builder = struct {
             const TableInfo = TableInfoType(Table);
 
-            key_min: Key = undefined,
-            key_max: Key = undefined,
+            key_min: Key = undefined, // Inclusive.
+            key_max: Key = undefined, // Inclusive.
 
             index_block: BlockPtr,
             filter_block: BlockPtr,
@@ -496,6 +533,7 @@ pub fn TableType(
                 // For each block we write the sorted values, initialize the Eytzinger layout,
                 // complete the block header, and add the block's max key to the table index.
 
+                assert(options.address > 0);
                 assert(builder.value > 0);
 
                 const block = builder.data_block;
@@ -503,6 +541,7 @@ pub fn TableType(
                 assert(values_max.len == data.value_count_max);
 
                 const values = values_max[0..builder.value];
+                const key_max = key_from_value(&values[values.len - 1]);
 
                 if (config.verify) {
                     var a = &values[0];
@@ -533,6 +572,7 @@ pub fn TableType(
                 const block_padding = block[data.padding_offset..][0..data.padding_size];
                 mem.set(u8, values_padding, 0);
                 mem.set(u8, block_padding, 0);
+                assert(compare_keys(key_from_value(&values[values.len - 1]), key_max) == .eq);
 
                 const header_bytes = block[0..@sizeOf(vsr.Header)];
                 const header = mem.bytesAsValue(vsr.Header, header_bytes);
@@ -541,15 +581,14 @@ pub fn TableType(
                     .cluster = options.cluster,
                     .op = options.address,
                     .request = @intCast(u32, values.len),
-                    .size = block_size - @intCast(u32, values_padding.len - block_padding.len),
+                    .size = block_size - @intCast(u32, values_padding.len + block_padding.len),
                     .command = .block,
+                    .operation = BlockType.data.operation(),
                 };
 
                 header.set_checksum_body(block[@sizeOf(vsr.Header)..header.size]);
                 header.set_checksum();
 
-                const key_max = key_from_value(&values[values.len - 1]);
-
                 const current = builder.data_block_count;
                 index_data_keys(builder.index_block)[current] = key_max;
                 index_data_addresses(builder.index_block)[current] = options.address;
@@ -559,7 +598,7 @@ pub fn TableType(
                 builder.key_max = key_max;
 
                 if (current == 0 and values.len == 1) {
-                    assert(compare_keys(builder.key_min, builder.key_max) != .gt);
+                    assert(compare_keys(builder.key_min, builder.key_max) == .eq);
                 } else {
                     assert(compare_keys(builder.key_min, builder.key_max) == .lt);
                 }
@@ -587,6 +626,7 @@ pub fn TableType(
 
             pub fn filter_block_finish(builder: *Builder, options: FilterFinishOptions) void {
                 assert(!builder.filter_block_empty());
+                assert(options.address > 0);
 
                 const header_bytes = builder.filter_block[0..@sizeOf(vsr.Header)];
                 const header = mem.bytesAsValue(vsr.Header, header_bytes);
@@ -595,6 +635,7 @@ pub fn TableType(
                     .op = options.address,
                     .size = block_size - filter.padding_size,
                     .command = .block,
+                    .operation = BlockType.filter.operation(),
                 };
 
                 const body = builder.filter_block[@sizeOf(vsr.Header)..header.size];
@@ -626,6 +667,7 @@ pub fn TableType(
             };
 
             pub fn index_block_finish(builder: *Builder, options: IndexFinishOptions) TableInfo {
+                assert(options.address > 0);
                 assert(builder.data_block_count > 0);
                 assert(builder.value == 0);
                 assert(builder.data_blocks_in_filter == 0);
@@ -658,6 +700,7 @@ pub fn TableType(
                     .timestamp = options.snapshot_min,
                     .size = index.size,
                     .command = .block,
+                    .operation = BlockType.index.operation(),
                 };
                 header.set_checksum_body(index_block[@sizeOf(vsr.Header)..header.size]);
                 header.set_checksum();
@@ -764,14 +807,6 @@ pub fn TableType(
             return slice[0..index_filter_blocks_used(index_block)];
         }
 
-        inline fn index_snapshot_min(index_block: BlockPtrConst) u32 {
-            const header = mem.bytesAsValue(vsr.Header, index_block[0..@sizeOf(vsr.Header)]);
-            const value = @intCast(u32, header.offset);
-            assert(value > 0);
-            assert(value < snapshot_latest);
-            return value;
-        }
-
         inline fn index_blocks_used(index_block: BlockPtrConst) u32 {
             return index_block_count + index_filter_blocks_used(index_block) +
                 index_data_blocks_used(index_block);
@@ -794,7 +829,7 @@ pub fn TableType(
         }
 
         /// Returns the zero-based index of the data block that may contain the key.
-        /// May be called on an index block only when the key is already in range of the table.
+        /// May be called on an index block only when the key is in range of the table.
         inline fn index_data_block_for_key(index_block: BlockPtrConst, key: Key) u32 {
             // Because we store key_max in the index block we can use the raw binary search
             // here and avoid the extra comparison. If the search finds an exact match, we
@@ -806,6 +841,7 @@ pub fn TableType(
                 compare_keys,
                 Table.index_data_keys_used(index_block),
                 key,
+                .{},
             );
             assert(data_block_index < index_data_blocks_used(index_block));
             return data_block_index;
@@ -819,7 +855,7 @@ pub fn TableType(
         };
 
         /// Returns all data stored in the index block relating to a given key.
-        /// May be called on an index block only when the key is already in range of the table.
+        /// May be called on an index block only when the key is in range of the table.
         pub inline fn index_blocks_for_key(index_block: BlockPtrConst, key: Key) IndexBlocks {
             const d = Table.index_data_block_for_key(index_block, key);
             const f = @divFloor(d, filter.data_block_count_max);
@@ -885,22 +921,21 @@ pub fn TableType(
                 key,
             );
 
-            if (values.len > 0) {
-                const result = binary_search.binary_search_values(
-                    Key,
-                    Value,
-                    key_from_value,
-                    compare_keys,
-                    values,
-                    key,
-                );
-                if (result.exact) {
-                    const value = &values[result.index];
-                    if (config.verify) {
-                        assert(compare_keys(key, key_from_value(value)) == .eq);
-                    }
-                    return value;
+            const result = binary_search.binary_search_values(
+                Key,
+                Value,
+                key_from_value,
+                compare_keys,
+                values,
+                key,
+                .{},
+            );
+            if (result.exact) {
+                const value = &values[result.index];
+                if (config.verify) {
+                    assert(compare_keys(key, key_from_value(value)) == .eq);
                 }
+                return value;
             }
 
             if (config.verify) {