tigerbeetle-node 0.11.13 → 0.12.0

package/src/tigerbeetle/src/lsm/binary_search.zig

@@ -1,341 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const math = std.math;
-
-pub const Config = struct {
-    verify: bool = false,
-};
-
-// TODO Add prefeching when @prefetch is available: https://github.com/ziglang/zig/issues/3600.
-//
-// TODO The Zig self hosted compiler will implement inlining itself before passing the IR to llvm,
-// which should eliminate the current poor codegen of key_from_value/compare_keys.
-
-/// Returns either the index of the first value equal to `key`,
-/// or if there is no such value then the index where `key` would be inserted.
-///
-/// In other words, return `i` such that both:
-/// * key_from_value(values[i])  >= key or i == values.len
-/// * key_value_from(values[i-1]) < key or i == 0
-///
-/// Doesn't perform the extra key comparison to determine if the match is exact.
-pub fn binary_search_values_raw(
-    comptime Key: type,
-    comptime Value: type,
-    comptime key_from_value: fn (*const Value) callconv(.Inline) Key,
-    comptime compare_keys: fn (Key, Key) callconv(.Inline) math.Order,
-    values: []const Value,
-    key: Key,
-    comptime config: Config,
-) u32 {
-    if (values.len == 0) return 0;
-
-    if (config.verify) {
-        // Input must be sorted by key.
-        for (values) |_, i| {
-            assert(i == 0 or
-                compare_keys(key_from_value(&values[i - 1]), key_from_value(&values[i])) != .gt);
-        }
-    }
-
-    var offset: usize = 0;
-    var length: usize = values.len;
-    while (length > 1) {
-        if (config.verify) {
-            assert(offset == 0 or
-                compare_keys(key_from_value(&values[offset - 1]), key) != .gt);
-            assert(offset + length == values.len or
-                compare_keys(key_from_value(&values[offset + length]), key) != .lt);
-        }
-
-        const half = length / 2;
-        const mid = offset + half;
-
-        // This trick seems to be what's needed to get llvm to emit branchless code for this,
-        // a ternary-style if expression was generated as a jump here for whatever reason.
-        const next_offsets = [_]usize{ offset, mid };
-        offset = next_offsets[@boolToInt(compare_keys(key_from_value(&values[mid]), key) == .lt)];
-
-        length -= half;
-    }
-
-    if (config.verify) {
-        assert(length == 1);
-        assert(offset == 0 or
-            compare_keys(key_from_value(&values[offset - 1]), key) != .gt);
-        assert(offset + length == values.len or
-            compare_keys(key_from_value(&values[offset + length]), key) != .lt);
-    }
-
-    offset += @boolToInt(compare_keys(key_from_value(&values[offset]), key) == .lt);
-
-    if (config.verify) {
-        assert(offset == 0 or
-            compare_keys(key_from_value(&values[offset - 1]), key) == .lt);
-        assert(offset == values.len or
-            compare_keys(key_from_value(&values[offset]), key) != .lt);
-    }
-
-    return @intCast(u32, offset);
-}
-
-pub inline fn binary_search_keys_raw(
-    comptime Key: type,
-    comptime compare_keys: fn (Key, Key) callconv(.Inline) math.Order,
-    keys: []const Key,
-    key: Key,
-    comptime config: Config,
-) u32 {
-    return binary_search_values_raw(
-        Key,
-        Key,
-        struct {
-            inline fn key_from_key(k: *const Key) Key {
-                return k.*;
-            }
-        }.key_from_key,
-        compare_keys,
-        keys,
-        key,
-        config,
-    );
-}
-
-const BinarySearchResult = struct {
-    index: u32,
-    exact: bool,
-};
-
-pub inline fn binary_search_values(
-    comptime Key: type,
-    comptime Value: type,
-    comptime key_from_value: fn (*const Value) callconv(.Inline) Key,
-    comptime compare_keys: fn (Key, Key) callconv(.Inline) math.Order,
-    values: []const Value,
-    key: Key,
-    comptime config: Config,
-) BinarySearchResult {
-    const index = binary_search_values_raw(Key, Value, key_from_value, compare_keys, values, key, config);
-    return .{
-        .index = index,
-        .exact = index < values.len and compare_keys(key_from_value(&values[index]), key) == .eq,
-    };
-}
-
-pub inline fn binary_search_keys(
-    comptime Key: type,
-    comptime compare_keys: fn (Key, Key) callconv(.Inline) math.Order,
-    keys: []const Key,
-    key: Key,
-    comptime config: Config,
-) BinarySearchResult {
-    const index = binary_search_keys_raw(Key, compare_keys, keys, key, config);
-    return .{
-        .index = index,
-        .exact = index < keys.len and compare_keys(keys[index], key) == .eq,
-    };
-}
-
-const test_binary_search = struct {
-    const fuzz = @import("../testing/fuzz.zig");
-
-    const log = false;
-
-    const gpa = std.testing.allocator;
-
-    inline fn compare_keys(a: u32, b: u32) math.Order {
-        return math.order(a, b);
-    }
-
-    fn less_than_key(_: void, a: u32, b: u32) bool {
-        return a < b;
-    }
-
-    fn exhaustive_search(keys_count: u32) !void {
-        const keys = try gpa.alloc(u32, keys_count);
-        defer gpa.free(keys);
-
-        for (keys) |*key, i| key.* = @intCast(u32, 7 * i + 3);
-
-        var target_key: u32 = 0;
-        while (target_key < keys_count + 13) : (target_key += 1) {
-            var expect: BinarySearchResult = .{ .index = 0, .exact = false };
-            for (keys) |key, i| {
-                switch (compare_keys(key, target_key)) {
-                    .lt => expect.index = @intCast(u32, i) + 1,
-                    .eq => {
-                        expect.exact = true;
-                        break;
-                    },
-                    .gt => break,
-                }
-            }
-
-            if (log) {
-                std.debug.print("keys:", .{});
-                for (keys) |k| std.debug.print("{},", .{k});
-                std.debug.print("\n", .{});
-                std.debug.print("target key: {}\n", .{target_key});
-            }
-
-            const actual = binary_search_keys(
-                u32,
-                compare_keys,
-                keys,
-                target_key,
-                .{ .verify = true },
-            );
-
-            if (log) std.debug.print("expected: {}, actual: {}\n", .{ expect, actual });
-            try std.testing.expectEqual(expect.index, actual.index);
-            try std.testing.expectEqual(expect.exact, actual.exact);
-        }
-    }
-
-    fn explicit_search(
-        keys: []const u32,
-        target_keys: []const u32,
-        expected_results: []const BinarySearchResult,
-    ) !void {
-        assert(target_keys.len == expected_results.len);
-
-        for (target_keys) |target_key, i| {
-            if (log) {
-                std.debug.print("keys:", .{});
-                for (keys) |k| std.debug.print("{},", .{k});
-                std.debug.print("\n", .{});
-                std.debug.print("target key: {}\n", .{target_key});
-            }
-            const expect = expected_results[i];
-            const actual = binary_search_keys(
-                u32,
-                compare_keys,
-                keys,
-                target_key,
-                .{ .verify = true },
-            );
-            try std.testing.expectEqual(expect.index, actual.index);
-            try std.testing.expectEqual(expect.exact, actual.exact);
-        }
-    }
-
-    fn random_search(random: std.rand.Random, iter: usize) !void {
-        const keys_count = @minimum(
-            @as(usize, 1E6),
-            fuzz.random_int_exponential(random, usize, iter),
-        );
-
-        const keys = try std.testing.allocator.alloc(u32, keys_count);
-        defer std.testing.allocator.free(keys);
-
-        for (keys) |*key| key.* = fuzz.random_int_exponential(random, u32, 100);
-        std.sort.sort(u32, keys, {}, less_than_key);
-        const target_key = fuzz.random_int_exponential(random, u32, 100);
-
-        var expect: BinarySearchResult = .{ .index = 0, .exact = false };
-        for (keys) |key, i| {
-            switch (compare_keys(key, target_key)) {
-                .lt => expect.index = @intCast(u32, i) + 1,
-                .eq => {
-                    expect.exact = true;
-                    break;
-                },
-                .gt => break,
-            }
-        }
-
-        const actual = binary_search_keys(
-            u32,
-            compare_keys,
-            keys,
-            target_key,
-            .{ .verify = true },
-        );
-
-        if (log) std.debug.print("expected: {}, actual: {}\n", .{ expect, actual });
-        try std.testing.expectEqual(expect.index, actual.index);
-        try std.testing.expectEqual(expect.exact, actual.exact);
-    }
-};
-
-// TODO test search on empty slice
-test "binary search: exhaustive" {
-    if (test_binary_search.log) std.debug.print("\n", .{});
-    var i: u32 = 1;
-    while (i < 300) : (i += 1) {
-        try test_binary_search.exhaustive_search(i);
-    }
-}
-
-test "binary search: explicit" {
-    if (test_binary_search.log) std.debug.print("\n", .{});
-    try test_binary_search.explicit_search(
-        &[_]u32{},
-        &[_]u32{0},
-        &[_]BinarySearchResult{
-            .{ .index = 0, .exact = false },
-        },
-    );
-    try test_binary_search.explicit_search(
-        &[_]u32{1},
-        &[_]u32{ 0, 1, 2 },
-        &[_]BinarySearchResult{
-            .{ .index = 0, .exact = false },
-            .{ .index = 0, .exact = true },
-            .{ .index = 1, .exact = false },
-        },
-    );
-    try test_binary_search.explicit_search(
-        &[_]u32{ 1, 3 },
-        &[_]u32{ 0, 1, 2, 3, 4 },
-        &[_]BinarySearchResult{
-            .{ .index = 0, .exact = false },
-            .{ .index = 0, .exact = true },
-            .{ .index = 1, .exact = false },
-            .{ .index = 1, .exact = true },
-            .{ .index = 2, .exact = false },
-        },
-    );
-    try test_binary_search.explicit_search(
-        &[_]u32{ 1, 3, 5, 8, 9, 11 },
-        &[_]u32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
-        &[_]BinarySearchResult{
-            .{ .index = 0, .exact = false },
-            .{ .index = 0, .exact = true },
-            .{ .index = 1, .exact = false },
-            .{ .index = 1, .exact = true },
-            .{ .index = 2, .exact = false },
-            .{ .index = 2, .exact = true },
-            .{ .index = 3, .exact = false },
-            .{ .index = 3, .exact = false },
-            .{ .index = 3, .exact = true },
-            .{ .index = 4, .exact = true },
-            .{ .index = 5, .exact = false },
-            .{ .index = 5, .exact = true },
-            .{ .index = 6, .exact = false },
-            .{ .index = 6, .exact = false },
-        },
-    );
-}
-
-test "binary search: duplicates" {
-    if (test_binary_search.log) std.debug.print("\n", .{});
-    try test_binary_search.explicit_search(
-        &[_]u32{ 0, 0, 3, 3, 3, 5, 5, 5, 5 },
-        &[_]u32{ 1, 2, 4, 6 },
-        &[_]BinarySearchResult{
-            .{ .index = 2, .exact = false },
-            .{ .index = 2, .exact = false },
-            .{ .index = 5, .exact = false },
-            .{ .index = 9, .exact = false },
-        },
-    );
-}
-
-test "binary search: random" {
-    var rng = std.rand.DefaultPrng.init(42);
-    var i: usize = 0;
-    while (i < 2048) : (i += 1) {
-        try test_binary_search.random_search(rng.random(), i);
-    }
-}

package/src/tigerbeetle/src/lsm/bloom_filter.zig

@@ -1,125 +0,0 @@
-//! Implementation of Split block Bloom filters: https://arxiv.org/pdf/2101.01719v4.pdf
-
-const std = @import("std");
-const assert = std.debug.assert;
-const mem = std.mem;
-const meta = std.meta;
-
-pub const Fingerprint = struct {
-    /// Hash value used to map key to block.
-    hash: u32,
-    /// Mask of bits set in the block for the key.
-    mask: meta.Vector(8, u32),
-
-    pub fn create(key: []const u8) Fingerprint {
-        const hash = std.hash.Wyhash.hash(0, key);
-        const hash_lower = @truncate(u32, hash);
-        const hash_upper = @intCast(u32, hash >> 32);
-
-        // TODO These constants are from the paper and we understand them to be arbitrary odd
-        // integers. Experimentally compare the performance of these with other randomly chosen
-        // odd integers to verify/improve our understanding.
-        const odd_integers: meta.Vector(8, u32) = [8]u32{
-            0x47b6137b,
-            0x44974d91,
-            0x8824ad5b,
-            0xa2b7289d,
-            0x705495c7,
-            0x2df1424b,
-            0x9efc4947,
-            0x5c6bfb31,
-        };
-
-        // Multiply-shift hashing. This produces 8 values in the range 0 to 31 (2^5 - 1).
-        const bit_indexes = (odd_integers *% @splat(8, hash_lower)) >> @splat(8, @as(u5, 32 - 5));
-
-        return .{
-            .hash = hash_upper,
-            .mask = @splat(8, @as(u32, 1)) << @intCast(meta.Vector(8, u5), bit_indexes),
-        };
-    }
-};
-
-/// Add the key with the given fingerprint to the filter.
-/// filter.len must be a multiple of 32.
-pub fn add(fingerprint: Fingerprint, filter: []u8) void {
-    comptime assert(@sizeOf(meta.Vector(8, u32)) == 32);
-
-    assert(filter.len > 0);
-    assert(filter.len % @sizeOf(meta.Vector(8, u32)) == 0);
-
-    const blocks = mem.bytesAsSlice([8]u32, filter);
-    const index = block_index(fingerprint.hash, filter.len);
-
-    const current: meta.Vector(8, u32) = blocks[index];
-    blocks[index] = current | fingerprint.mask;
-}
-
-/// Check if the key with the given fingerprint may have been added to the filter.
-/// filter.len must be a multiple of 32.
-pub fn may_contain(fingerprint: Fingerprint, filter: []const u8) bool {
-    comptime assert(@sizeOf(meta.Vector(8, u32)) == 32);
-
-    assert(filter.len > 0);
-    assert(filter.len % @sizeOf(meta.Vector(8, u32)) == 0);
-
-    const blocks = mem.bytesAsSlice([8]u32, filter);
-    const index = block_index(fingerprint.hash, filter.len);
-
-    const current: meta.Vector(8, u32) = blocks[index];
-    return @reduce(.Or, ~current & fingerprint.mask) == 0;
-}
-
-inline fn block_index(hash: u32, size: usize) u32 {
-    assert(size > 0);
-
-    const block_count = @divExact(size, @sizeOf(meta.Vector(8, u32)));
-    return @intCast(u32, (@as(u64, hash) * block_count) >> 32);
-}
-
-test {
-    _ = std.testing.refAllDecls(@This());
-}
-
-const test_bloom_filter = struct {
-    const fuzz = @import("../testing/fuzz.zig");
-    const block_size = @import("../constants.zig").block_size;
-
-    fn random_keys(random: std.rand.Random, iter: usize) !void {
-        const keys_count = @minimum(
-            @as(usize, 1E6),
-            fuzz.random_int_exponential(random, usize, iter),
-        );
-
-        const keys = try std.testing.allocator.alloc(u32, keys_count);
-        defer std.testing.allocator.free(keys);
-
-        for (keys) |*key| key.* = random.int(u32);
-
-        // `block_size` is currently the only size bloom_filter that we use.
-        const filter = try std.testing.allocator.alloc(u8, block_size);
-        std.mem.set(u8, filter, 0);
-        defer std.testing.allocator.free(filter);
-
-        for (keys) |key| {
-            add(Fingerprint.create(std.mem.asBytes(&key)), filter);
-        }
-        for (keys) |key| {
-            try std.testing.expect(may_contain(Fingerprint.create(std.mem.asBytes(&key)), filter));
-        }
-
-        // TODO Test the false positive rate:
-        // * Calculate the expected false positive rate
-        // * Test with a large number of random keys.
-        // * Use Chernoff bound or similar to determine a reasonable test cutoff.
-    }
-};
-
-test "bloom filter: random" {
-    var rng = std.rand.DefaultPrng.init(42);
-    const iterations_max: usize = (1 << 12);
-    var iterations: usize = 0;
-    while (iterations < iterations_max) : (iterations += 1) {
-        try test_bloom_filter.random_keys(rng.random(), iterations);
-    }
-}