tigerbeetle-node 0.11.0 → 0.11.2

package/src/tigerbeetle/src/vsr/superblock_free_set_fuzz.zig

@@ -0,0 +1,332 @@
+//! Fuzz FreeSet reserve/acquire/release flow.
+//!
+//! This fuzzer does *not* cover FreeSet encoding/decoding.
+const std = @import("std");
+const assert = std.debug.assert;
+const log = std.log.scoped(.fuzz_vsr_superblock_free_set);
+
+const FreeSet = @import("./superblock_free_set.zig").FreeSet;
+const Reservation = @import("./superblock_free_set.zig").Reservation;
+const fuzz = @import("../test/fuzz.zig");
+
+pub fn main() !void {
+    const allocator = std.testing.allocator;
+    const args = try fuzz.parse_fuzz_args(allocator);
+
+    var prng = std.rand.DefaultPrng.init(args.seed);
+
+    const blocks_count = FreeSet.shard_size * (1 + prng.random().uintLessThan(usize, 10));
+    const events = try generate_events(allocator, prng.random(), blocks_count);
+    defer allocator.free(events);
+
+    try run_fuzz(allocator, prng.random(), blocks_count, events);
+}
+
+fn run_fuzz(
+    allocator: std.mem.Allocator,
+    random: std.rand.Random,
+    blocks_count: usize,
+    events: []const FreeSetEvent,
+) !void {
+    var free_set = try FreeSet.init(allocator, blocks_count);
+    defer free_set.deinit(allocator);
+
+    var free_set_model = try FreeSetModel.init(allocator, blocks_count);
+    defer free_set_model.deinit(allocator);
+
+    var active_reservations = std.ArrayList(Reservation).init(allocator);
+    defer active_reservations.deinit();
+
+    var active_addresses = std.ArrayList(u64).init(allocator);
+    defer active_addresses.deinit();
+
+    for (events) |event| {
+        log.debug("event={}", .{event});
+        switch (event) {
+            .reserve => |reserve| {
+                const reservation_actual = free_set.reserve(reserve.blocks);
+                const reservation_expect = free_set_model.reserve(reserve.blocks);
+                assert(std.meta.eql(reservation_expect, reservation_actual));
+
+                if (reservation_expect) |reservation| {
+                    assert(reserve.blocks == free_set_model.count_free_reserved(reservation));
+                    assert(reserve.blocks == free_set.count_free_reserved(reservation));
+                    try active_reservations.append(reservation);
+                }
+            },
+            .forfeit => {
+                random.shuffle(Reservation, active_reservations.items);
+                for (active_reservations.items) |reservation| {
+                    free_set.forfeit(reservation);
+                    free_set_model.forfeit(reservation);
+                }
+                active_reservations.clearRetainingCapacity();
+            },
+            .acquire => |data| {
+                if (active_reservations.items.len == 0) continue;
+                const reservation = active_reservations.items[
+                    data.reservation % active_reservations.items.len
+                ];
+                const address_actual = free_set.acquire(reservation);
+                const address_expect = free_set_model.acquire(reservation);
+                assert(std.meta.eql(address_expect, address_actual));
+                if (address_expect) |address| {
+                    try active_addresses.append(address);
+                }
+            },
+            .release => |data| {
+                if (active_addresses.items.len == 0) continue;
+
+                const address_index = data.address % active_addresses.items.len;
+                const address = active_addresses.swapRemove(address_index);
+                free_set.release(address);
+                free_set_model.release(address);
+            },
+            .checkpoint => {
+                random.shuffle(Reservation, active_reservations.items);
+                for (active_reservations.items) |reservation| {
+                    free_set.forfeit(reservation);
+                    free_set_model.forfeit(reservation);
+                }
+                active_reservations.clearRetainingCapacity();
+
+                free_set.checkpoint();
+                free_set_model.checkpoint();
+            },
+        }
+
+        assert(free_set_model.count_reservations() == free_set.count_reservations());
+        assert(free_set_model.count_free() == free_set.count_free());
+        assert(free_set_model.count_acquired() == free_set.count_acquired());
+        assert(std.meta.eql(
+            free_set_model.highest_address_acquired(),
+            free_set.highest_address_acquired(),
+        ));
+
+        for (active_reservations.items) |reservation| {
+            assert(
+                free_set_model.count_free_reserved(reservation) ==
+                    free_set.count_free_reserved(reservation),
+            );
+        }
+    }
+}
+
+const FreeSetEventType = std.meta.Tag(FreeSetEvent);
+const FreeSetEvent = union(enum) {
+    reserve: struct { blocks: usize },
+    forfeit: void,
+    acquire: struct { reservation: usize },
+    release: struct { address: usize },
+    checkpoint: void,
+};
+
+fn generate_events(
+    allocator: std.mem.Allocator,
+    random: std.rand.Random,
+    blocks_count: usize,
+) ![]const FreeSetEvent {
+    const event_distribution = fuzz.Distribution(FreeSetEventType){
+        .reserve = 1 + random.float(f64) * 100,
+        .forfeit = 1,
+        .acquire = random.float(f64) * 1000,
+        .release = if (random.boolean()) 0 else 500 * random.float(f64),
+        .checkpoint = random.floatExp(f64) * 10,
+    };
+
+    const events = try allocator.alloc(FreeSetEvent, std.math.min(
+        @as(usize, 2_000_000),
+        fuzz.random_int_exponential(random, usize, blocks_count * 100),
+    ));
+    errdefer allocator.free(events);
+
+    log.info("event_distribution = {d:.2}", .{event_distribution});
+    log.info("event_count = {d}", .{events.len});
+
+    const reservation_blocks_mean = 1 + random.uintLessThan(usize, @divFloor(blocks_count, 20));
+    for (events) |*event| {
+        event.* = switch (fuzz.random_enum(random, FreeSetEventType, event_distribution)) {
+            .reserve => FreeSetEvent{ .reserve = .{
+                .blocks = 1 + fuzz.random_int_exponential(random, usize, reservation_blocks_mean),
+            } },
+            .forfeit => FreeSetEvent{ .forfeit = {} },
+            .acquire => FreeSetEvent{ .acquire = .{ .reservation = random.int(usize) } },
+            .release => FreeSetEvent{ .release = .{
+                .address = random.int(usize),
+            } },
+            .checkpoint => FreeSetEvent{ .checkpoint = {} },
+        };
+    }
+    return events;
+}
+
+const FreeSetModel = struct {
+    /// Set bits indicate acquired blocks.
+    blocks_acquired: std.DynamicBitSetUnmanaged,
+
+    /// Set bits indicate blocks that will be released at the next checkpoint.
+    blocks_released: std.DynamicBitSetUnmanaged,
+
+    /// Set bits indicate blocks that are currently reserved and not yet forfeited.
+    blocks_reserved: std.DynamicBitSetUnmanaged,
+
+    reservation_count: usize = 0,
+    reservation_session: usize = 1,
+
+    fn init(allocator: std.mem.Allocator, blocks_count: usize) !FreeSetModel {
+        var blocks_acquired = try std.DynamicBitSetUnmanaged.initEmpty(allocator, blocks_count);
+        errdefer blocks_acquired.deinit(allocator);
+
+        var blocks_released = try std.DynamicBitSetUnmanaged.initEmpty(allocator, blocks_count);
+        errdefer blocks_released.deinit(allocator);
+
+        var blocks_reserved = try std.DynamicBitSetUnmanaged.initEmpty(allocator, blocks_count);
+        errdefer blocks_reserved.deinit(allocator);
+
+        return FreeSetModel{
+            .blocks_acquired = blocks_acquired,
+            .blocks_released = blocks_released,
+            .blocks_reserved = blocks_reserved,
+        };
+    }
+
+    fn deinit(set: *FreeSetModel, allocator: std.mem.Allocator) void {
+        set.blocks_acquired.deinit(allocator);
+        set.blocks_released.deinit(allocator);
+        set.blocks_reserved.deinit(allocator);
+    }
+
+    pub fn count_reservations(set: FreeSetModel) usize {
+        return set.reservation_count;
+    }
+
+    pub fn count_free(set: FreeSetModel) usize {
+        return set.blocks_acquired.capacity() - set.blocks_acquired.count();
+    }
+
+    pub fn count_free_reserved(set: FreeSetModel, reservation: Reservation) usize {
+        assert(reservation.block_count > 0);
+        set.assert_reservation_active(reservation);
+
+        var count: usize = 0;
+        var iterator = set.blocks_acquired.iterator(.{ .kind = .unset });
+        while (iterator.next()) |block| {
+            if (block >= reservation.block_base and
+                block < reservation.block_base + reservation.block_count)
+            {
+                count += 1;
+            }
+        }
+        return count;
+    }
+
+    pub fn count_acquired(set: FreeSetModel) usize {
+        return set.blocks_acquired.count();
+    }
+
+    pub fn highest_address_acquired(set: FreeSetModel) ?u64 {
+        const block = set.blocks_acquired.iterator(.{
+            .direction = .reverse,
+        }).next() orelse return null;
+        return block + 1;
+    }
+
+    pub fn reserve(set: *FreeSetModel, reserve_count: usize) ?Reservation {
+        assert(reserve_count > 0);
+
+        var blocks_found_free: usize = 0;
+        var iterator = set.blocks_acquired.iterator(.{ .kind = .unset });
+        while (iterator.next()) |block| {
+            if (block < set.blocks_reserved.count()) {
+                assert(set.blocks_reserved.isSet(block));
+                continue;
+            }
+
+            blocks_found_free += 1;
+            if (blocks_found_free == reserve_count) {
+                const block_base = set.blocks_reserved.count();
+                const block_count = block + 1 - block_base;
+
+                var i: usize = 0;
+                while (i < block_count) : (i += 1) set.blocks_reserved.set(block_base + i);
+
+                set.reservation_count += 1;
+                return Reservation{
+                    .block_base = block_base,
+                    .block_count = block_count,
+                    .session = set.reservation_session,
+                };
+            }
+        }
+        return null;
+    }
+
+    pub fn forfeit(set: *FreeSetModel, reservation: Reservation) void {
+        set.assert_reservation_active(reservation);
+        set.reservation_count -= 1;
+
+        var i: usize = 0;
+        while (i < reservation.block_count) : (i += 1) {
+            set.blocks_reserved.unset(reservation.block_base + i);
+        }
+
+        if (set.reservation_count == 0) {
+            set.reservation_session +%= 1;
+            assert(set.blocks_reserved.count() == 0);
+        }
+    }
+
+    pub fn acquire(set: *FreeSetModel, reservation: Reservation) ?u64 {
+        assert(reservation.block_count > 0);
+        assert(reservation.block_base < set.blocks_acquired.capacity());
+        assert(reservation.session == set.reservation_session);
+        set.assert_reservation_active(reservation);
+
+        var iterator = set.blocks_acquired.iterator(.{ .kind = .unset });
+        while (iterator.next()) |block| {
+            if (block >= reservation.block_base and
+                block < reservation.block_base + reservation.block_count)
+            {
+                assert(!set.blocks_acquired.isSet(block));
+                set.blocks_acquired.set(block);
+
+                const address = block + 1;
+                return address;
+            }
+        }
+        return null;
+    }
+
+    pub fn is_free(set: *FreeSetModel, address: u64) bool {
+        return !set.blocks_acquired.isSet(address - 1);
+    }
+
+    pub fn release(set: *FreeSetModel, address: u64) void {
+        const block = address - 1;
+        set.blocks_released.set(block);
+    }
+
+    pub fn checkpoint(set: *FreeSetModel) void {
+        assert(set.blocks_reserved.count() == 0);
+
+        var iterator = set.blocks_released.iterator(.{});
+        while (iterator.next()) |block| {
+            assert(set.blocks_released.isSet(block));
+            assert(set.blocks_acquired.isSet(block));
+
+            set.blocks_released.unset(block);
+            set.blocks_acquired.unset(block);
+        }
+        assert(set.blocks_released.count() == 0);
+    }
+
+    fn assert_reservation_active(set: FreeSetModel, reservation: Reservation) void {
+        assert(set.reservation_count > 0);
+        assert(set.reservation_session == reservation.session);
+
+        var i: usize = 0;
+        while (i < reservation.block_count) : (i += 1) {
+            assert(set.blocks_reserved.isSet(reservation.block_base + i));
+        }
+    }
+};

package/src/tigerbeetle/src/vsr/superblock_fuzz.zig

@@ -0,0 +1,349 @@
+//! Fuzz SuperBlock open()/checkpoint()/view_change().
+//!
+//! Invariants checked:
+//!
+//! - Crashing during a checkpoint() or view_change().
+//!   - open() finds a quorum, even with the interference of disk faults.
+//!   - open()'s quorum never regresses.
+//! - Calling checkpoint() and view_change() concurrently is safe.
+//!   - VSRState will not leak before the corresponding checkpoint()/view_change().
+//!   - Trailers will not leak before the corresponding checkpoint().
+//! - view_change_in_progress() reports the correct state.
+//!
+const std = @import("std");
+const assert = std.debug.assert;
+const log = std.log.scoped(.fuzz_vsr_superblock);
+
+const config = @import("../config.zig");
+const util = @import("../util.zig");
+const vsr = @import("../vsr.zig");
+const Storage = @import("../test/storage.zig").Storage;
+const MessagePool = @import("../message_pool.zig").MessagePool;
+const superblock_zone_size = @import("superblock.zig").superblock_zone_size;
+const data_file_size_min = @import("superblock.zig").data_file_size_min;
+const VSRState = @import("superblock.zig").SuperBlockSector.VSRState;
+const SuperBlockType = @import("superblock.zig").SuperBlockType;
+const SuperBlock = SuperBlockType(Storage);
+const fuzz = @import("../test/fuzz.zig");
+
+/// Total calls to checkpoint() + view_change().
+const transitions_count_total = 10;
+const cluster = 0;
+
+pub fn main() !void {
+    const allocator = std.testing.allocator;
+    const args = try fuzz.parse_fuzz_args(allocator);
+
+    try run_fuzz(allocator, args.seed);
+}
+
+fn run_fuzz(allocator: std.mem.Allocator, seed: u64) !void {
+    var prng = std.rand.DefaultPrng.init(seed);
+    const random = prng.random();
+
+    const storage_options = .{
+        .seed = random.int(u64),
+        // SuperBlock's IO is all serial, so latencies never reorder reads/writes.
+        .read_latency_min = 1,
+        .read_latency_mean = 1,
+        .write_latency_min = 1,
+        .write_latency_mean = 1,
+        // Storage will never inject more faults than the superblock is able to recover from,
+        // so a 100% fault probability is allowed.
+        .read_fault_probability = 25 + random.uintLessThan(u8, 76),
+        .write_fault_probability = 25 + random.uintLessThan(u8, 76),
+        .crash_fault_probability = 50 + random.uintLessThan(u8, 51),
+        .faulty_superblock = true,
+    };
+
+    var storage = try Storage.init(allocator, superblock_zone_size, storage_options);
+    defer storage.deinit(allocator);
+
+    var storage_verify = try Storage.init(allocator, superblock_zone_size, storage_options);
+    defer storage_verify.deinit(allocator);
+
+    var message_pool = try MessagePool.init(allocator, .replica);
+    defer message_pool.deinit(allocator);
+
+    var superblock = try SuperBlock.init(allocator, &storage, &message_pool);
+    defer superblock.deinit(allocator);
+
+    var superblock_verify = try SuperBlock.init(allocator, &storage_verify, &message_pool);
+    defer superblock_verify.deinit(allocator);
+
+    var sequence_states = Environment.SequenceStates.init(allocator);
+    defer sequence_states.deinit();
+
+    var env = Environment{
+        .sequence_states = sequence_states,
+        .superblock = &superblock,
+        .superblock_verify = &superblock_verify,
+    };
+
+    try env.format();
+    while (env.pending.count() > 0) env.superblock.storage.tick();
+
+    env.open();
+    while (env.pending.count() > 0) env.superblock.storage.tick();
+
+    try env.verify();
+    assert(env.pending.count() == 0);
+    assert(env.latest_sequence == 1);
+
+    var transitions: usize = 0;
+    while (transitions < transitions_count_total or env.pending.count() > 0) {
+        if (transitions < transitions_count_total) {
+            // TODO bias the RNG
+            if (env.pending.count() == 0) {
+                transitions += 1;
+                if (random.boolean()) {
+                    try env.checkpoint();
+                } else {
+                    try env.view_change();
+                }
+            }
+
+            if (env.pending.count() == 1 and random.uintLessThan(u8, 6) == 0) {
+                transitions += 1;
+                if (env.pending.contains(.view_change)) {
+                    try env.checkpoint();
+                } else {
+                    try env.view_change();
+                }
+            }
+        }
+
+        assert(env.pending.count() > 0);
+        assert(env.pending.count() <= 2);
+        try env.tick();
+
+        // Trailers are only updated on-disk by checkpoint(), never view_change().
+        // Trailers must not be mutated while a checkpoint() is in progress.
+        if (!env.pending.contains(.checkpoint) and random.boolean()) {
+            const range = env.superblock.free_set.reserve(1).?;
+            _ = env.superblock.free_set.acquire(range).?;
+            env.superblock.free_set.forfeit(range);
+        }
+    }
+}
+
+const Environment = struct {
+    /// Track the expected value of parameters at a particular sequence.
+    /// Indexed by sequence.
+    const SequenceStates = std.ArrayList(struct {
+        vsr_state: VSRState,
+        /// Track the expected `checksum(free_set)`.
+        /// Note that this is a checksum of the decoded free set; it is not the same as
+        /// `SuperBlockSector.free_set_checksum`.
+        free_set: u128,
+    });
+
+    sequence_states: SequenceStates,
+
+    superblock: *SuperBlock,
+    superblock_verify: *SuperBlock,
+
+    /// Verify that the working superblock after open() never regresses.
+    latest_sequence: u64 = 0,
+    latest_checksum: u128 = 0,
+    latest_parent: u128 = 0,
+    latest_vsr_state: VSRState = std.mem.zeroInit(VSRState, .{}),
+
+    context_format: SuperBlock.Context = undefined,
+    context_open: SuperBlock.Context = undefined,
+    context_checkpoint: SuperBlock.Context = undefined,
+    context_view_change: SuperBlock.Context = undefined,
+    context_verify: SuperBlock.Context = undefined,
+
+    // Set bits indicate pending operations.
+    pending: std.enums.EnumSet(SuperBlock.Context.Caller) = .{},
+    pending_verify: bool = false,
+
+    /// After every write to `superblock`'s storage, verify that the superblock can be opened,
+    /// and the quorum never regresses.
+    fn tick(env: *Environment) !void {
+        assert(env.pending.count() <= 2);
+        assert(env.superblock.storage.reads.len + env.superblock.storage.writes.len <= 1);
+        assert(!env.pending.contains(.format));
+        assert(!env.pending.contains(.open));
+        assert(!env.pending_verify);
+        assert(env.pending.contains(.view_change) == env.superblock.view_change_in_progress());
+
+        const write = env.superblock.storage.writes.peek();
+        env.superblock.storage.tick();
+
+        if (write) |w| {
+            if (w.done_at_tick <= env.superblock.storage.ticks) try env.verify();
+        }
+    }
+
+    /// Verify that the superblock will recover safely if the replica crashes immediately after
+    /// the most recent write.
+    fn verify(env: *Environment) !void {
+        assert(!env.pending_verify);
+
+        {
+            // Reset `superblock_verify` so that it can be reused.
+            env.superblock_verify.opened = false;
+            var free_set_iterator = env.superblock_verify.free_set.blocks.iterator(.{
+                .kind = .unset,
+            });
+            while (free_set_iterator.next()) |block_bit| {
+                const block_address = block_bit + 1;
+                env.superblock_verify.free_set.release(block_address);
+            }
+            env.superblock_verify.free_set.checkpoint();
+        }
+
+        // Duplicate the `superblock`'s storage so it is not modified by `superblock_verify`'s
+        // repairs. Immediately reset() it to simulate a crash (potentially injecting additional
+        // faults for pending writes) and clear the read/write queues.
+        env.superblock_verify.storage.copy(env.superblock.storage);
+        env.superblock_verify.storage.reset();
+        env.superblock_verify.open(verify_callback, &env.context_verify);
+
+        env.pending_verify = true;
+        while (env.pending_verify) env.superblock_verify.storage.tick();
+
+        assert(env.superblock_verify.working.checksum == env.superblock.working.checksum or
+            env.superblock_verify.working.checksum == env.superblock.staging.checksum);
+
+        // Verify the sequence we read from disk is monotonically increasing.
+        if (env.latest_sequence < env.superblock_verify.working.sequence) {
+            assert(env.latest_sequence + 1 == env.superblock_verify.working.sequence);
+
+            if (env.latest_checksum != 0) {
+                if (env.latest_sequence + 1 == env.superblock_verify.working.sequence) {
+                    // After a checkpoint() or view_change(), the parent points to the previous
+                    // working sector.
+                    assert(env.superblock_verify.working.parent == env.latest_checksum);
+                }
+            }
+
+            assert(env.latest_vsr_state.monotonic(env.superblock_verify.working.vsr_state));
+
+            const expect = env.sequence_states.items[env.superblock_verify.working.sequence];
+            assert(std.meta.eql(expect.vsr_state, env.superblock_verify.working.vsr_state));
+            assert(expect.free_set == checksum_free_set(env.superblock_verify));
+
+            env.latest_sequence = env.superblock_verify.working.sequence;
+            env.latest_checksum = env.superblock_verify.working.checksum;
+            env.latest_parent = env.superblock_verify.working.parent;
+            env.latest_vsr_state = env.superblock_verify.working.vsr_state;
+        } else {
+            assert(env.latest_sequence == env.superblock_verify.working.sequence);
+            assert(env.latest_checksum == env.superblock_verify.working.checksum);
+            assert(env.latest_parent == env.superblock_verify.working.parent);
+        }
+    }
+
+    fn verify_callback(context: *SuperBlock.Context) void {
+        const env = @fieldParentPtr(Environment, "context_verify", context);
+        assert(env.pending_verify);
+        env.pending_verify = false;
+    }
+
+    fn format(env: *Environment) !void {
+        assert(env.pending.count() == 0);
+        env.pending.insert(.format);
+        env.superblock.format(format_callback, &env.context_format, .{
+            .cluster = cluster,
+            .replica = 0,
+            // Include extra space (beyond what Storage actually needs) because SuperBlock will
+            // update the sector's size according to the highest FreeSet block allocated.
+            .size_max = data_file_size_min + 1000 * config.block_size,
+        });
+
+        assert(env.sequence_states.items.len == 0);
+        try env.sequence_states.append(undefined); // skip sequence=0
+        try env.sequence_states.append(.{
+            .vsr_state = VSRState.root(cluster),
+            .free_set = checksum_free_set(env.superblock),
+        });
+    }
+
+    fn format_callback(context: *SuperBlock.Context) void {
+        const env = @fieldParentPtr(Environment, "context_format", context);
+        assert(env.pending.contains(.format));
+        env.pending.remove(.format);
+    }
+
+    fn open(env: *Environment) void {
+        assert(env.pending.count() == 0);
+        env.pending.insert(.open);
+        env.superblock.open(open_callback, &env.context_open);
+    }
+
+    fn open_callback(context: *SuperBlock.Context) void {
+        const env = @fieldParentPtr(Environment, "context_open", context);
+        assert(env.pending.contains(.open));
+        env.pending.remove(.open);
+
+        assert(env.superblock.working.sequence == 1);
+        assert(env.superblock.working.replica == 0);
+        assert(env.superblock.working.cluster == cluster);
+    }
+
+    fn view_change(env: *Environment) !void {
+        assert(!env.pending.contains(.view_change));
+        assert(env.pending.count() < 2);
+
+        const vsr_state = .{
+            .commit_min_checksum = env.superblock.staging.vsr_state.commit_min_checksum,
+            .commit_min = env.superblock.staging.vsr_state.commit_min,
+            .commit_max = env.superblock.staging.vsr_state.commit_max + 3,
+            .view_normal = env.superblock.staging.vsr_state.view_normal + 4,
+            .view = env.superblock.staging.vsr_state.view + 5,
+        };
+
+        assert(env.sequence_states.items.len == env.superblock.staging.sequence + 1);
+        try env.sequence_states.append(.{
+            .vsr_state = vsr_state,
+            .free_set = env.sequence_states.items[env.sequence_states.items.len - 1].free_set,
+        });
+
+        env.pending.insert(.view_change);
+        env.superblock.view_change(view_change_callback, &env.context_view_change, vsr_state);
+    }
+
+    fn view_change_callback(context: *SuperBlock.Context) void {
+        const env = @fieldParentPtr(Environment, "context_view_change", context);
+        assert(env.pending.contains(.view_change));
+        env.pending.remove(.view_change);
+    }
+
+    fn checkpoint(env: *Environment) !void {
+        assert(!env.pending.contains(.checkpoint));
+        assert(env.pending.count() < 2);
+
+        const vsr_state = .{
+            .commit_min_checksum = env.superblock.staging.vsr_state.commit_min_checksum + 1,
+            .commit_min = env.superblock.staging.vsr_state.commit_min + 1,
+            .commit_max = env.superblock.staging.vsr_state.commit_max + 1,
+            .view_normal = env.superblock.staging.vsr_state.view_normal + 1,
+            .view = env.superblock.staging.vsr_state.view + 1,
+        };
+
+        assert(env.sequence_states.items.len == env.superblock.staging.sequence + 1);
+        try env.sequence_states.append(.{
+            .vsr_state = vsr_state,
+            .free_set = checksum_free_set(env.superblock),
+        });
+
+        env.pending.insert(.checkpoint);
+        env.superblock.checkpoint(checkpoint_callback, &env.context_checkpoint, vsr_state);
+    }
+
+    fn checkpoint_callback(context: *SuperBlock.Context) void {
+        const env = @fieldParentPtr(Environment, "context_checkpoint", context);
+        assert(env.pending.contains(.checkpoint));
+        env.pending.remove(.checkpoint);
+    }
+};
+
+fn checksum_free_set(superblock: *const SuperBlock) u128 {
+    const mask_bits = @bitSizeOf(std.DynamicBitSetUnmanaged.MaskInt);
+    const count_bits = superblock.free_set.blocks.bit_length;
+    const count_words = util.div_ceil(count_bits, mask_bits);
+    return vsr.checksum(std.mem.sliceAsBytes(superblock.free_set.blocks.masks[0..count_words]));
+}