tigerbeetle-node 0.11.6 → 0.11.7

package/src/tigerbeetle/src/lsm/level_iterator.zig

@@ -68,6 +68,9 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
         values: ValuesRingBuffer,
         tables: TablesRingBuffer,
 
+        // Used for verifying key order when constants.verify == true.
+        key_prev: ?Key,
+
         pub fn init(allocator: mem.Allocator) !LevelIterator {
             var values = try ValuesRingBuffer.init(allocator);
             errdefer values.deinit(allocator);
@@ -96,6 +99,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
                         .{ .table_iterator = table_b },
                     },
                 },
+                .key_prev = null,
             };
         }
 
@@ -138,6 +142,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
                 .direction = context.direction,
                 .values = .{ .buffer = it.values.buffer },
                 .tables = .{ .buffer = it.tables.buffer },
+                .key_prev = null,
             };
 
             assert(it.key_exclusive == null);
@@ -281,7 +286,7 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
         /// Returns either:
         /// - the next Key, if available.
         /// - error.Empty when there are no values remaining to iterate.
-        /// - error.Drained when the iterator isn't empty, but the values 
+        /// - error.Drained when the iterator isn't empty, but the values
         ///   still need to be buffered into memory via tick().
         pub fn peek(it: LevelIterator) error{ Empty, Drained }!Key {
             if (it.values.head_ptr_const()) |value| return key_from_value(value);
@@ -299,6 +304,18 @@ pub fn LevelIteratorType(comptime Table: type, comptime Storage: type) type {
 
         /// This may only be called after peek() returns a Key (and not Empty or Drained)
         pub fn pop(it: *LevelIterator) Value {
+            const value = it.pop_internal();
+
+            if (constants.verify) {
+                const key = Table.key_from_value(&value);
+                if (it.key_prev) |k| assert(Table.compare_keys(k, key) == .lt);
+                it.key_prev = key;
+            }
+
+            return value;
+        }
+
+        fn pop_internal(it: *LevelIterator) Value {
             if (it.values.pop()) |value| return value;
 
             const table_iterator = &it.tables.head_ptr().?.table_iterator;

package/src/tigerbeetle/src/lsm/manifest.zig

@@ -531,7 +531,7 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
                     .ascending,
                     KeyRange{ .key_min = range.key_min, .key_max = range.key_max },
                 );
-                if (it.next() == null) {
+                if (it.next() != null) {
                     // If the range is being compacted into the last level then this is unreachable,
                     // as the last level has no subsequent levels and must always drop tombstones.
                     assert(level_b != constants.lsm_levels - 1);
@@ -585,5 +585,33 @@ pub fn ManifestType(comptime Table: type, comptime Storage: type) type {
             manifest.checkpoint_callback = null;
             callback(manifest);
         }
+
+        pub fn verify(manifest: *Manifest, snapshot: u64) void {
+            for (manifest.levels) |*level| {
+                var key_max_prev: ?Key = null;
+                var table_info_iter = level.iterator(
+                    .visible,
+                    &.{snapshot},
+                    .ascending,
+                    null,
+                );
+                while (table_info_iter.next()) |table_info| {
+                    if (key_max_prev) |k| {
+                        assert(compare_keys(k, table_info.key_min) == .lt);
+                    }
+                    // We could have key_min == key_max if there is only one value.
+                    assert(compare_keys(table_info.key_min, table_info.key_max) != .gt);
+                    key_max_prev = table_info.key_max;
+
+                    Table.verify(
+                        Storage,
+                        manifest.manifest_log.grid.superblock.storage,
+                        table_info.address,
+                        table_info.key_min,
+                        table_info.key_max,
+                    );
+                }
+            }
+        }
     };
 }

package/src/tigerbeetle/src/lsm/manifest_log.zig

@@ -404,14 +404,14 @@ pub fn ManifestLogType(comptime Storage: type, comptime TableInfo: type) type {
                 return;
             }
 
-            const block = manifest_log.blocks.head().?;
-            verify_block(block, null, null);
+            const block = manifest_log.blocks.head_ptr().?;
+            verify_block(block.*, null, null);
 
-            const header = mem.bytesAsValue(vsr.Header, block[0..@sizeOf(vsr.Header)]);
-            const address = Block.address(block);
+            const header = mem.bytesAsValue(vsr.Header, block.*[0..@sizeOf(vsr.Header)]);
+            const address = Block.address(block.*);
             assert(address > 0);
 
-            const entry_count = Block.entry_count(block);
+            const entry_count = Block.entry_count(block.*);
 
             if (manifest_log.blocks_closed == 1 and manifest_log.blocks.count == 1) {
                 // This might be the last block of a checkpoint, which can be a partial block.

package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig

@@ -319,7 +319,7 @@ const Environment = struct {
 
     fn wait(env: *Environment, manifest_log: *ManifestLog) void {
         while (env.pending > 0) {
-            manifest_log.grid.tick();
+            // manifest_log.grid.tick();
             manifest_log.superblock.storage.tick();
         }
     }
@@ -635,7 +635,7 @@ fn verify_manifest_compaction_set(
     var compact_blocks_checked: u32 = 0;
 
     // This test doesn't include any actual table blocks, so all blocks are manifest blocks.
-    var blocks = superblock.free_set.blocks.iterator(.{ .kind = .unset });
+    var blocks = superblock.free_set.blocks.iterator(.{ .kind = .set });
     while (blocks.next()) |block_index| {
         const block_address = block_index + 1;
         const block = superblock.storage.grid_block(block_address);

package/src/tigerbeetle/src/lsm/set_associative_cache.zig

@@ -188,21 +188,26 @@ pub fn SetAssociativeCache(
             mem.set(u64, self.clocks.words, 0);
         }
 
-        /// Returns whether an entry with the given key is cached, 
+        /// Returns whether an entry with the given key is cached,
         /// without modifying the entry's counter.
         pub fn exists(self: *Self, key: Key) bool {
             const set = self.associate(key);
             return self.search(set, key) != null;
         }
 
-        pub fn get(self: *Self, key: Key) ?*align(value_alignment) Value {
+        pub fn get_index(self: *Self, key: Key) ?usize {
             const set = self.associate(key);
             const way = self.search(set, key) orelse return null;
 
             const count = self.counts.get(set.offset + way);
             self.counts.set(set.offset + way, count +| 1);
 
-            return @alignCast(value_alignment, &set.values[way]);
+            return set.offset + way;
+        }
+
+        pub fn get(self: *Self, key: Key) ?*align(value_alignment) Value {
+            const index = self.get_index(key) orelse return null;
+            return @alignCast(value_alignment, &self.values[index]);
         }
 
         /// Remove a key from the set associative cache if present.
@@ -211,6 +216,7 @@ pub fn SetAssociativeCache(
             const way = self.search(set, key) orelse return;
 
             self.counts.set(set.offset + way, 0);
+            set.values[way] = undefined;
         }
 
         /// Hint that the key is less likely to be accessed in the future, without actually removing
@@ -248,39 +254,21 @@ pub fn SetAssociativeCache(
             return @ptrCast(*const Ways, &result).*;
         }
 
-        pub fn insert(self: *Self, key: Key) *align(value_alignment) Value {
-            return self.insert_preserve_locked(
-                void,
-                struct {
-                    inline fn locked(_: void, _: *const Value) bool {
-                        return false;
-                    }
-                }.locked,
-                {},
-                key,
-            );
+        /// Insert a value, evicting an older entry if needed.
+        pub fn insert(self: *Self, value: *const Value) void {
+            _ = self.insert_index(value);
         }
 
-        /// Add a key, evicting an older entry if needed, and return a pointer to the value.
-        /// The key must not already be in the cache.
-        /// Never evicts keys for which locked() returns true.
-        /// The caller must guarantee that locked() returns true for less than layout.ways keys.
-        pub fn insert_preserve_locked(
-            self: *Self,
-            comptime Context: type,
-            comptime locked: fn (
-                Context,
-                *align(value_alignment) const Value,
-            ) callconv(.Inline) bool,
-            context: Context,
-            key: Key,
-        ) *align(value_alignment) Value {
+        /// Insert a value, evicting an older entry if needed.
+        /// Return the index at which the value was inserted.
+        pub fn insert_index(self: *Self, value: *const Value) usize {
+            const key = key_from_value(value);
             const set = self.associate(key);
             if (self.search(set, key)) |way| {
-                // Remove the old entry for this key.
-                // It should be a different value, but since we are returning a value pointer we
-                // can't check against the new one.
-                self.counts.set(set.offset + way, 0);
+                // Overwrite the old entry for this key.
+                self.counts.set(set.offset + way, 1);
+                set.values[way] = value.*;
+                return set.offset + way;
             }
 
             const clock_index = @divExact(set.offset, layout.ways);
@@ -299,11 +287,6 @@ pub fn SetAssociativeCache(
                 safety_count += 1;
                 way +%= 1;
             }) {
-                // We pass a value pointer to the callback here so that a cache miss
-                // can be avoided if the caller is able to determine if the value is
-                // locked by comparing pointers directly.
-                if (locked(context, @alignCast(value_alignment, &set.values[way]))) continue;
-
                 var count = self.counts.get(set.offset + way);
                 if (count == 0) break; // Way is already free.
 
@@ -316,10 +299,11 @@ pub fn SetAssociativeCache(
             assert(self.counts.get(set.offset + way) == 0);
 
             set.tags[way] = set.tag;
+            set.values[way] = value.*;
             self.counts.set(set.offset + way, 1);
             self.clocks.set(clock_index, way +% 1);
 
-            return @alignCast(value_alignment, &set.values[way]);
+            return set.offset + way;
         }
 
         const Set = struct {
@@ -430,7 +414,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.insert(key).* = key;
+                    sac.insert(&key);
                     try expect(sac.counts.get(i) == 1);
                     try expectEqual(key, sac.get(key).?.*);
                     try expect(sac.counts.get(i) == 2);
@@ -443,7 +427,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.insert(key).* = key;
+                sac.insert(&key);
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);
@@ -460,34 +444,6 @@ fn set_associative_cache_test(
 
             if (log) sac.associate(0).inspect(sac);
 
-            // Lock all other slots, causing key layout.ways * sac.sets to be evicted despite having the
-            // highest count.
-            {
-                {
-                    assert(sac.counts.get(0) == 2);
-                    var i: usize = 1;
-                    while (i < layout.ways) : (i += 1) assert(sac.counts.get(i) == 1);
-                }
-
-                const key = (layout.ways + 1) * sac.sets;
-                const expect_evicted = layout.ways * sac.sets;
-
-                sac.insert_preserve_locked(
-                    u64,
-                    struct {
-                        inline fn locked(only_unlocked: u64, value: *const Value) bool {
-                            return value.* != only_unlocked;
-                        }
-                    }.locked,
-                    expect_evicted,
-                    key,
-                ).* = key;
-
-                try expectEqual(@as(?*Value, null), sac.get(expect_evicted));
-            }
-
-            if (log) sac.associate(0).inspect(sac);
-
             // Ensure removal works.
             {
                 const key = 5 * sac.sets;
@@ -512,7 +468,7 @@ fn set_associative_cache_test(
                     try expectEqual(i, sac.clocks.get(0));
 
                     const key = i * sac.sets;
-                    sac.insert(key).* = key;
+                    sac.insert(&key);
                     try expect(sac.counts.get(i) == 1);
                     var j: usize = 2;
                     while (j <= math.maxInt(SAC.Count)) : (j += 1) {
@@ -530,7 +486,7 @@ fn set_associative_cache_test(
             // Insert another element into the first set, causing key 0 to be evicted.
             {
                 const key = layout.ways * sac.sets;
-                sac.insert(key).* = key;
+                sac.insert(&key);
                 try expect(sac.counts.get(0) == 1);
                 try expectEqual(key, sac.get(key).?.*);
                 try expect(sac.counts.get(0) == 2);

package/src/tigerbeetle/src/lsm/table.zig

@@ -629,6 +629,11 @@ pub fn TableType(
                     assert(compare_keys(builder.key_min, builder.key_max) == .lt);
                 }
 
+                if (current > 0) {
+                    const key_max_prev = index_data_keys(builder.index_block)[current - 1];
+                    assert(compare_keys(key_max_prev, key_from_value(&values[0])) == .lt);
+                }
+
                 builder.data_block_count += 1;
                 builder.value = 0;
 
@@ -979,6 +984,43 @@ pub fn TableType(
 
             return null;
         }
+
+        pub fn verify(
+            comptime Storage: type,
+            storage: *Storage,
+            index_address: u64,
+            key_min: ?Key,
+            key_max: ?Key,
+        ) void {
+            if (Storage != @import("../test/storage.zig").Storage)
+                // Too complicated to do async verification
+                return;
+
+            const index_block = storage.grid_block(index_address);
+            const addresses = index_data_addresses(index_block);
+            const data_blocks_used = index_data_blocks_used(index_block);
+            var data_block_index: usize = 0;
+            while (data_block_index < data_blocks_used) : (data_block_index += 1) {
+                const address = addresses[data_block_index];
+                const data_block = storage.grid_block(address);
+                const values = data_block_values_used(data_block);
+                if (values.len > 0) {
+                    if (data_block_index == 0) {
+                        assert(key_min == null or
+                            compare_keys(key_min.?, key_from_value(&values[0])) == .eq);
+                    }
+                    if (data_block_index == data_blocks_used - 1) {
+                        assert(key_max == null or
+                            compare_keys(key_from_value(&values[values.len - 1]), key_max.?) == .eq);
+                    }
+                    var a = &values[0];
+                    for (values[1..]) |*b| {
+                        assert(compare_keys(key_from_value(a), key_from_value(b)) == .lt);
+                        a = b;
+                    }
+                }
+            }
+        }
     };
 }
 

package/src/tigerbeetle/src/lsm/table_iterator.zig

@@ -52,6 +52,9 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
         /// This field is only used for safety checks, it does not affect the behavior.
         read_pending: bool = false,
 
+        // Used for verifying key order when constants.verify == true.
+        key_prev: ?Table.Key,
+
         pub fn init(allocator: mem.Allocator) !TableIterator {
             const index_block = try allocator.alignedAlloc(
                 u8,
@@ -95,6 +98,7 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                     },
                 },
                 .value = undefined,
+                .key_prev = null,
             };
         }
 
@@ -132,10 +136,21 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
                 .values = .{ .buffer = it.values.buffer },
                 .data_blocks = .{ .buffer = it.data_blocks.buffer },
                 .value = 0,
+                .key_prev = null,
             };
 
             assert(it.values.empty());
             assert(it.data_blocks.empty());
+
+            if (constants.verify) {
+                Table.verify(
+                    Storage,
+                    context.grid.superblock.storage,
+                    context.address,
+                    null,
+                    null,
+                );
+            }
         }
 
         /// Try to buffer at least a full block of values to be peek()'d.
@@ -291,6 +306,18 @@ pub fn TableIteratorType(comptime Table: type, comptime Storage: type) type {
 
         /// This may only be called after peek() returns a Key (and not Empty or Drained)
         pub fn pop(it: *TableIterator) Table.Value {
+            const value = it.pop_internal();
+
+            if (constants.verify) {
+                const key = Table.key_from_value(&value);
+                if (it.key_prev) |k| assert(Table.compare_keys(k, key) == .lt);
+                it.key_prev = key;
+            }
+
+            return value;
+        }
+
+        fn pop_internal(it: *TableIterator) Table.Value {
             assert(!it.read_pending);
             assert(!it.read_table_index);
 

package/src/tigerbeetle/src/lsm/table_mutable.zig

@@ -186,7 +186,7 @@ pub fn TableMutableType(comptime Table: type) type {
                     if (tombstone(value)) {
                         cache.remove(key_from_value(value));
                     } else {
-                        cache.insert(key_from_value(value)).* = value.*;
+                        cache.insert(value);
                     }
                 }
             }

package/src/tigerbeetle/src/lsm/test.zig

@@ -35,8 +35,7 @@ const Environment = struct {
     const node_count = 1024;
     const cache_entries_max = 2 * 1024 * 1024;
     const forest_options = StateMachine.forest_options(.{
-        // Ignored by StateMachine.forest_options().
-        .lsm_forest_node_count = undefined,
+        .lsm_forest_node_count = node_count,
         .cache_entries_accounts = cache_entries_max,
         .cache_entries_transfers = cache_entries_max,
         .cache_entries_posted = cache_entries_max,
@@ -121,7 +120,7 @@ const Environment = struct {
     }
 
     fn tick(env: *Environment) !void {
-        env.grid.tick();
+        // env.grid.tick();
         try env.io.tick();
     }
 

package/src/tigerbeetle/src/lsm/tree.zig

@@ -212,13 +212,26 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             var manifest = try Manifest.init(allocator, node_pool, grid, tree_hash);
             errdefer manifest.deinit(allocator);
 
-            var compaction_table_immutable = try CompactionTableImmutable.init(allocator, tree_name);
+            var compaction_table_immutable = try CompactionTableImmutable.init(
+                allocator,
+                std.fmt.comptimePrint("{s}(immutable->0)", .{tree_name}),
+            );
             errdefer compaction_table_immutable.deinit(allocator);
 
             var compaction_table: [@divFloor(constants.lsm_levels, 2)]CompactionTable = undefined;
-            for (compaction_table) |*compaction, i| {
-                errdefer for (compaction_table[0..i]) |*c| c.deinit(allocator);
-                compaction.* = try CompactionTable.init(allocator, tree_name);
+            {
+                comptime var i: usize = 0;
+                inline while (i < compaction_table.len) : (i += 1) {
+                    errdefer for (compaction_table[0..i]) |*c| c.deinit(allocator);
+                    const compaction_name = std.fmt.comptimePrint("{s}({}->{}/{}->{})", .{
+                        tree_name,
+                        2 * i,
+                        2 * i + 1,
+                        2 * i + 1,
+                        2 * i + 2,
+                    });
+                    compaction_table[i] = try CompactionTable.init(allocator, compaction_name);
+                }
             }
             errdefer for (compaction_table) |*c| c.deinit(allocator);
 
@@ -580,10 +593,14 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             assert(tree.compaction_io_pending == 0);
             assert(tree.compaction_callback == null);
 
+            if (constants.verify) {
+                tree.manifest.verify(tree.compaction_op);
+            }
+
             tracer.start(
                 &tree.tracer_slot,
                 .{ .tree = .{ .tree_name = tree_name } },
-                .{ .tree_compaction_beat = .{ .tree_name = tree_name } },
+                .tree_compaction_beat,
                 @src(),
             );
 
@@ -959,9 +976,13 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type, comptime tree_
             tracer.end(
                 &tree.tracer_slot,
                 .{ .tree = .{ .tree_name = tree_name } },
-                .{ .tree_compaction_beat = .{ .tree_name = tree_name } },
+                .tree_compaction_beat,
             );
 
+            if (constants.verify) {
+                tree.manifest.verify(tree.compaction_op);
+            }
+
             // Invoke the compact() callback after the manifest compacts at the end of the beat.
             const callback = tree.compaction_callback.?;
             tree.compaction_callback = null;

package/src/tigerbeetle/src/lsm/tree_fuzz.zig

@@ -181,7 +181,7 @@ fn EnvironmentType(comptime table_usage: TableUsage) type {
         }
 
         fn tick(env: *Environment) void {
-            env.grid.tick();
+            // env.grid.tick();
             env.storage.tick();
         }
 

package/src/tigerbeetle/src/simulator.zig

@@ -269,11 +269,6 @@ pub fn main() !void {
         }
     };
 
-    // Disable most faults at startup, so that the replicas don't get stuck in recovery mode.
-    for (cluster.storages) |*storage, i| {
-        storage.faulty = replica_normal_min <= i;
-    }
-
     var tick: u64 = 0;
     while (tick < ticks_max) : (tick += 1) {
         const health_options = &cluster.options.health_options;

package/src/tigerbeetle/src/storage.zig

@@ -5,8 +5,8 @@ const assert = std.debug.assert;
 const log = std.log.scoped(.storage);
 
 const IO = @import("io.zig").IO;
+const FIFO = @import("fifo.zig").FIFO;
 const constants = @import("constants.zig");
-const fatal = @import("cli.zig").fatal;
 const vsr = @import("vsr.zig");
 
 pub const Storage = struct {
@@ -68,9 +68,17 @@ pub const Storage = struct {
         offset: u64,
     };
 
+    pub const NextTick = struct {
+        next: ?*NextTick = null,
+        callback: fn (next_tick: *NextTick) void,
+    };
+
     io: *IO,
     fd: os.fd_t,
 
+    next_tick_queue: FIFO(NextTick) = .{},
+    next_tick_completion: IO.Completion = undefined,
+
     pub fn init(io: *IO, fd: os.fd_t) !Storage {
         return Storage{
             .io = io,
@@ -79,6 +87,7 @@ pub const Storage = struct {
     }
 
     pub fn deinit(storage: *Storage) void {
+        assert(storage.next_tick_queue.empty());
         assert(storage.fd != IO.INVALID_FILE);
         storage.fd = IO.INVALID_FILE;
     }
@@ -90,6 +99,43 @@ pub const Storage = struct {
         };
     }
 
+    pub fn on_next_tick(
+        storage: *Storage,
+        callback: fn (next_tick: *Storage.NextTick) void,
+        next_tick: *Storage.NextTick,
+    ) void {
+        next_tick.* = .{ .callback = callback };
+
+        const was_empty = storage.next_tick_queue.empty();
+        storage.next_tick_queue.push(next_tick);
+
+        if (was_empty) {
+            storage.io.timeout(
+                *Storage,
+                storage,
+                timeout_callback,
+                &storage.next_tick_completion,
+                0, // 0ns timeout means to resolve as soon as possible - like a yield
+            );
+        }
+    }
+
+    fn timeout_callback(
+        storage: *Storage,
+        completion: *IO.Completion,
+        result: IO.TimeoutError!void,
+    ) void {
+        assert(completion == &storage.next_tick_completion);
+        _ = result catch |e| switch (e) {
+            error.Canceled => unreachable,
+            error.Unexpected => unreachable,
+        };
+
+        var queue = storage.next_tick_queue;
+        storage.next_tick_queue = .{};
+        while (queue.pop()) |next_tick| next_tick.callback(next_tick);
+    }
+
     pub fn read_sectors(
         self: *Storage,
         callback: fn (read: *Storage.Read) void,
@@ -113,18 +159,24 @@ pub const Storage = struct {
             .target_max = buffer.len,
         };
 
-        self.start_read(read, 0);
+        self.start_read(read, null);
         assert(read.target().len > 0);
     }
 
-    fn start_read(self: *Storage, read: *Storage.Read, bytes_read: usize) void {
-        assert(bytes_read <= read.target().len);
+    fn start_read(self: *Storage, read: *Storage.Read, bytes_read: ?usize) void {
+        const bytes = bytes_read orelse 0;
+        assert(bytes <= read.target().len);
 
-        read.offset += bytes_read;
-        read.buffer = read.buffer[bytes_read..];
+        read.offset += bytes;
+        read.buffer = read.buffer[bytes..];
 
         const target = read.target();
         if (target.len == 0) {
+            // Resolving the read inline means start_read() must not have been called from
+            // read_sectors(). If it was, this is a synchronous callback resolution and should
+            // be reported.
+            assert(bytes_read != null);
+
             read.callback(read);
             return;
         }

package/src/tigerbeetle/src/test/cluster.zig

@@ -137,6 +137,8 @@ pub const Cluster = struct {
             storage_options.replica_index = @intCast(u8, replica_index);
             storage_options.faulty_wal_areas = faulty_wal_areas[replica_index];
             storage.* = try Storage.init(allocator, options.storage_size_limit, storage_options);
+            // Disable most faults at startup, so that the replicas don't get stuck in recovery mode.
+            storage.faulty = replica_index >= vsr.quorums(options.replica_count).view_change;
         }
         errdefer for (cluster.storages) |*storage| storage.deinit(allocator);
 
@@ -330,6 +332,7 @@ pub const Cluster = struct {
             .{
                 .replica_count = @intCast(u8, cluster.replicas.len),
                 .storage = &cluster.storages[replica_index],
+                // TODO Test restarting with a higher storage limit.
                 .storage_size_limit = cluster.options.storage_size_limit,
                 .message_pool = &cluster.pools[replica_index],
                 .time = time,