tigerbeetle-node 0.5.2 → 0.6.0

package/src/tigerbeetle/src/vsr/journal.zig

@@ -510,11 +510,7 @@ pub fn Journal(comptime Replica: type, comptime Storage: type) type {
             const physical_size = vsr.sector_ceil(exact.size);
             assert(physical_size >= exact.size);
 
-            const message = replica.message_bus.get_message() orelse {
-                self.read_prepare_log(op, checksum, "no message available");
-                callback(replica, null, null);
-                return;
-            };
+            const message = replica.message_bus.get_message();
             defer replica.message_bus.unref(message);
 
             // Skip the disk read if the header is all we need:
@@ -658,7 +654,7 @@ pub fn Journal(comptime Replica: type, comptime Storage: type) type {
             }
             assert(offset < self.size_headers);
 
-            const message = replica.message_bus.get_message() orelse unreachable;
+            const message = replica.message_bus.get_message();
             defer replica.message_bus.unref(message);
 
             // We use the count of reads executing to know when both versions have finished reading:

package/src/tigerbeetle/src/vsr/replica.zig

@@ -52,7 +52,7 @@ const Prepare = struct {
     message: *Message,
 
     /// Unique prepare_ok messages for the same view, op number and checksum from ALL replicas.
-    ok_from_all_replicas: QuorumMessages = QuorumMessagesNull,
+    ok_from_all_replicas: QuorumCounter = QuorumCounterNull,
 
     /// Whether a quorum of prepare_ok messages has been received for this prepare.
     ok_quorum_received: bool = false,
@@ -61,6 +61,9 @@ const Prepare = struct {
 const QuorumMessages = [config.replicas_max]?*Message;
 const QuorumMessagesNull = [_]?*Message{null} ** config.replicas_max;
 
+const QuorumCounter = std.StaticBitSet(config.replicas_max);
+const QuorumCounterNull = QuorumCounter.initEmpty();
+
 pub fn Replica(
     comptime StateMachine: type,
     comptime MessageBus: type,
@@ -141,13 +144,13 @@ pub fn Replica(
         loopback_queue: ?*Message = null,
 
         /// Unique start_view_change messages for the same view from OTHER replicas (excluding ourself).
-        start_view_change_from_other_replicas: QuorumMessages = QuorumMessagesNull,
+        start_view_change_from_other_replicas: QuorumCounter = QuorumCounterNull,
 
         /// Unique do_view_change messages for the same view from ALL replicas (including ourself).
         do_view_change_from_all_replicas: QuorumMessages = QuorumMessagesNull,
 
         /// Unique nack_prepare messages for the same view from OTHER replicas (excluding ourself).
-        nack_prepare_from_other_replicas: QuorumMessages = QuorumMessagesNull,
+        nack_prepare_from_other_replicas: QuorumCounter = QuorumCounterNull,
 
         /// Whether a replica has received a quorum of start_view_change messages for the view change:
         start_view_change_quorum: bool = false,
@@ -379,9 +382,6 @@ pub fn Replica(
                 var it = self.pipeline.iterator();
                 while (it.next()) |prepare| {
                     self.message_bus.unref(prepare.message);
-                    for (prepare.ok_from_all_replicas) |message| {
-                        if (message) |m| self.message_bus.unref(m);
-                    }
                 }
             }
 
@@ -391,15 +391,9 @@ pub fn Replica(
                 self.loopback_queue = null;
             }
 
-            for (self.start_view_change_from_other_replicas) |message| {
-                if (message) |m| self.message_bus.unref(m);
-            }
             for (self.do_view_change_from_all_replicas) |message| {
                 if (message) |m| self.message_bus.unref(m);
             }
-            for (self.nack_prepare_from_other_replicas) |message| {
-                if (message) |m| self.message_bus.unref(m);
-            }
         }
 
         /// Time is measured in logical ticks that are incremented on every call to tick().
@@ -608,7 +602,7 @@ pub fn Replica(
 
             log.debug("{}: on_request: prepare {}", .{ self.replica, message.header.checksum });
 
-            self.pipeline.push(.{ .message = message.ref() }) catch unreachable;
+            self.pipeline.push_assume_capacity(.{ .message = message.ref() });
             assert(self.pipeline.count >= 1);
 
             if (self.pipeline.count == 1) {
@@ -735,7 +729,7 @@ pub fn Replica(
             // Wait until we have `f + 1` prepare_ok messages (including ourself) for quorum:
             const threshold = self.quorum_replication;
 
-            const count = self.add_message_and_receive_quorum_exactly_once(
+            const count = self.count_message_and_receive_quorum_exactly_once(
                 &prepare.ok_from_all_replicas,
                 message,
                 threshold,
@@ -878,32 +872,18 @@ pub fn Replica(
             assert(self.status == .view_change);
             assert(message.header.view == self.view);
 
-            if (self.leader_index(self.view) == self.replica) {
-                // If we are the leader of the new view, then wait until we have a message to send a
-                // do_view_change message to ourself. The on_do_view_change() handler will panic if
-                // we received a start_view_change quorum without a do_view_change to ourself.
-                if (self.message_bus.get_message()) |available| {
-                    self.message_bus.unref(available);
-                } else {
-                    log.err("{}: on_start_view_change: waiting for message for do_view_change", .{
-                        self.replica,
-                    });
-                    return;
-                }
-            }
-
             // Wait until we have `f` messages (excluding ourself) for quorum:
             assert(self.replica_count > 1);
             const threshold = self.quorum_view_change - 1;
 
-            const count = self.add_message_and_receive_quorum_exactly_once(
+            const count = self.count_message_and_receive_quorum_exactly_once(
                 &self.start_view_change_from_other_replicas,
                 message,
                 threshold,
             ) orelse return;
 
             assert(count == threshold);
-            assert(self.start_view_change_from_other_replicas[self.replica] == null);
+            assert(!self.start_view_change_from_other_replicas.isSet(self.replica));
             log.debug("{}: on_start_view_change: view={} quorum received", .{
                 self.replica,
                 self.view,
@@ -956,7 +936,7 @@ pub fn Replica(
             assert(self.replica_count > 1);
             const threshold = self.quorum_view_change;
 
-            const count = self.add_message_and_receive_quorum_exactly_once(
+            const count = self.reference_message_and_receive_quorum_exactly_once(
                 &self.do_view_change_from_all_replicas,
                 message,
                 threshold,
@@ -1097,12 +1077,7 @@ pub fn Replica(
             assert(message.header.replica != self.replica);
             assert(self.leader());
 
-            const start_view = self.create_view_change_message(.start_view) orelse {
-                log.err("{}: on_request_start_view: dropping start_view, no message available", .{
-                    self.replica,
-                });
-                return;
-            };
+            const start_view = self.create_view_change_message(.start_view);
             defer self.message_bus.unref(start_view);
 
             assert(start_view.references == 1);
@@ -1126,10 +1101,7 @@ pub fn Replica(
                 return;
             }
 
-            const response = self.message_bus.get_message() orelse {
-                log.err("{}: on_recovery: ignoring (waiting for message)", .{self.replica});
-                return;
-            };
+            const response = self.message_bus.get_message();
             defer self.message_bus.unref(response);
 
             response.header.* = .{
@@ -1276,14 +1248,7 @@ pub fn Replica(
             assert(message.header.view == self.view);
             assert(message.header.replica != self.replica);
 
-            const response = self.message_bus.get_message() orelse {
-                log.err("{}: on_request_headers: ignoring (op={}..{}, no message available)", .{
-                    self.replica,
-                    message.header.commit,
-                    message.header.op,
-                });
-                return;
-            };
+            const response = self.message_bus.get_message();
             defer self.message_bus.unref(response);
 
             response.header.* = .{
@@ -1410,14 +1375,14 @@ pub fn Replica(
             assert(threshold < self.replica_count);
 
             // Wait until we have `threshold` messages for quorum:
-            const count = self.add_message_and_receive_quorum_exactly_once(
+            const count = self.count_message_and_receive_quorum_exactly_once(
                 &self.nack_prepare_from_other_replicas,
                 message,
                 threshold,
             ) orelse return;
 
             assert(count == threshold);
-            assert(self.nack_prepare_from_other_replicas[self.replica] == null);
+            assert(!self.nack_prepare_from_other_replicas.isSet(self.replica));
             log.debug("{}: on_nack_prepare: quorum received", .{self.replica});
 
             self.discard_uncommitted_ops_from(op, checksum);
@@ -1487,18 +1452,32 @@ pub fn Replica(
             // The list of remote replicas yet to send a prepare_ok:
             var waiting: [config.replicas_max]u8 = undefined;
             var waiting_len: usize = 0;
-            for (prepare.ok_from_all_replicas[0..self.replica_count]) |received, replica| {
-                if (received == null and replica != self.replica) {
+            var ok_from_all_replicas_iterator = prepare.ok_from_all_replicas.iterator(.{
+                .kind = .unset,
+            });
+            while (ok_from_all_replicas_iterator.next()) |replica| {
+                // Ensure we don't wait for replicas that don't exist.
+                // The bits between `replica_count` and `replicas_max` are always unset,
+                // since they don't actually represent replicas.
+                if (replica == self.replica_count) {
+                    assert(self.replica_count < config.replicas_max);
+                    break;
+                }
+                assert(replica < self.replica_count);
+
+                if (replica != self.replica) {
                     waiting[waiting_len] = @intCast(u8, replica);
                     waiting_len += 1;
                 }
+            } else {
+                assert(self.replica_count == config.replicas_max);
             }
 
             if (waiting_len == 0) {
                 self.prepare_timeout.reset();
 
                 log.debug("{}: on_prepare_timeout: waiting for journal", .{self.replica});
-                assert(prepare.ok_from_all_replicas[self.replica] == null);
+                assert(!prepare.ok_from_all_replicas.isSet(self.replica));
 
                 // We may be slow and waiting for the write to complete.
                 //
@@ -1590,7 +1569,7 @@ pub fn Replica(
             self.repair();
         }
 
-        fn add_message_and_receive_quorum_exactly_once(
+        fn reference_message_and_receive_quorum_exactly_once(
             self: *Self,
             messages: *QuorumMessages,
             message: *Message,
@@ -1604,18 +1583,6 @@ pub fn Replica(
             assert(message.header.replica < self.replica_count);
             assert(message.header.view == self.view);
             switch (message.header.command) {
-                .prepare_ok => {
-                    if (self.replica_count <= 2) assert(threshold == self.replica_count);
-
-                    assert(self.status == .normal);
-                    assert(self.leader());
-                },
-                .start_view_change => {
-                    assert(self.replica_count > 1);
-                    if (self.replica_count == 2) assert(threshold == 1);
-
-                    assert(self.status == .view_change);
-                },
                 .do_view_change => {
                     assert(self.replica_count > 1);
                     if (self.replica_count == 2) assert(threshold == 2);
@@ -1623,13 +1590,6 @@ pub fn Replica(
                     assert(self.status == .view_change);
                     assert(self.leader_index(self.view) == self.replica);
                 },
-                .nack_prepare => {
-                    assert(self.replica_count > 1);
-                    if (self.replica_count == 2) assert(threshold >= 1);
-
-                    assert(self.status == .view_change);
-                    assert(self.leader_index(self.view) == self.replica);
-                },
                 else => unreachable,
             }
 
@@ -1673,6 +1633,76 @@ pub fn Replica(
             return count;
         }
 
+        fn count_message_and_receive_quorum_exactly_once(
+            self: *Self,
+            counter: *QuorumCounter,
+            message: *Message,
+            threshold: u32,
+        ) ?usize {
+            assert(threshold >= 1);
+            assert(threshold <= self.replica_count);
+
+            assert(QuorumCounter.bit_length == config.replicas_max);
+            assert(message.header.cluster == self.cluster);
+            assert(message.header.replica < self.replica_count);
+            assert(message.header.view == self.view);
+
+            switch (message.header.command) {
+                .prepare_ok => {
+                    if (self.replica_count <= 2) assert(threshold == self.replica_count);
+
+                    assert(self.status == .normal);
+                    assert(self.leader());
+                },
+                .start_view_change => {
+                    assert(self.replica_count > 1);
+                    if (self.replica_count == 2) assert(threshold == 1);
+
+                    assert(self.status == .view_change);
+                },
+                .nack_prepare => {
+                    assert(self.replica_count > 1);
+                    if (self.replica_count == 2) assert(threshold >= 1);
+
+                    assert(self.status == .view_change);
+                    assert(self.leader_index(self.view) == self.replica);
+                },
+                else => unreachable,
+            }
+
+            const command: []const u8 = @tagName(message.header.command);
+
+            // Do not allow duplicate messages to trigger multiple passes through a state transition:
+            if (counter.isSet(message.header.replica)) {
+                log.debug("{}: on_{s}: ignoring (duplicate message)", .{ self.replica, command });
+                return null;
+            }
+
+            // Record the first receipt of this message:
+            counter.set(message.header.replica);
+            assert(counter.isSet(message.header.replica));
+
+            // Count the number of unique messages now received:
+            const count = counter.count();
+            log.debug("{}: on_{s}: {} message(s)", .{ self.replica, command, count });
+            assert(count <= self.replica_count);
+
+            // Wait until we have exactly `threshold` messages for quorum:
+            if (count < threshold) {
+                log.debug("{}: on_{s}: waiting for quorum", .{ self.replica, command });
+                return null;
+            }
+
+            // This is not the first time we have had quorum, the state transition has already happened:
+            if (count > threshold) {
+                log.debug("{}: on_{s}: ignoring (quorum received already)", .{ self.replica, command });
+                return null;
+            }
+
+            assert(count == threshold);
+            return count;
+        }
+
         fn append(self: *Self, message: *Message) void {
             assert(self.status == .normal);
             assert(message.header.command == .prepare);
@@ -1834,14 +1864,7 @@ pub fn Replica(
                 return;
             }
 
-            // TODO We can optimize this to commit into the client table reply if it exists.
-            const reply = self.message_bus.get_message() orelse {
-                log.err("{}: commit_ops_commit: waiting for message", .{self.replica});
-                return;
-            };
-            defer self.message_bus.unref(reply);
-
-            self.commit_op(prepare.?, reply);
+            self.commit_op(prepare.?);
 
             assert(self.commit_min == op);
             assert(self.commit_min <= self.commit_max);
@@ -1851,7 +1874,7 @@ pub fn Replica(
             self.commit_ops_read();
         }
 
-        fn commit_op(self: *Self, prepare: *const Message, reply: *Message) void {
+        fn commit_op(self: *Self, prepare: *const Message) void {
             // TODO Can we add more checks around allowing commit_op() during a view change?
             assert(self.status == .normal or self.status == .view_change);
             assert(prepare.header.command == .prepare);
@@ -1875,6 +1898,9 @@ pub fn Replica(
                 @tagName(prepare.header.operation.cast(StateMachine)),
             });
 
+            const reply = self.message_bus.get_message();
+            defer self.message_bus.unref(reply);
+
             const reply_body_size = @intCast(u32, self.state_machine.commit(
                 prepare.header.client,
                 prepare.header.operation.cast(StateMachine),
@@ -1939,27 +1965,16 @@ pub fn Replica(
                     return;
                 }
 
-                const count = self.count_quorum(
-                    &prepare.ok_from_all_replicas,
-                    .prepare_ok,
-                    prepare.message.header.checksum,
-                );
+                const count = prepare.ok_from_all_replicas.count();
                 assert(count >= self.quorum_replication);
+                assert(count <= self.replica_count);
 
-                // TODO We can optimize this to commit into the client table reply if it exists.
-                const reply = self.message_bus.get_message() orelse {
-                    // Eventually handled by on_prepare_timeout().
-                    log.err("{}: commit_pipeline: waiting for message", .{self.replica});
-                    return;
-                };
-                defer self.message_bus.unref(reply);
-
-                self.commit_op(prepare.message, reply);
+                self.commit_op(prepare.message);
 
                 assert(self.commit_min == self.commit_max);
                 assert(self.commit_max == prepare.message.header.op);
 
-                self.unref_prepare_message_and_quorum_messages(prepare);
+                self.message_bus.unref(prepare.message);
                 assert(self.pipeline.pop() != null);
             }
 
@@ -1984,15 +1999,6 @@ pub fn Replica(
                     assert(m.header.context == context);
                     assert(m.header.replica == replica);
                     switch (command) {
-                        .prepare_ok => {
-                            if (self.status == .normal) {
-                                assert(self.leader());
-                                assert(m.header.view == self.view);
-                            } else {
-                                assert(self.status == .view_change);
-                                assert(m.header.view < self.view);
-                            }
-                        },
                         .start_view_change => {
                             assert(m.header.replica != self.replica);
                             assert(m.header.view == self.view);
@@ -2090,13 +2096,13 @@ pub fn Replica(
         }
 
         /// The caller owns the returned message, if any, which has exactly 1 reference.
-        fn create_view_change_message(self: *Self, command: Command) ?*Message {
+        fn create_view_change_message(self: *Self, command: Command) *Message {
             assert(command == .do_view_change or command == .start_view);
 
             // We may send a start_view message in normal status to resolve a follower's view jump:
             assert(self.status == .normal or self.status == .view_change);
 
-            const message = self.message_bus.get_message() orelse return null;
+            const message = self.message_bus.get_message();
             defer self.message_bus.unref(message);
 
             message.header.* = .{
@@ -2146,12 +2152,12 @@ pub fn Replica(
         }
 
         /// The caller owns the returned message, if any, which has exactly 1 reference.
-        fn create_message_from_header(self: *Self, header: Header) ?*Message {
+        fn create_message_from_header(self: *Self, header: Header) *Message {
             assert(header.replica == self.replica);
             assert(header.view == self.view or header.command == .request_start_view);
             assert(header.size == @sizeOf(Header));
 
-            const message = self.message_bus.pool.get_header_only_message() orelse return null;
+            const message = self.message_bus.pool.get_message();
             defer self.message_bus.unref(message);
 
             message.header.* = header;
@@ -3206,7 +3212,7 @@ pub fn Replica(
                 prepare.?.header.checksum,
             });
 
-            self.pipeline.push(.{ .message = prepare.?.ref() }) catch unreachable;
+            self.pipeline.push_assume_capacity(.{ .message = prepare.?.ref() });
             assert(self.pipeline.count >= 1);
 
             self.repairing_pipeline = true;
@@ -3336,17 +3342,11 @@ pub fn Replica(
                     assert(nack_prepare_op <= op);
                     if (nack_prepare_op != op) {
                         self.nack_prepare_op = op;
-                        self.reset_quorum_messages(
-                            &self.nack_prepare_from_other_replicas,
-                            .nack_prepare,
-                        );
+                        self.reset_quorum_counter(&self.nack_prepare_from_other_replicas);
                     }
                 } else {
                     self.nack_prepare_op = op;
-                    self.reset_quorum_messages(
-                        &self.nack_prepare_from_other_replicas,
-                        .nack_prepare,
-                    );
+                    self.reset_quorum_counter(&self.nack_prepare_from_other_replicas);
                 }
 
                 assert(self.nack_prepare_op.? == op);
@@ -3421,7 +3421,7 @@ pub fn Replica(
         /// Stops the prepare timeout and resets the timeouts counter.
         fn reset_pipeline(self: *Self) void {
             while (self.pipeline.pop()) |prepare| {
-                self.unref_prepare_message_and_quorum_messages(&prepare);
+                self.message_bus.unref(prepare.message);
             }
 
             self.prepare_timeout.stop();
@@ -3460,18 +3460,33 @@ pub fn Replica(
             log.debug("{}: reset {} {s} message(s)", .{ self.replica, count, @tagName(command) });
         }
 
+        fn reset_quorum_counter(self: *Self, counter: *QuorumCounter) void {
+            var counter_iterator = counter.iterator(.{});
+            while (counter_iterator.next()) |replica| {
+                assert(replica < self.replica_count);
+            }
+
+            counter.setIntersection(QuorumCounterNull);
+            assert(counter.count() == 0);
+
+            var replica: usize = 0;
+            while (replica < self.replica_count) : (replica += 1) {
+                assert(!counter.isSet(replica));
+            }
+        }
+
         fn reset_quorum_do_view_change(self: *Self) void {
             self.reset_quorum_messages(&self.do_view_change_from_all_replicas, .do_view_change);
             self.do_view_change_quorum = false;
         }
 
         fn reset_quorum_nack_prepare(self: *Self) void {
-            self.reset_quorum_messages(&self.nack_prepare_from_other_replicas, .nack_prepare);
+            self.reset_quorum_counter(&self.nack_prepare_from_other_replicas);
             self.nack_prepare_op = null;
         }
 
         fn reset_quorum_start_view_change(self: *Self) void {
-            self.reset_quorum_messages(&self.start_view_change_from_other_replicas, .start_view_change);
+            self.reset_quorum_counter(&self.start_view_change_from_other_replicas);
             self.start_view_change_quorum = false;
         }
 
@@ -3576,17 +3591,12 @@ pub fn Replica(
             assert(self.status == .view_change);
             assert(self.start_view_change_quorum);
             assert(!self.do_view_change_quorum);
-            const count_start_view_change = self.count_quorum(
-                &self.start_view_change_from_other_replicas,
-                .start_view_change,
-                0,
-            );
+
+            const count_start_view_change = self.start_view_change_from_other_replicas.count();
             assert(count_start_view_change >= self.quorum_view_change - 1);
+            assert(count_start_view_change <= self.replica_count - 1);
 
-            const message = self.create_view_change_message(.do_view_change) orelse {
-                log.err("{}: send_do_view_change: waiting for message", .{self.replica});
-                return;
-            };
+            const message = self.create_view_change_message(.do_view_change);
             defer self.message_bus.unref(message);
 
             assert(message.references == 1);
@@ -3618,25 +3628,14 @@ pub fn Replica(
         }
 
         fn send_header_to_client(self: *Self, client: u128, header: Header) void {
-            const message = self.create_message_from_header(header) orelse {
-                log.err("{}: no header-only message available, dropping message to client {}", .{
-                    self.replica,
-                    client,
-                });
-                return;
-            };
+            const message = self.create_message_from_header(header);
             defer self.message_bus.unref(message);
 
             self.message_bus.send_message_to_client(client, message);
         }
 
         fn send_header_to_other_replicas(self: *Self, header: Header) void {
-            const message = self.create_message_from_header(header) orelse {
-                log.err("{}: no header-only message available, dropping message to replicas", .{
-                    self.replica,
-                });
-                return;
-            };
+            const message = self.create_message_from_header(header);
             defer self.message_bus.unref(message);
 
             var replica: u8 = 0;
@@ -3648,13 +3647,7 @@ pub fn Replica(
         }
 
         fn send_header_to_replica(self: *Self, replica: u8, header: Header) void {
-            const message = self.create_message_from_header(header) orelse {
-                log.err("{}: no header-only message available, dropping message to replica {}", .{
-                    self.replica,
-                    replica,
-                });
-                return;
-            };
+            const message = self.create_message_from_header(header);
             defer self.message_bus.unref(message);
 
             self.send_message_to_replica(replica, message);
@@ -3912,10 +3905,7 @@ pub fn Replica(
             assert(self.journal.faulty.len == 0);
             assert(self.nack_prepare_op == null);
 
-            const start_view = self.create_view_change_message(.start_view) orelse {
-                log.err("{}: start_view_as_the_new_leader: waiting for message", .{self.replica});
-                return;
-            };
+            const start_view = self.create_view_change_message(.start_view);
             defer self.message_bus.unref(start_view);
 
             self.transition_to_normal_status(self.view);
@@ -4022,19 +4012,6 @@ pub fn Replica(
             self.send_start_view_change();
         }
 
-        fn unref_prepare_message_and_quorum_messages(
-            self: *Self,
-            prepare: *const Prepare,
-        ) void {
-            self.message_bus.unref(prepare.message);
-            for (prepare.ok_from_all_replicas) |received, replica| {
-                if (received) |prepare_ok| {
-                    assert(replica < self.replica_count);
-                    self.message_bus.unref(prepare_ok);
-                }
-            }
-        }
-
         fn update_client_table_entry(self: *Self, reply: *Message) void {
             assert(reply.header.command == .reply);
             assert(reply.header.operation != .register);