tigerbeetle-node 0.5.0 → 0.5.1

package/src/tigerbeetle/src/main.zig

@@ -20,12 +20,20 @@ const vsr = @import("vsr.zig");
 const Replica = vsr.Replica(StateMachine, MessageBus, Storage, Time);
 
 pub fn main() !void {
-    var arena_allocator = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-    const arena = &arena_allocator.allocator;
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
 
-    switch (cli.parse_args(arena)) {
-        .init => |args| try init(arena, args.cluster, args.replica, args.dir_fd),
-        .start => |args| try start(arena, args.cluster, args.replica, args.addresses, args.dir_fd),
+    const allocator = arena.allocator();
+
+    switch (cli.parse_args(allocator)) {
+        .init => |args| try init(args.cluster, args.replica, args.dir_fd),
+        .start => |args| try start(
+            allocator,
+            args.cluster,
+            args.replica,
+            args.addresses,
+            args.dir_fd,
+        ),
     }
 }
 
@@ -34,7 +42,7 @@ const filename_fmt = "cluster_{d:0>10}_replica_{d:0>3}.tigerbeetle";
 const filename_len = fmt.count(filename_fmt, .{ 0, 0 });
 
 /// Create a .tigerbeetle data file for the given args and exit
-fn init(arena: *mem.Allocator, cluster: u32, replica: u8, dir_fd: os.fd_t) !void {
+fn init(cluster: u32, replica: u8, dir_fd: os.fd_t) !void {
     // Add 1 for the terminating null byte
     var buffer: [filename_len + 1]u8 = undefined;
     const filename = fmt.bufPrintZ(&buffer, filename_fmt, .{ cluster, replica }) catch unreachable;
@@ -53,7 +61,7 @@ fn init(arena: *mem.Allocator, cluster: u32, replica: u8, dir_fd: os.fd_t) !void
 
 /// Run as a replica server defined by the given args
 fn start(
-    arena: *mem.Allocator,
+    allocator: mem.Allocator,
     cluster: u32,
     replica_index: u8,
     addresses: []std.net.Address,
@@ -75,14 +83,14 @@ fn start(
     );
     var io = try IO.init(128, 0);
     var state_machine = try StateMachine.init(
-        arena,
+        allocator,
         config.accounts_max,
         config.transfers_max,
         config.commits_max,
     );
     var storage = try Storage.init(config.journal_size_max, storage_fd, &io);
     var message_bus = try MessageBus.init(
-        arena,
+        allocator,
         cluster,
         addresses,
         replica_index,
@@ -90,7 +98,7 @@ fn start(
     );
     var time: Time = .{};
     var replica = try Replica.init(
-        arena,
+        allocator,
         cluster,
         @intCast(u8, addresses.len),
         replica_index,

package/src/tigerbeetle/src/message_bus.zig

@@ -1,10 +1,10 @@
 const std = @import("std");
+const builtin = @import("builtin");
 const assert = std.debug.assert;
 const mem = std.mem;
 const os = std.os;
 
-const is_darwin = std.Target.current.isDarwin();
-const sock_flags = os.SOCK_CLOEXEC | (if (is_darwin) os.SOCK_NONBLOCK else 0);
+const is_linux = builtin.target.os.tag == .linux;
 
 const config = @import("config.zig");
 const log = std.log.scoped(.message_bus);
@@ -54,8 +54,8 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
         /// The callback to be called when a message is received. Use set_on_message() to set
         /// with type safety for the context pointer.
-        on_message_callback: ?fn (context: ?*c_void, message: *Message) void = null,
-        on_message_context: ?*c_void = null,
+        on_message_callback: ?fn (context: ?*anyopaque, message: *Message) void = null,
+        on_message_context: ?*anyopaque = null,
 
         /// This slice is allocated with a fixed size in the init function and never reallocated.
         connections: []Connection,
@@ -75,7 +75,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
         /// Initialize the MessageBus for the given cluster, configuration and replica/client process.
         pub fn init(
-            allocator: *mem.Allocator,
+            allocator: mem.Allocator,
             cluster: u32,
             configuration: []std.net.Address,
             process: switch (process_type) {
@@ -124,7 +124,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
             // Pre-allocate enough memory to hold all possible connections in the client map.
             if (process_type == .replica) {
-                try bus.process.clients.ensureCapacity(allocator, config.connections_max);
+                try bus.process.clients.ensureTotalCapacity(allocator, config.connections_max);
             }
 
             return bus;
@@ -140,7 +140,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             assert(bus.on_message_context == null);
 
             bus.on_message_callback = struct {
-                fn wrapper(_context: ?*c_void, message: *Message) void {
+                fn wrapper(_context: ?*anyopaque, message: *Message) void {
                     on_message(@intToPtr(Context, @ptrToInt(_context)), message);
                 }
             }.wrapper;
@@ -148,13 +148,13 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
         }
 
         /// TODO This is required by the Client.
-        pub fn deinit(bus: *Self) void {}
+        pub fn deinit(_: *Self) void {}
 
         fn init_tcp(address: std.net.Address) !os.socket_t {
-            const fd = try os.socket(
+            const fd = try IO.openSocket(
                 address.any.family,
-                os.SOCK_STREAM | sock_flags,
-                os.IPPROTO_TCP,
+                os.SOCK.STREAM,
+                os.IPPROTO.TCP,
             );
             errdefer os.close(fd);
 
@@ -164,71 +164,61 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 }
             }.set;
 
-            // darwin doesn't support os.MSG_NOSIGNAL, but instead a socket option to avoid SIGPIPE.
-            if (is_darwin) {
-                try set(fd, os.SOL_SOCKET, os.SO_NOSIGPIPE, 1);
-            }
-
             // Set tcp recv buffer size
             if (config.tcp_rcvbuf > 0) rcvbuf: {
-                if (!is_darwin) {
+                if (is_linux) {
                     // Requires CAP_NET_ADMIN privilege (settle for SO_RCVBUF in case of an EPERM):
-                    if (set(fd, os.SOL_SOCKET, os.SO_RCVBUFFORCE, config.tcp_rcvbuf)) |_| {
+                    if (set(fd, os.SOL.SOCKET, os.SO.RCVBUFFORCE, config.tcp_rcvbuf)) |_| {
                         break :rcvbuf;
                     } else |err| switch (err) {
                         error.PermissionDenied => {},
                         else => |e| return e,
                     }
                 }
-                try set(fd, os.SOL_SOCKET, os.SO_RCVBUF, config.tcp_rcvbuf);
+                try set(fd, os.SOL.SOCKET, os.SO.RCVBUF, config.tcp_rcvbuf);
             }
 
             // Set tcp send buffer size
             if (config.tcp_sndbuf > 0) sndbuf: {
-                if (!is_darwin) {
+                if (is_linux) {
                     // Requires CAP_NET_ADMIN privilege (settle for SO_SNDBUF in case of an EPERM):
-                    if (set(fd, os.SOL_SOCKET, os.SO_SNDBUFFORCE, config.tcp_sndbuf)) |_| {
+                    if (set(fd, os.SOL.SOCKET, os.SO.SNDBUFFORCE, config.tcp_sndbuf)) |_| {
                         break :sndbuf;
                     } else |err| switch (err) {
                         error.PermissionDenied => {},
                         else => |e| return e,
                     }
                 }
-                try set(fd, os.SOL_SOCKET, os.SO_SNDBUF, config.tcp_sndbuf);
+                try set(fd, os.SOL.SOCKET, os.SO.SNDBUF, config.tcp_sndbuf);
             }
 
             // Set tcp keep alive
             if (config.tcp_keepalive) {
-                try set(fd, os.SOL_SOCKET, os.SO_KEEPALIVE, 1);
-                if (!is_darwin) {
-                    try set(fd, os.IPPROTO_TCP, os.TCP_KEEPIDLE, config.tcp_keepidle);
-                    try set(fd, os.IPPROTO_TCP, os.TCP_KEEPINTVL, config.tcp_keepintvl);
-                    try set(fd, os.IPPROTO_TCP, os.TCP_KEEPCNT, config.tcp_keepcnt);
+                try set(fd, os.SOL.SOCKET, os.SO.KEEPALIVE, 1);
+                if (is_linux) {
+                    try set(fd, os.IPPROTO.TCP, os.TCP_KEEPIDLE, config.tcp_keepidle);
+                    try set(fd, os.IPPROTO.TCP, os.TCP_KEEPINTVL, config.tcp_keepintvl);
+                    try set(fd, os.IPPROTO.TCP, os.TCP_KEEPCNT, config.tcp_keepcnt);
                 }
             }
 
             // Set tcp user timeout
             if (config.tcp_user_timeout > 0) {
-                if (!is_darwin) {
-                    try set(fd, os.IPPROTO_TCP, os.TCP_USER_TIMEOUT, config.tcp_user_timeout);
+                if (is_linux) {
+                    try set(fd, os.IPPROTO.TCP, os.TCP_USER_TIMEOUT, config.tcp_user_timeout);
                 }
             }
 
             // Set tcp no-delay
             if (config.tcp_nodelay) {
-                const TCP_NODELAY: ?u32 = if (@hasDecl(os, "TCP_NODELAY"))
-                    @as(u32, os.TCP_NODELAY)
-                else if (is_darwin)
-                    @as(u32, 1)
-                else
-                    null;
-
-                if (TCP_NODELAY) |tcp_nodelay| {
-                    try set(fd, os.IPPROTO_TCP, tcp_nodelay, 1);
-                }
+                // TODO: use the version in the zig standard library when upgrading to 0.9
+                // https://github.com/rust-lang/libc/search?q=TCP_NODELAY
+                // https://github.com/ziglang/zig/search?q=TCP_NODELAY
+                const TCP_NODELAY: u32 = 1;
+                try set(fd, os.IPPROTO.TCP, TCP_NODELAY, 1);
             }
 
-            try set(fd, os.SOL_SOCKET, os.SO_REUSEADDR, 1);
+            try set(fd, os.SOL.SOCKET, os.SO.REUSEADDR, 1);
             try os.bind(fd, &address.any, address.getOsSockLen());
             try os.listen(fd, config.tcp_backlog);
 
@@ -289,7 +279,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 if (connection.state == .terminating) return;
             }
 
-            log.notice("all connections in use but not all replicas are connected, " ++
+            log.info("all connections in use but not all replicas are connected, " ++
                 "attempting to disconnect a client", .{});
             for (bus.connections) |*connection| {
                 if (connection.peer == .client) {
@@ -298,7 +288,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 }
             }
 
-            log.notice("failed to disconnect a client as no peer was a known client, " ++
+            log.info("failed to disconnect a client as no peer was a known client, " ++
                 "attempting to disconnect an unknown peer.", .{});
             for (bus.connections) |*connection| {
                 if (connection.peer == .unknown) {
@@ -332,7 +322,6 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 on_accept,
                 &bus.process.accept_completion,
                 bus.process.accept_fd,
-                sock_flags,
             );
         }
 
@@ -341,6 +330,8 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             completion: *IO.Completion,
             result: IO.AcceptError!os.socket_t,
         ) void {
+            _ = completion;
+
             comptime assert(process_type == .replica);
             assert(bus.process.accept_connection != null);
             defer bus.process.accept_connection = null;
@@ -459,7 +450,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 // The first replica's network address family determines the
                 // family for all other replicas:
                 const family = bus.configuration[0].any.family;
-                connection.fd = os.socket(family, os.SOCK_STREAM | sock_flags, 0) catch return;
+                connection.fd = IO.openSocket(family, os.SOCK.STREAM, os.IPPROTO.TCP) catch return;
                 connection.peer = .{ .replica = replica };
                 connection.state = .connecting;
                 bus.connections_used += 1;
@@ -469,7 +460,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
                 var attempts = &bus.replicas_connect_attempts[replica];
                 const ms = vsr.exponential_backoff_with_jitter(
-                    &bus.prng,
+                    bus.prng.random(),
                     config.connection_delay_min_ms,
                     config.connection_delay_max_ms,
                     attempts.*,
@@ -600,7 +591,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 connection.send_queue.push(message.ref()) catch |err| switch (err) {
                     error.NoSpaceLeft => {
                         bus.unref(message);
-                        log.notice("message queue for peer {} full, dropping message", .{
+                        log.info("message queue for peer {} full, dropping message", .{
                             connection.peer,
                         });
                         return;
@@ -633,12 +624,12 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                         //
                         // TODO: Investigate differences between shutdown() on Linux vs Darwin.
                         // Especially how this interacts with our assumptions around pending I/O.
-                        const rc = os.system.shutdown(connection.fd, os.SHUT_RDWR);
+                        const rc = os.system.shutdown(connection.fd, os.SHUT.RDWR);
                         switch (os.errno(rc)) {
-                            0 => {},
-                            os.EBADF => unreachable,
-                            os.EINVAL => unreachable,
-                            os.ENOTCONN => {
+                            .SUCCESS => {},
+                            .BADF => unreachable,
+                            .INVAL => unreachable,
+                            .NOTCONN => {
                                 // This should only happen if we for some reason decide to terminate
                                 // a connection while a connect operation is in progress.
                                 // This is fine though, we simply continue with the logic below and
@@ -653,7 +644,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                                 //assert(connection.recv_submitted);
                                 //assert(!connection.send_submitted);
                             },
-                            os.ENOTSOCK => unreachable,
+                            .NOTSOCK => unreachable,
                             else => |err| os.unexpectedErrno(err) catch {},
                         }
                     },
@@ -886,7 +877,6 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                     &connection.recv_completion,
                     connection.fd,
                     connection.recv_message.?.buffer[connection.recv_progress..config.message_size_max],
-                    if (is_darwin) 0 else os.MSG_NOSIGNAL,
                 );
             }
 
@@ -929,7 +919,6 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                     &connection.send_completion,
                     connection.fd,
                     message.buffer[connection.send_progress..message.header.size],
-                    if (is_darwin) 0 else os.MSG_NOSIGNAL,
                 );
             }
 

package/src/tigerbeetle/src/message_pool.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const builtin = @import("builtin");
 const assert = std.debug.assert;
 const mem = std.mem;
 
@@ -51,7 +52,7 @@ pub const MessagePool = struct {
     /// List of currently usused header-sized messages
     header_only_free_list: ?*Message,
 
-    pub fn init(allocator: *mem.Allocator) error{OutOfMemory}!MessagePool {
+    pub fn init(allocator: mem.Allocator) error{OutOfMemory}!MessagePool {
         var ret: MessagePool = .{
             .free_list = null,
             .header_only_free_list = null,
@@ -119,7 +120,7 @@ pub const MessagePool = struct {
     pub fn unref(pool: *MessagePool, message: *Message) void {
         message.references -= 1;
         if (message.references == 0) {
-            if (std.builtin.mode == .Debug) mem.set(u8, message.buffer, undefined);
+            if (builtin.mode == .Debug) mem.set(u8, message.buffer, undefined);
             if (message.header_only()) {
                 message.next = pool.header_only_free_list;
                 pool.header_only_free_list = message;

package/src/tigerbeetle/src/simulator.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const builtin = @import("builtin");
 const assert = std.debug.assert;
 const mem = std.mem;
 
@@ -17,7 +18,7 @@ const output = std.log.scoped(.state_checker);
 
 /// Set this to `false` if you want to see how literally everything works.
 /// This will run much slower but will trace all logic across the cluster.
-const log_state_transitions_only = std.builtin.mode != .Debug;
+const log_state_transitions_only = builtin.mode != .Debug;
 
 /// You can fine tune your log levels even further (debug/info/notice/warn/err/crit/alert/emerg):
 pub const log_level: std.log.Level = if (log_state_transitions_only) .info else .debug;
@@ -40,13 +41,13 @@ pub fn main() !void {
         break :seed_from_arg parse_seed(arg_two);
     };
 
-    if (std.builtin.mode == .ReleaseFast or std.builtin.mode == .ReleaseSmall) {
+    if (builtin.mode == .ReleaseFast or builtin.mode == .ReleaseSmall) {
         // We do not support ReleaseFast or ReleaseSmall because they disable assertions.
         @panic("the simulator must be run with -OReleaseSafe");
     }
 
     if (seed == seed_random) {
-        if (std.builtin.mode != .ReleaseSafe) {
+        if (builtin.mode != .ReleaseSafe) {
             // If no seed is provided, than Debug is too slow and ReleaseSafe is much faster.
             @panic("no seed provided: the simulator must be run with -OReleaseSafe");
         }
@@ -56,53 +57,53 @@ pub fn main() !void {
     }
 
     var prng = std.rand.DefaultPrng.init(seed);
-    const random = &prng.random;
+    const random = prng.random();
 
-    const replica_count = 1 + prng.random.uintLessThan(u8, config.replicas_max);
-    const client_count = 1 + prng.random.uintLessThan(u8, config.clients_max);
+    const replica_count = 1 + random.uintLessThan(u8, config.replicas_max);
+    const client_count = 1 + random.uintLessThan(u8, config.clients_max);
     const node_count = replica_count + client_count;
 
     const ticks_max = 100_000_000;
     const transitions_max = config.journal_size_max / config.message_size_max;
-    const request_probability = 1 + prng.random.uintLessThan(u8, 99);
-    const idle_on_probability = prng.random.uintLessThan(u8, 20);
-    const idle_off_probability = 10 + prng.random.uintLessThan(u8, 10);
+    const request_probability = 1 + random.uintLessThan(u8, 99);
+    const idle_on_probability = random.uintLessThan(u8, 20);
+    const idle_off_probability = 10 + random.uintLessThan(u8, 10);
 
-    cluster = try Cluster.create(allocator, &prng.random, .{
+    cluster = try Cluster.create(allocator, random, .{
         .cluster = 0,
         .replica_count = replica_count,
         .client_count = client_count,
-        .seed = prng.random.int(u64),
+        .seed = random.int(u64),
         .network_options = .{
             .packet_simulator_options = .{
                 .replica_count = replica_count,
                 .client_count = client_count,
                 .node_count = node_count,
 
-                .seed = prng.random.int(u64),
-                .one_way_delay_mean = 3 + prng.random.uintLessThan(u16, 10),
-                .one_way_delay_min = prng.random.uintLessThan(u16, 3),
-                .packet_loss_probability = prng.random.uintLessThan(u8, 30),
-                .path_maximum_capacity = 20 + prng.random.uintLessThan(u8, 20),
-                .path_clog_duration_mean = prng.random.uintLessThan(u16, 500),
-                .path_clog_probability = prng.random.uintLessThan(u8, 2),
-                .packet_replay_probability = prng.random.uintLessThan(u8, 50),
+                .seed = random.int(u64),
+                .one_way_delay_mean = 3 + random.uintLessThan(u16, 10),
+                .one_way_delay_min = random.uintLessThan(u16, 3),
+                .packet_loss_probability = random.uintLessThan(u8, 30),
+                .path_maximum_capacity = 20 + random.uintLessThan(u8, 20),
+                .path_clog_duration_mean = random.uintLessThan(u16, 500),
+                .path_clog_probability = random.uintLessThan(u8, 2),
+                .packet_replay_probability = random.uintLessThan(u8, 50),
 
                 .partition_mode = random_partition_mode(random),
-                .partition_probability = prng.random.uintLessThan(u8, 3),
-                .unpartition_probability = 1 + prng.random.uintLessThan(u8, 10),
-                .partition_stability = 100 + prng.random.uintLessThan(u32, 100),
-                .unpartition_stability = prng.random.uintLessThan(u32, 20),
+                .partition_probability = random.uintLessThan(u8, 3),
+                .unpartition_probability = 1 + random.uintLessThan(u8, 10),
+                .partition_stability = 100 + random.uintLessThan(u32, 100),
+                .unpartition_stability = random.uintLessThan(u32, 20),
             },
         },
         .storage_options = .{
-            .seed = prng.random.int(u64),
-            .read_latency_min = prng.random.uintLessThan(u16, 3),
-            .read_latency_mean = 3 + prng.random.uintLessThan(u16, 10),
-            .write_latency_min = prng.random.uintLessThan(u16, 3),
-            .write_latency_mean = 3 + prng.random.uintLessThan(u16, 10),
-            .read_fault_probability = prng.random.uintLessThan(u8, 10),
-            .write_fault_probability = prng.random.uintLessThan(u8, 10),
+            .seed = random.int(u64),
+            .read_latency_min = random.uintLessThan(u16, 3),
+            .read_latency_mean = 3 + random.uintLessThan(u16, 10),
+            .write_latency_min = random.uintLessThan(u16, 3),
+            .write_latency_mean = 3 + random.uintLessThan(u16, 10),
+            .read_fault_probability = random.uintLessThan(u8, 10),
+            .write_fault_probability = random.uintLessThan(u8, 10),
         },
     });
     defer cluster.destroy();
@@ -206,7 +207,7 @@ pub fn main() !void {
     }
 
     if (cluster.state_checker.transitions < transitions_max) {
-        output.emerg("you can reproduce this failure with seed={}", .{seed});
+        output.err("you can reproduce this failure with seed={}", .{seed});
         @panic("unable to complete transitions_max before ticks_max");
     }
 
@@ -216,13 +217,13 @@ pub fn main() !void {
 }
 
 /// Returns true, `p` percent of the time, else false.
-fn chance(random: *std.rand.Random, p: u8) bool {
+fn chance(random: std.rand.Random, p: u8) bool {
     assert(p <= 100);
     return random.uintLessThan(u8, 100) < p;
 }
 
 /// Returns the next argument for the simulator or null (if none available)
-fn args_next(args: *std.process.ArgIterator, allocator: *std.mem.Allocator) ?[:0]const u8 {
+fn args_next(args: *std.process.ArgIterator, allocator: std.mem.Allocator) ?[:0]const u8 {
     const err_or_bytes = args.next(allocator) orelse return null;
     return err_or_bytes catch @panic("Unable to extract next value from args");
 }
@@ -232,7 +233,7 @@ fn on_change_replica(replica: *Replica) void {
     cluster.state_checker.check_state(replica.replica);
 }
 
-fn send_request(random: *std.rand.Random) bool {
+fn send_request(random: std.rand.Random) bool {
     const client_index = random.uintLessThan(u8, cluster.options.client_count);
 
     const client = &cluster.clients[client_index];
@@ -280,11 +281,14 @@ fn client_callback(
     operation: StateMachine.Operation,
     results: Client.Error![]const u8,
 ) void {
+    _ = operation;
+    _ = results catch unreachable;
+
     assert(user_data == 0);
 }
 
 /// Returns a random partitioning mode, excluding .custom
-fn random_partition_mode(random: *std.rand.Random) PartitionMode {
+fn random_partition_mode(random: std.rand.Random) PartitionMode {
     const typeInfo = @typeInfo(PartitionMode).Enum;
     var enumAsInt = random.uintAtMost(typeInfo.tag_type, typeInfo.fields.len - 2);
     if (enumAsInt >= @enumToInt(PartitionMode.custom)) enumAsInt += 1;
@@ -310,8 +314,8 @@ pub fn log(
     const prefix = if (log_state_transitions_only) "" else prefix_default;
 
     // Print the message to stdout, silently ignoring any errors
-    const held = std.debug.getStderrMutex().acquire();
-    defer held.release();
     const stderr = std.io.getStdErr().writer();
+    std.debug.getStderrMutex().lock();
+    defer std.debug.getStderrMutex().unlock();
     nosuspend stderr.print(prefix ++ format ++ "\n", args) catch return;
 }