tigerbeetle-node 0.5.2 → 0.6.0

package/src/tigerbeetle/src/io.zig

@@ -6,9 +6,11 @@ const os = std.os;
 const FIFO = @import("fifo.zig").FIFO;
 const IO_Linux = @import("io/linux.zig").IO;
 const IO_Darwin = @import("io/darwin.zig").IO;
+const IO_Windows = @import("io/windows.zig").IO;
 
 pub const IO = switch (builtin.target.os.tag) {
     .linux => IO_Linux,
+    .windows => IO_Windows,
     .macos, .tvos, .watchos, .ios => IO_Darwin,
     else => @compileError("IO is not supported for platform"),
 };

package/src/tigerbeetle/src/main.zig

@@ -20,14 +20,18 @@ const vsr = @import("vsr.zig");
 const Replica = vsr.Replica(StateMachine, MessageBus, Storage, Time);
 
 pub fn main() !void {
+    var io = try IO.init(128, 0);
+    defer io.deinit();
+
     var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
     defer arena.deinit();
 
     const allocator = arena.allocator();
 
-    switch (cli.parse_args(allocator)) {
-        .init => |args| try init(args.cluster, args.replica, args.dir_fd),
+    switch (try cli.parse_args(allocator)) {
+        .init => |args| try init(&io, args.cluster, args.replica, args.dir_fd),
         .start => |args| try start(
+            &io,
             allocator,
             args.cluster,
             args.replica,
@@ -42,14 +46,14 @@ const filename_fmt = "cluster_{d:0>10}_replica_{d:0>3}.tigerbeetle";
 const filename_len = fmt.count(filename_fmt, .{ 0, 0 });
 
 /// Create a .tigerbeetle data file for the given args and exit
-fn init(cluster: u32, replica: u8, dir_fd: os.fd_t) !void {
+fn init(io: *IO, cluster: u32, replica: u8, dir_fd: os.fd_t) !void {
     // Add 1 for the terminating null byte
     var buffer: [filename_len + 1]u8 = undefined;
     const filename = fmt.bufPrintZ(&buffer, filename_fmt, .{ cluster, replica }) catch unreachable;
     assert(filename.len == filename_len);
 
     // TODO Expose data file size on the CLI.
-    _ = try Storage.open(
+    _ = try io.open_file(
         dir_fd,
         filename,
         config.journal_size_max, // TODO Double-check that we have space for redundant headers.
@@ -61,6 +65,7 @@ fn init(cluster: u32, replica: u8, dir_fd: os.fd_t) !void {
 
 /// Run as a replica server defined by the given args
 fn start(
+    io: *IO,
     allocator: mem.Allocator,
     cluster: u32,
     replica_index: u8,
@@ -75,26 +80,26 @@ fn start(
     assert(filename.len == filename_len);
 
     // TODO Expose data file size on the CLI.
-    const storage_fd = try Storage.open(
+    const storage_fd = try io.open_file(
         dir_fd,
         filename,
         config.journal_size_max, // TODO Double-check that we have space for redundant headers.
         false,
     );
-    var io = try IO.init(128, 0);
+
     var state_machine = try StateMachine.init(
         allocator,
         config.accounts_max,
         config.transfers_max,
         config.commits_max,
     );
-    var storage = try Storage.init(config.journal_size_max, storage_fd, &io);
+    var storage = try Storage.init(config.journal_size_max, storage_fd, io);
     var message_bus = try MessageBus.init(
         allocator,
         cluster,
         addresses,
         replica_index,
-        &io,
+        io,
     );
     var time: Time = .{};
     var replica = try Replica.init(

package/src/tigerbeetle/src/message_bus.zig

@@ -17,14 +17,21 @@ const IO = @import("io.zig").IO;
 const MessagePool = @import("message_pool.zig").MessagePool;
 const Message = MessagePool.Message;
 
-const SendQueue = RingBuffer(*Message, config.connection_send_queue_max);
-
 pub const MessageBusReplica = MessageBusImpl(.replica);
 pub const MessageBusClient = MessageBusImpl(.client);
 
-const ProcessType = enum { replica, client };
+fn MessageBusImpl(comptime process_type: vsr.ProcessType) type {
+    const SendQueue = RingBuffer(*Message, switch (process_type) {
+        .replica => config.connection_send_queue_max_replica,
+        // A client has at most 1 in-flight request, plus pings.
+        .client => config.connection_send_queue_max_client,
+    });
+
+    const tcp_sndbuf = switch (process_type) {
+        .replica => config.tcp_sndbuf_replica,
+        .client => config.tcp_sndbuf_client,
+    };
 
-fn MessageBusImpl(comptime process_type: ProcessType) type {
     return struct {
         const Self = @This();
 
@@ -105,14 +112,14 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             };
 
             var bus: Self = .{
-                .pool = try MessagePool.init(allocator),
+                .pool = try MessagePool.init(allocator, process_type),
                 .io = io,
                 .cluster = cluster,
                 .configuration = configuration,
                 .process = switch (process_type) {
                     .replica => .{
                         .replica = process,
-                        .accept_fd = try init_tcp(configuration[process]),
+                        .accept_fd = try init_tcp(io, configuration[process]),
                     },
                     .client => {},
                 },
@@ -150,13 +157,13 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
         /// TODO This is required by the Client.
         pub fn deinit(_: *Self) void {}
 
-        fn init_tcp(address: std.net.Address) !os.socket_t {
-            const fd = try IO.openSocket(
+        fn init_tcp(io: *IO, address: std.net.Address) !os.socket_t {
+            const fd = try io.open_socket(
                 address.any.family,
                 os.SOCK.STREAM,
                 os.IPPROTO.TCP,
             );
-            errdefer os.close(fd);
+            errdefer os.closeSocket(fd);
 
             const set = struct {
                 fn set(_fd: os.socket_t, level: u32, option: u32, value: c_int) !void {
@@ -177,17 +184,17 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 try set(fd, os.SOL.SOCKET, os.SO.RCVBUF, config.tcp_rcvbuf);
             }
 
-            if (config.tcp_sndbuf > 0) sndbuf: {
+            if (tcp_sndbuf > 0) sndbuf: {
                 if (is_linux) {
                     // Requires CAP_NET_ADMIN privilege (settle for SO_SNDBUF in case of an EPERM):
-                    if (set(fd, os.SOL.SOCKET, os.SO.SNDBUFFORCE, config.tcp_sndbuf)) |_| {
+                    if (set(fd, os.SOL.SOCKET, os.SO.SNDBUFFORCE, tcp_sndbuf)) |_| {
                         break :sndbuf;
                     } else |err| switch (err) {
                         error.PermissionDenied => {},
                         else => |e| return e,
                     }
                 }
-                try set(fd, os.SOL.SOCKET, os.SO.SNDBUF, config.tcp_sndbuf);
+                try set(fd, os.SOL.SOCKET, os.SO.SNDBUF, tcp_sndbuf);
             }
 
             if (config.tcp_keepalive) {
@@ -338,7 +345,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             bus.process.accept_connection.?.on_accept(bus, fd);
         }
 
-        pub fn get_message(bus: *Self) ?*Message {
+        pub fn get_message(bus: *Self) *Message {
             return bus.pool.get_message();
         }
 
@@ -399,11 +406,11 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 terminating,
             } = .free,
             /// This is guaranteed to be valid only while state is connected.
-            /// It will be reset to -1 during the shutdown process and is always -1 if the
-            /// connection is unused (i.e. peer == .none). We use -1 instead of undefined here
+            /// It will be reset to IO.INVALID_SOCKET during the shutdown process and is always IO.INVALID_SOCKET if the
+            /// connection is unused (i.e. peer == .none). We use IO.INVALID_SOCKET instead of undefined here
             /// for safety to ensure an error if the invalid value is ever used, instead of
             /// potentially performing an action on an active fd.
-            fd: os.socket_t = -1,
+            fd: os.socket_t = IO.INVALID_SOCKET,
 
             /// This completion is used for all recv operations.
             /// It is also used for the initial connect when establishing a replica connection.
@@ -439,12 +446,12 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
                 assert(connection.peer == .none);
                 assert(connection.state == .free);
-                assert(connection.fd == -1);
+                assert(connection.fd == IO.INVALID_SOCKET);
 
                 // The first replica's network address family determines the
                 // family for all other replicas:
                 const family = bus.configuration[0].any.family;
-                connection.fd = IO.openSocket(family, os.SOCK.STREAM, os.IPPROTO.TCP) catch return;
+                connection.fd = bus.io.open_socket(family, os.SOCK.STREAM, os.IPPROTO.TCP) catch return;
                 connection.peer = .{ .replica = replica };
                 connection.state = .connecting;
                 bus.connections_used += 1;
@@ -545,7 +552,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             pub fn on_accept(connection: *Connection, bus: *Self, fd: os.socket_t) void {
                 assert(connection.peer == .none);
                 assert(connection.state == .accepting);
-                assert(connection.fd == -1);
+                assert(connection.fd == IO.INVALID_SOCKET);
 
                 connection.peer = .unknown;
                 connection.state = .connected;
@@ -562,7 +569,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
                 assert(connection.peer == .unknown or connection.peer == .replica);
                 assert(connection.state == .connected);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
 
                 assert(connection.recv_submitted == false);
                 assert(connection.recv_message == null);
@@ -582,15 +589,14 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                     .terminating => return,
                     .free, .accepting => unreachable,
                 }
-                connection.send_queue.push(message.ref()) catch |err| switch (err) {
-                    error.NoSpaceLeft => {
-                        bus.unref(message);
-                        log.info("message queue for peer {} full, dropping message", .{
-                            connection.peer,
-                        });
-                        return;
-                    },
-                };
+                if (connection.send_queue.full()) {
+                    log.info("message queue for peer {} full, dropping {s} message", .{
+                        connection.peer,
+                        @tagName(message.header.command),
+                    });
+                    return;
+                }
+                connection.send_queue.push_assume_capacity(message.ref());
                 // If the connection has not yet been established we can't send yet.
                 // Instead on_connect() will call send().
                 if (connection.state == .connecting) {
@@ -610,7 +616,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             pub fn terminate(connection: *Connection, bus: *Self, how: enum { shutdown, close }) void {
                 assert(connection.peer != .none);
                 assert(connection.state != .free);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
                 switch (how) {
                     .shutdown => {
                         // The shutdown syscall will cause currently in progress send/recv
@@ -618,12 +624,8 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                         //
                         // TODO: Investigate differences between shutdown() on Linux vs Darwin.
                         // Especially how this interacts with our assumptions around pending I/O.
-                        const rc = os.system.shutdown(connection.fd, os.SHUT.RDWR);
-                        switch (os.errno(rc)) {
-                            .SUCCESS => {},
-                            .BADF => unreachable,
-                            .INVAL => unreachable,
-                            .NOTCONN => {
+                        os.shutdown(connection.fd, .both) catch |err| switch (err) {
+                            error.SocketNotConnected => {
                                 // This should only happen if we for some reason decide to terminate
                                 // a connection while a connect operation is in progress.
                                 // This is fine though, we simply continue with the logic below and
@@ -638,9 +640,9 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                                 //assert(connection.recv_submitted);
                                 //assert(!connection.send_submitted);
                             },
-                            .NOTSOCK => unreachable,
-                            else => |err| os.unexpectedErrno(err) catch {},
-                        }
+                            // Ignore all the remaining errors for now
+                            error.ConnectionAborted, error.ConnectionResetByPeer, error.BlockingOperationInProgress, error.NetworkSubsystemFailed, error.SystemResources, error.Unexpected => {},
+                        };
                     },
                     .close => {},
                 }
@@ -652,7 +654,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             fn parse_messages(connection: *Connection, bus: *Self) void {
                 assert(connection.peer != .none);
                 assert(connection.state == .connected);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
 
                 while (connection.parse_message(bus)) |message| {
                     defer bus.unref(message);
@@ -734,14 +736,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                 // `references` and `header` metadata.
                 if (connection.recv_progress == header.size) return connection.recv_message.?.ref();
 
-                const message = bus.get_message() orelse {
-                    // TODO Decrease the probability of this happening by:
-                    // 1. using a header-only message if possible.
-                    // 2. determining a true upper limit for static allocation.
-                    log.err("no free buffer available to deliver message from {}", .{connection.peer});
-                    connection.terminate(bus, .shutdown);
-                    return null;
-                };
+                const message = bus.get_message();
                 mem.copy(u8, message.buffer, data[0..header.size]);
                 return message;
             }
@@ -779,7 +774,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
 
                 assert(connection.peer != .none);
                 assert(connection.state == .connected);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
 
                 if (connection.peer != .unknown) return;
 
@@ -826,14 +821,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                     return;
                 }
 
-                const new_message = bus.get_message() orelse {
-                    // TODO Decrease the probability of this happening by:
-                    // 1. using a header-only message if possible.
-                    // 2. determining a true upper limit for static allocation.
-                    log.err("no free buffer available to recv message from {}", .{connection.peer});
-                    connection.terminate(bus, .shutdown);
-                    return;
-                };
+                const new_message = bus.get_message();
                 defer bus.unref(new_message);
 
                 if (connection.recv_message) |recv_message| {
@@ -857,7 +845,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             fn recv(connection: *Connection, bus: *Self) void {
                 assert(connection.peer != .none);
                 assert(connection.state == .connected);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
 
                 assert(!connection.recv_submitted);
                 connection.recv_submitted = true;
@@ -902,7 +890,7 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
             fn send(connection: *Connection, bus: *Self) void {
                 assert(connection.peer == .client or connection.peer == .replica);
                 assert(connection.state == .connected);
-                assert(connection.fd != -1);
+                assert(connection.fd != IO.INVALID_SOCKET);
                 const message = connection.send_queue.head() orelse return;
                 assert(!connection.send_submitted);
                 connection.send_submitted = true;
@@ -959,8 +947,8 @@ fn MessageBusImpl(comptime process_type: ProcessType) type {
                     bus.unref(message);
                     connection.recv_message = null;
                 }
-                assert(connection.fd != -1);
-                defer connection.fd = -1;
+                assert(connection.fd != IO.INVALID_SOCKET);
+                defer connection.fd = IO.INVALID_SOCKET;
                 // It's OK to use the send completion here as we know that no send
                 // operation is currently in progress.
                 bus.io.close(*Self, bus, on_close, &connection.send_completion, connection.fd);

package/src/tigerbeetle/src/message_pool.zig

@@ -17,15 +17,55 @@ comptime {
 /// message to be shifted to make space for 0 padding to vsr.sector_ceil.
 const message_size_max_padded = config.message_size_max + config.sector_size;
 
-/// A pool of reference-counted Messages, memory for which is allocated only once
-/// during initialization and reused thereafter. The config.message_bus_messages_max
-/// and config.message_bus_headers_max values determine the size of this pool.
+/// The number of full-sized messages allocated at initialization by the replica message pool.
+/// There must be enough messages to ensure that the replica can always progress, to avoid deadlock.
+pub const messages_max_replica = messages_max: {
+    var sum: usize = 0;
+
+    sum += config.io_depth_read + config.io_depth_write; // Journal I/O
+    sum += config.clients_max; // Replica.client_table
+    sum += 1; // Replica.loopback_queue
+    sum += config.pipelining_max; // Replica.pipeline
+    sum += config.replicas_max; // Replica.do_view_change_from_all_replicas quorum (all others are bitsets)
+    sum += config.connections_max; // Connection.recv_message
+    sum += config.connections_max * config.connection_send_queue_max_replica; // Connection.send_queue
+    sum += 1; // Handle bursts (e.g. Connection.parse_message)
+    // Handle Replica.commit_op's reply:
+    // (This is separate from the burst +1 because they may occur concurrently).
+    sum += 1;
+    sum += 20; // TODO Our network simulator allows up to 20 messages for path_capacity_max.
+
+    break :messages_max sum;
+};
+
+/// The number of full-sized messages allocated at initialization by the client message pool.
+pub const messages_max_client = messages_max: {
+    var sum: usize = 0;
+
+    sum += config.replicas_max; // Connection.recv_message
+    sum += config.replicas_max * config.connection_send_queue_max_client; // Connection.send_queue
+    sum += config.client_request_queue_max; // Client.request_queue
+    // Handle bursts (e.g. Connection.parse_message, or sending a ping when the send queue is full).
+    sum += 1;
+    sum += 20; // TODO Our network simulator allows up to 20 messages for path_capacity_max.
+
+    break :messages_max sum;
+};
+
+comptime {
+    // These conditions are necessary (but not sufficient) to prevent deadlocks.
+    assert(messages_max_replica > config.replicas_max);
+    assert(messages_max_client > config.client_request_queue_max);
+}
+
+/// A pool of reference-counted Messages, memory for which is allocated only once during
+/// initialization and reused thereafter. The messages_max values determine the size of this pool.
 pub const MessagePool = struct {
     pub const Message = struct {
         // TODO: replace this with a header() function to save memory
         header: *Header,
-        /// Unless this Message is header only, this buffer is in aligned to config.sector_size
-        /// and casting to that alignment in order to perform Direct I/O is safe.
+        /// This buffer is aligned to config.sector_size and casting to that alignment in order
+        /// to perform Direct I/O is safe.
         buffer: []u8,
         references: u32 = 0,
         next: ?*Message,
@@ -39,27 +79,23 @@ pub const MessagePool = struct {
         pub fn body(message: *Message) []u8 {
             return message.buffer[@sizeOf(Header)..message.header.size];
         }
-
-        fn header_only(message: Message) bool {
-            const ret = message.buffer.len == @sizeOf(Header);
-            assert(ret or message.buffer.len == message_size_max_padded);
-            return ret;
-        }
     };
 
     /// List of currently unused messages of message_size_max_padded
     free_list: ?*Message,
-    /// List of currently usused header-sized messages
-    header_only_free_list: ?*Message,
 
-    pub fn init(allocator: mem.Allocator) error{OutOfMemory}!MessagePool {
+    pub fn init(allocator: mem.Allocator, process_type: vsr.ProcessType) error{OutOfMemory}!MessagePool {
+        const messages_max: usize = switch (process_type) {
+            .replica => messages_max_replica,
+            .client => messages_max_client,
+        };
+
         var ret: MessagePool = .{
             .free_list = null,
-            .header_only_free_list = null,
         };
         {
             var i: usize = 0;
-            while (i < config.message_bus_messages_max) : (i += 1) {
+            while (i < messages_max) : (i += 1) {
                 const buffer = try allocator.allocAdvanced(
                     u8,
                     config.sector_size,
@@ -75,45 +111,20 @@ pub const MessagePool = struct {
                 ret.free_list = message;
             }
         }
-        {
-            var i: usize = 0;
-            while (i < config.message_bus_headers_max) : (i += 1) {
-                const header = try allocator.create(Header);
-                const message = try allocator.create(Message);
-                message.* = .{
-                    .header = header,
-                    .buffer = mem.asBytes(header),
-                    .next = ret.header_only_free_list,
-                };
-                ret.header_only_free_list = message;
-            }
-        }
 
         return ret;
     }
 
-    /// Get an unused message with a buffer of config.message_size_max. If no such message is
-    /// available, an error is returned. The returned message has exactly one reference.
-    pub fn get_message(pool: *MessagePool) ?*Message {
-        const ret = pool.free_list orelse return null;
-        pool.free_list = ret.next;
-        ret.next = null;
-        assert(!ret.header_only());
-        assert(ret.references == 0);
-        ret.references = 1;
-        return ret;
-    }
+    /// Get an unused message with a buffer of config.message_size_max.
+    /// The returned message has exactly one reference.
+    pub fn get_message(pool: *MessagePool) *Message {
+        const message = pool.free_list.?;
+        pool.free_list = message.next;
+        message.next = null;
+        assert(message.references == 0);
 
-    /// Get an unused message with a buffer only large enough to hold a header. If no such message
-    /// is available, an error is returned. The returned message has exactly one reference.
-    pub fn get_header_only_message(pool: *MessagePool) ?*Message {
-        const ret = pool.header_only_free_list orelse return null;
-        pool.header_only_free_list = ret.next;
-        ret.next = null;
-        assert(ret.header_only());
-        assert(ret.references == 0);
-        ret.references = 1;
-        return ret;
+        message.references = 1;
+        return message;
     }
 
     /// Decrement the reference count of the message, possibly freeing it.
@@ -121,13 +132,8 @@ pub const MessagePool = struct {
         message.references -= 1;
         if (message.references == 0) {
             if (builtin.mode == .Debug) mem.set(u8, message.buffer, undefined);
-            if (message.header_only()) {
-                message.next = pool.header_only_free_list;
-                pool.header_only_free_list = message;
-            } else {
-                message.next = pool.free_list;
-                pool.free_list = message;
-            }
+            message.next = pool.free_list;
+            pool.free_list = message;
         }
     }
 };

package/src/tigerbeetle/src/ring_buffer.zig

@@ -76,6 +76,13 @@ pub fn RingBuffer(comptime T: type, comptime size: usize) type {
             self.advance_tail();
         }
 
+        /// Add an element to a RingBuffer, and assert that the capacity is sufficient.
+        pub fn push_assume_capacity(self: *Self, item: T) void {
+            self.push(item) catch |err| switch (err) {
+                error.NoSpaceLeft => unreachable,
+            };
+        }
+
         /// Remove and return the next item, if any.
         pub fn pop(self: *Self) ?T {
             const result = self.head() orelse return null;

package/src/tigerbeetle/src/simulator.zig

@@ -84,7 +84,7 @@ pub fn main() !void {
                 .one_way_delay_mean = 3 + random.uintLessThan(u16, 10),
                 .one_way_delay_min = random.uintLessThan(u16, 3),
                 .packet_loss_probability = random.uintLessThan(u8, 30),
-                .path_maximum_capacity = 20 + random.uintLessThan(u8, 20),
+                .path_maximum_capacity = 2 + random.uintLessThan(u8, 19),
                 .path_clog_duration_mean = random.uintLessThan(u16, 500),
                 .path_clog_probability = random.uintLessThan(u8, 2),
                 .packet_replay_probability = random.uintLessThan(u8, 50),
@@ -213,7 +213,7 @@ pub fn main() !void {
 
     assert(cluster.state_checker.convergence());
 
-    output.info("\n          PASSED", .{});
+    output.info("\n          PASSED ({} ticks)", .{tick});
 }
 
 /// Returns true, `p` percent of the time, else false.
@@ -244,7 +244,7 @@ fn send_request(random: std.rand.Random) bool {
     if (client.request_queue.full()) return false;
     if (checker_request_queue.full()) return false;
 
-    const message = client.get_message() orelse return false;
+    const message = client.get_message();
     defer client.unref(message);
 
     const body_size_max = config.message_size_max - @sizeOf(Header);
@@ -265,7 +265,7 @@ fn send_request(random: std.rand.Random) bool {
     // While hashing the client ID with the request body prevents input collisions across clients,
     // it's still possible for the same client to generate the same body, and therefore input hash.
     const client_input = StateMachine.hash(client.id, body);
-    checker_request_queue.push(client_input) catch unreachable;
+    checker_request_queue.push_assume_capacity(client_input);
     std.log.scoped(.test_client).debug("client {} sending input={x}", .{
         client_index,
         client_input,