tigerbeetle-node 0.5.0 → 0.5.1

package/src/tigerbeetle/src/io/benchmark.zig

@@ -0,0 +1,238 @@
+const std = @import("std");
+const os = std.os;
+const assert = std.debug.assert;
+
+const Time = @import("../time.zig").Time;
+const IO = @import("../io.zig").IO;
+
+// 1 MB: larger than socket buffer so forces io_pending on darwin
+// Configure this value to smaller amounts to test IO scheduling overhead
+const buffer_size = 1 * 1024 * 1024;
+
+// max time for the benchmark to run
+const run_duration = 1 * std.time.ns_per_s;
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    const allocator = &gpa.allocator;
+    defer {
+        const leaks = gpa.deinit();
+        assert(!leaks);
+    }
+
+    const buffer = try allocator.alloc(u8, buffer_size * 2);
+    defer allocator.free(buffer);
+    std.mem.set(u8, buffer, 0);
+
+    var timer = Time{};
+    const started = timer.monotonic();
+    var self = Context{
+        .io = try IO.init(32, 0),
+        .timer = &timer,
+        .started = started,
+        .current = started,
+        .tx = .{ .buffer = buffer[0 * buffer_size ..][0..buffer_size] },
+        .rx = .{ .buffer = buffer[1 * buffer_size ..][0..buffer_size] },
+    };
+
+    defer {
+        self.io.deinit();
+        const elapsed_ns = self.current - started;
+        const transferred_mb = @intToFloat(f64, self.transferred) / 1024 / 1024;
+
+        std.debug.print("IO throughput test: took {}ms @ {d:.2} MB/s\n", .{
+            elapsed_ns / std.time.ns_per_ms,
+            transferred_mb / (@intToFloat(f64, elapsed_ns) / std.time.ns_per_s),
+        });
+    }
+
+    // Setup the server socket
+    self.server.fd = try IO.openSocket(os.AF_INET, os.SOCK_STREAM, os.IPPROTO_TCP);
+    defer os.closeSocket(self.server.fd);
+
+    const address = try std.net.Address.parseIp4("127.0.0.1", 3131);
+    try os.setsockopt(
+        self.server.fd,
+        os.SOL_SOCKET,
+        os.SO_REUSEADDR,
+        &std.mem.toBytes(@as(c_int, 1)),
+    );
+    try os.bind(self.server.fd, &address.any, address.getOsSockLen());
+    try os.listen(self.server.fd, 1);
+
+    // Start accepting the client
+    self.io.accept(
+        *Context,
+        &self,
+        Context.on_accept,
+        &self.server.completion,
+        self.server.fd,
+    );
+
+    // Setup the client connection
+    self.tx.socket.fd = try IO.openSocket(os.AF_INET, os.SOCK_STREAM, os.IPPROTO_TCP);
+    defer os.closeSocket(self.tx.socket.fd);
+
+    self.io.connect(
+        *Context,
+        &self,
+        Context.on_connect,
+        &self.tx.socket.completion,
+        self.tx.socket.fd,
+        address,
+    );
+
+    // Run the IO loop, calling either tick() or run_for_ns() at "pseudo-random"
+    // to benchmark each io-driving execution path
+    var tick: usize = 0xdeadbeef;
+    while (self.is_running()) : (tick +%= 1) {
+        if (tick % 61 == 0) {
+            const timeout_ns = tick % (10 * std.time.ns_per_ms);
+            try self.io.run_for_ns(@intCast(u63, timeout_ns));
+        } else {
+            try self.io.tick();
+        }
+    }
+
+    // Assert that everything is connected
+    assert(self.server.fd != -1);
+    assert(self.tx.socket.fd != -1);
+    assert(self.rx.socket.fd != -1);
+
+    // Close the accepted client socket.
+    // The actual client socket + server socket are closed by defer
+    os.closeSocket(self.rx.socket.fd);
+}
+
+const Context = struct {
+    io: IO,
+    tx: Pipe,
+    rx: Pipe,
+    timer: *Time,
+    started: u64,
+    current: u64,
+    server: Socket = .{},
+    transferred: u64 = 0,
+
+    const Socket = struct {
+        fd: os.socket_t = -1,
+        completion: IO.Completion = undefined,
+    };
+    const Pipe = struct {
+        socket: Socket = .{},
+        buffer: []u8,
+        transferred: usize = 0,
+    };
+
+    fn is_running(self: Context) bool {
+        // Make sure that we're connected
+        if (self.rx.socket.fd == -1) return true;
+
+        // Make sure that we haven't run too long as configured
+        const elapsed = self.current - self.started;
+        return elapsed < run_duration;
+    }
+
+    fn on_accept(
+        self: *Context,
+        completion: *IO.Completion,
+        result: IO.AcceptError!os.socket_t,
+    ) void {
+        assert(self.rx.socket.fd == -1);
+        assert(&self.server.completion == completion);
+        self.rx.socket.fd = result catch |err| std.debug.panic("accept error {}", .{err});
+
+        // Start reading data from the accepted client socket
+        assert(self.rx.transferred == 0);
+        self.do_transfer("rx", .read, 0);
+    }
+
+    fn on_connect(
+        self: *Context,
+        completion: *IO.Completion,
+        result: IO.ConnectError!void,
+    ) void {
+        assert(self.tx.socket.fd != -1);
+        assert(&self.tx.socket.completion == completion);
+
+        // Start sending data to the server's accepted client
+        assert(self.tx.transferred == 0);
+        self.do_transfer("tx", .write, 0);
+    }
+
+    const TransferType = enum {
+        read = 0,
+        write = 1,
+    };
+
+    fn do_transfer(
+        self: *Context,
+        comptime pipe_name: []const u8,
+        comptime transfer_type: TransferType,
+        bytes: usize,
+    ) void {
+        // The type of IO to perform and what type of IO to perform next (after the current one completes).
+        const transfer_info = switch (transfer_type) {
+            .read => .{
+                .IoError = IO.RecvError,
+                .io_func = "recv",
+                .next = TransferType.write,
+            },
+            .write => .{
+                .IoError = IO.SendError,
+                .io_func = "send",
+                .next = TransferType.read,
+            },
+        };
+
+        assert(bytes <= buffer_size);
+        self.transferred += bytes;
+
+        // Check in with the benchmark timer to stop sending/receiving data
+        self.current = self.timer.monotonic();
+        if (!self.is_running())
+            return;
+
+        // Select which connection (tx or rx) depending on the type of transfer
+        const pipe = &@field(self, pipe_name);
+        pipe.transferred += bytes;
+        assert(pipe.transferred <= pipe.buffer.len);
+
+        // There's still more data to transfer on the connection
+        if (pipe.transferred < pipe.buffer.len) {
+            // Callback which calls this function again when data is transferred.
+            // Effectively loops back above.
+            const on_transfer = struct {
+                fn on_transfer(
+                    _self: *Context,
+                    completion: *IO.Completion,
+                    result: transfer_info.IoError!usize,
+                ) void {
+                    const _bytes = result catch |err| {
+                        std.debug.panic("{s} error: {}", .{ transfer_info.io_func, err });
+                    };
+                    assert(&@field(_self, pipe_name).socket.completion == completion);
+                    _self.do_transfer(pipe_name, transfer_type, _bytes);
+                }
+            }.on_transfer;
+
+            // Perform the IO with the callback for the completion
+            return @field(self.io, transfer_info.io_func)(
+                *Context,
+                self,
+                on_transfer,
+                &pipe.socket.completion,
+                pipe.socket.fd,
+                pipe.buffer[pipe.transferred..],
+            );
+        }
+
+        // This transfer type completed transferring all the bytes.
+        // Now, switch the transfer type (transfer_info.next).
+        // This means if we read to the buffer, now we write it out.
+        // Inversely, if we wrote the buffer, now we read it back.
+        // This is basically a modified echo benchmark.
+        pipe.transferred = 0;
+        self.do_transfer(pipe_name, transfer_info.next, 0);
+    }
+};

package/src/tigerbeetle/src/{io_darwin.zig → io/darwin.zig}

@@ -1,19 +1,24 @@
 const std = @import("std");
 const os = std.os;
+const mem = std.mem;
 const assert = std.debug.assert;
 
-const FIFO = @import("fifo.zig").FIFO;
-const Time = @import("time.zig").Time;
-const buffer_limit = @import("io.zig").buffer_limit;
+const FIFO = @import("../fifo.zig").FIFO;
+const Time = @import("../time.zig").Time;
+const buffer_limit = @import("../io.zig").buffer_limit;
 
 pub const IO = struct {
     kq: os.fd_t,
     time: Time = .{},
+    io_inflight: usize = 0,
     timeouts: FIFO(Completion) = .{},
     completed: FIFO(Completion) = .{},
     io_pending: FIFO(Completion) = .{},
 
     pub fn init(entries: u12, flags: u32) !IO {
+        _ = entries;
+        _ = flags;
+
         const kq = try os.kqueue();
         assert(kq > -1);
         return IO{ .kq = kq };
@@ -27,9 +32,7 @@ pub const IO = struct {
 
     /// Pass all queued submissions to the kernel and peek for completions.
     pub fn tick(self: *IO) !void {
-        // TODO This is a hack to block 1ms every 10ms tick on macOS while we fix `flush(false)`:
-        // What we're seeing for the tigerbeetle-node client is that recv() syscalls never complete.
-        return self.run_for_ns(std.time.ns_per_ms);
+        return self.flush(false);
     }
 
     /// Pass all queued submissions to the kernel and run for `nanoseconds`.
@@ -42,8 +45,11 @@ pub const IO = struct {
             fn callback(
                 timed_out_ptr: *bool,
                 _completion: *Completion,
-                _result: TimeoutError!void,
+                result: TimeoutError!void,
             ) void {
+                _ = _completion;
+                _ = result catch unreachable;
+
                 timed_out_ptr.* = true;
             }
         }.callback;
@@ -84,10 +90,13 @@ pub const IO = struct {
             // - tick() is non-blocking (wait_for_completions = false)
             // - run_for_ns() always submits a timeout
             if (change_events == 0 and self.completed.peek() == null) {
-                if (!wait_for_completions) return;
-                const timeout_ns = next_timeout orelse @panic("kevent() blocking forever");
-                ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
-                ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
+                if (wait_for_completions) {
+                    const timeout_ns = next_timeout orelse @panic("kevent() blocking forever");
+                    ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
+                    ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
+                } else if (self.io_inflight == 0) {
+                    return;
+                }
             }
 
             const new_events = try os.kevent(
@@ -103,6 +112,9 @@ pub const IO = struct {
                 self.io_pending.in = null;
             }
 
+            self.io_inflight += change_events;
+            self.io_inflight -= new_events;
+
             for (events[0..new_events]) |event| {
                 const completion = @intToPtr(*Completion, event.udata);
                 completion.next = null;
@@ -117,25 +129,25 @@ pub const IO = struct {
         }
     }
 
-    fn flush_io(self: *IO, events: []os.Kevent, io_pending_top: *?*Completion) usize {
+    fn flush_io(_: *IO, events: []os.Kevent, io_pending_top: *?*Completion) usize {
         for (events) |*event, flushed| {
             const completion = io_pending_top.* orelse return flushed;
             io_pending_top.* = completion.next;
 
             const event_info = switch (completion.operation) {
-                .accept => |op| [2]c_int{ op.socket, os.EVFILT_READ },
-                .connect => |op| [2]c_int{ op.socket, os.EVFILT_WRITE },
-                .read => |op| [2]c_int{ op.fd, os.EVFILT_READ },
-                .write => |op| [2]c_int{ op.fd, os.EVFILT_WRITE },
-                .recv => |op| [2]c_int{ op.socket, os.EVFILT_READ },
-                .send => |op| [2]c_int{ op.socket, os.EVFILT_WRITE },
+                .accept => |op| [2]c_int{ op.socket, os.system.EVFILT_READ },
+                .connect => |op| [2]c_int{ op.socket, os.system.EVFILT_WRITE },
+                .read => |op| [2]c_int{ op.fd, os.system.EVFILT_READ },
+                .write => |op| [2]c_int{ op.fd, os.system.EVFILT_WRITE },
+                .recv => |op| [2]c_int{ op.socket, os.system.EVFILT_READ },
+                .send => |op| [2]c_int{ op.socket, os.system.EVFILT_WRITE },
                 else => @panic("invalid completion operation queued for io"),
             };
 
             event.* = .{
                 .ident = @intCast(u32, event_info[0]),
                 .filter = @intCast(i16, event_info[1]),
-                .flags = os.EV_ADD | os.EV_ENABLE | os.EV_ONESHOT,
+                .flags = os.system.EV_ADD | os.system.EV_ENABLE | os.system.EV_ONESHOT,
                 .fflags = 0,
                 .data = 0,
                 .udata = @ptrToInt(completion),
@@ -175,7 +187,7 @@ pub const IO = struct {
     /// This struct holds the data needed for a single IO operation
     pub const Completion = struct {
         next: ?*Completion,
-        context: ?*c_void,
+        context: ?*anyopaque,
         callback: fn (*IO, *Completion) void,
         operation: Operation,
     };
@@ -183,7 +195,6 @@ pub const IO = struct {
     const Operation = union(enum) {
         accept: struct {
             socket: os.socket_t,
-            flags: u32,
         },
         close: struct {
             fd: os.fd_t,
@@ -195,13 +206,6 @@ pub const IO = struct {
         },
         fsync: struct {
             fd: os.fd_t,
-            flags: u32,
-        },
-        openat: struct {
-            fd: os.fd_t,
-            path: [*:0]const u8,
-            flags: u32,
-            mode: os.mode_t,
         },
         read: struct {
             fd: os.fd_t,
@@ -213,13 +217,11 @@ pub const IO = struct {
             socket: os.socket_t,
             buf: [*]u8,
             len: u32,
-            flags: u32,
         },
         send: struct {
             socket: os.socket_t,
             buf: [*]const u8,
             len: u32,
-            flags: u32,
         },
         timeout: struct {
             expires: u64,
@@ -285,7 +287,7 @@ pub const IO = struct {
         }
     }
 
-    pub const AcceptError = os.AcceptError;
+    pub const AcceptError = os.AcceptError || os.SetSockOptError;
 
     pub fn accept(
         self: *IO,
@@ -298,7 +300,6 @@ pub const IO = struct {
         ) void,
         completion: *Completion,
         socket: os.socket_t,
-        flags: u32,
     ) void {
         self.submit(
             context,
@@ -307,11 +308,33 @@ pub const IO = struct {
             .accept,
             .{
                 .socket = socket,
-                .flags = flags,
             },
             struct {
                 fn doOperation(op: anytype) AcceptError!os.socket_t {
-                    return os.accept(op.socket, null, null, op.flags);
+                    const fd = try os.accept(
+                        op.socket,
+                        null,
+                        null,
+                        os.SOCK.NONBLOCK | os.SOCK.CLOEXEC,
+                    );
+                    errdefer os.close(fd);
+
+                    // Darwin doesn't support os.MSG_NOSIGNAL to avoid getting SIGPIPE on socket send().
+                    // Instead, it uses the SO_NOSIGPIPE socket option which does the same for all send()s.
+                    os.setsockopt(
+                        fd,
+                        os.SOL.SOCKET,
+                        os.SO.NOSIGPIPE,
+                        &mem.toBytes(@as(c_int, 1)),
+                    ) catch |err| return switch (err) {
+                        error.TimeoutTooBig => unreachable,
+                        error.PermissionDenied => error.NetworkSubsystemFailed,
+                        error.AlreadyConnected => error.NetworkSubsystemFailed,
+                        error.InvalidProtocolOption => error.ProtocolFailure,
+                        else => |e| e,
+                    };
+
+                    return fd;
                 }
             },
         );
@@ -347,10 +370,10 @@ pub const IO = struct {
             struct {
                 fn doOperation(op: anytype) CloseError!void {
                     return switch (os.errno(os.system.close(op.fd))) {
-                        0 => {},
-                        os.EBADF => error.FileDescriptorInvalid,
-                        os.EINTR => {}, // A success, see https://github.com/ziglang/zig/issues/2425
-                        os.EIO => error.InputOutput,
+                        .SUCCESS => {},
+                        .BADF => error.FileDescriptorInvalid,
+                        .INTR => {}, // A success, see https://github.com/ziglang/zig/issues/2425
+                        .IO => error.InputOutput,
                         else => |errno| os.unexpectedErrno(errno),
                     };
                 }
@@ -399,15 +422,7 @@ pub const IO = struct {
         );
     }
 
-    pub const FsyncError = error{
-        FileDescriptorInvalid,
-        DiskQuota,
-        ArgumentsInvalid,
-        InputOutput,
-        NoSpaceLeft,
-        ReadOnlyFileSystem,
-        AccessDenied,
-    } || os.UnexpectedError;
+    pub const FsyncError = os.SyncError;
 
     pub fn fsync(
         self: *IO,
@@ -420,7 +435,6 @@ pub const IO = struct {
         ) void,
         completion: *Completion,
         fd: os.fd_t,
-        flags: u32,
     ) void {
         self.submit(
             context,
@@ -429,68 +443,10 @@ pub const IO = struct {
             .fsync,
             .{
                 .fd = fd,
-                .flags = flags,
             },
             struct {
                 fn doOperation(op: anytype) FsyncError!void {
-                    _ = os.fcntl(op.fd, os.F_FULLFSYNC, 1) catch return os.fsync(op.fd);
-                }
-            },
-        );
-    }
-
-    pub const OpenatError = error{
-        AccessDenied,
-        FileDescriptorInvalid,
-        DeviceBusy,
-        PathAlreadyExists,
-        FileTooBig,
-        ArgumentsInvalid,
-        IsDir,
-        SymLinkLoop,
-        ProcessFdQuotaExceeded,
-        NameTooLong,
-        SystemFdQuotaExceeded,
-        NoDevice,
-        FileNotFound,
-        SystemResources,
-        NoSpaceLeft,
-        NotDir,
-        FileLocksNotSupported,
-        BadPathName,
-        InvalidUtf8,
-        WouldBlock,
-    } || os.UnexpectedError;
-
-    pub fn openat(
-        self: *IO,
-        comptime Context: type,
-        context: Context,
-        comptime callback: fn (
-            context: Context,
-            completion: *Completion,
-            result: OpenatError!os.fd_t,
-        ) void,
-        completion: *Completion,
-        fd: os.fd_t,
-        path: [*:0]const u8,
-        flags: u32,
-        mode: os.mode_t,
-    ) void {
-        self.submit(
-            context,
-            callback,
-            completion,
-            .openat,
-            .{
-                .fd = fd,
-                .path = path,
-                .mode = mode,
-                .flags = flags,
-            },
-            struct {
-                fn doOperation(op: anytype) OpenatError!os.fd_t {
-                    return os.openatZ(op.fd, op.path, op.flags, op.mode);
+                    _ = os.fcntl(op.fd, os.F.FULLFSYNC, 1) catch return os.fsync(op.fd);
                 }
             },
         );
@@ -542,20 +498,20 @@ pub const IO = struct {
                             @bitCast(isize, op.offset),
                         );
                         return switch (os.errno(rc)) {
-                            0 => @intCast(usize, rc),
-                            os.EINTR => continue,
-                            os.EAGAIN => error.WouldBlock,
-                            os.EBADF => error.NotOpenForReading,
-                            os.ECONNRESET => error.ConnectionResetByPeer,
-                            os.EFAULT => unreachable,
-                            os.EINVAL => error.Alignment,
-                            os.EIO => error.InputOutput,
-                            os.EISDIR => error.IsDir,
-                            os.ENOBUFS => error.SystemResources,
-                            os.ENOMEM => error.SystemResources,
-                            os.ENXIO => error.Unseekable,
-                            os.EOVERFLOW => error.Unseekable,
-                            os.ESPIPE => error.Unseekable,
+                            .SUCCESS => @intCast(usize, rc),
+                            .INTR => continue,
+                            .AGAIN => error.WouldBlock,
+                            .BADF => error.NotOpenForReading,
+                            .CONNRESET => error.ConnectionResetByPeer,
+                            .FAULT => unreachable,
+                            .INVAL => error.Alignment,
+                            .IO => error.InputOutput,
+                            .ISDIR => error.IsDir,
+                            .NOBUFS => error.SystemResources,
+                            .NOMEM => error.SystemResources,
+                            .NXIO => error.Unseekable,
+                            .OVERFLOW => error.Unseekable,
+                            .SPIPE => error.Unseekable,
                             else => |err| os.unexpectedErrno(err),
                         };
                     }
@@ -578,7 +534,6 @@ pub const IO = struct {
         completion: *Completion,
         socket: os.socket_t,
         buffer: []u8,
-        flags: u32,
     ) void {
         self.submit(
             context,
@@ -589,11 +544,10 @@ pub const IO = struct {
                 .socket = socket,
                 .buf = buffer.ptr,
                 .len = @intCast(u32, buffer_limit(buffer.len)),
-                .flags = flags,
             },
             struct {
                 fn doOperation(op: anytype) RecvError!usize {
-                    return os.recv(op.socket, op.buf[0..op.len], op.flags);
+                    return os.recv(op.socket, op.buf[0..op.len], 0);
                 }
             },
         );
@@ -613,7 +567,6 @@ pub const IO = struct {
         completion: *Completion,
         socket: os.socket_t,
         buffer: []const u8,
-        flags: u32,
     ) void {
         self.submit(
             context,
@@ -624,11 +577,10 @@ pub const IO = struct {
                 .socket = socket,
                 .buf = buffer.ptr,
                 .len = @intCast(u32, buffer_limit(buffer.len)),
-                .flags = flags,
             },
             struct {
                 fn doOperation(op: anytype) SendError!usize {
-                    return os.send(op.socket, op.buf[0..op.len], op.flags);
+                    return os.send(op.socket, op.buf[0..op.len], 0);
                 }
             },
         );
@@ -698,4 +650,13 @@ pub const IO = struct {
             },
         );
     }
+
+    pub fn openSocket(family: u32, sock_type: u32, protocol: u32) !os.socket_t {
+        const fd = try os.socket(family, sock_type | os.SOCK.NONBLOCK, protocol);
+        errdefer os.close(fd);
+
+        // darwin doesn't support os.MSG_NOSIGNAL, but instead a socket option to avoid SIGPIPE.
+        try os.setsockopt(fd, os.SOL.SOCKET, os.SO.NOSIGPIPE, &mem.toBytes(@as(c_int, 1)));
+        return fd;
+    }
 };