tigerbeetle-node 0.5.2 → 0.6.0

package/README.md

@@ -9,13 +9,12 @@ The following steps will install the `tigerbeetle-node` module to your current w
 * NodeJS >= `14.0.0`. _(If the correct version is not installed, an installation error will occur)_ 
 
 > Your operating system should be Linux (kernel >= v5.6) or macOS. Windows support is not yet available but is in the works.
-
-> Ensure you are not currently in the project (tigerbeetle-node) directory, but the `parent` directory instead;
      
 ### YARN Package Manager
 
 ```sh
-yarn add tigerbeetle-node
+# Run the following from this directory
+yarn add
 ```
 or
 
@@ -146,7 +145,7 @@ const errors = await client.createTransfers([transfer])
 Two-phase transfers are supported natively by toggling the appropriate flag. TigerBeetle will then adjust the `credits_reserved` and `debits_reserved` fields of the appropriate accounts. A corresponding commit transfer then needs to be sent to accept or reject the transfer.
 | bit 0    | bit 1              | bit 2            |
 |----------|--------------------|------------------|
-| `linked` | `two_phase_commit` | `condition`      |
+| `linked` | `posting`          | `condition`      |
 
 The `condition` flag signals to TigerBeetle that a 256-bit cryptographic condition will be supplied in the `reserved` field. This will be validated against a supplied pre-image when the transfer is committed. Transfers within a batch may also be linked (see [linked events](#linked-events)).
 ```js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tigerbeetle-node",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "TigerBeetle Node.js client",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",

package/src/node.zig

@@ -579,12 +579,7 @@ fn request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_valu
         translate.throw(env, "Unknown operation.") catch return null;
     }
 
-    const message = context.client.get_message() orelse {
-        translate.throw(
-            env,
-            "Too many requests outstanding.",
-        ) catch return null;
-    };
+    const message = context.client.get_message();
     defer context.client.unref(message);
 
     const operation = @intToEnum(Operation, @intCast(u8, operation_int));
@@ -634,12 +629,7 @@ fn raw_request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_
     }
     const operation = @intToEnum(Operation, @intCast(u8, operation_int));
 
-    const message = context.client.get_message() orelse {
-        translate.throw(
-            env,
-            "Too many requests outstanding.",
-        ) catch return null;
-    };
+    const message = context.client.get_message();
     defer context.client.unref(message);
 
     const body_length = translate.bytes_from_buffer(

package/src/tigerbeetle/scripts/benchmark.bat

@@ -0,0 +1,46 @@
+@echo off
+setlocal enabledelayedexpansion
+
+if "%~1" equ ":main" (
+    shift /1
+    goto main
+)
+
+cmd /d /c "%~f0" :main %*
+set ZIG_RESULT=%ERRORLEVEL%
+taskkill /F /IM tigerbeetle.exe >nul
+
+if !ZIG_RESULT! equ 0 (
+    del /f benchmark.log
+) else (
+    echo.
+    echo Error running benchmark, here are more details
+    type benchmark.log
+)
+
+echo.
+exit /b
+
+:main
+zig\zig.exe build -Drelease-safe
+move zig-out\bin\tigerbeetle.exe . >nul
+
+for /l %%i in (0, 1, 0) do (
+    echo Initializing replica %%i
+    set ZIG_FILE=.\cluster_0000000000_replica_00%%i.tigerbeetle
+    if exist "!ZIG_FILE!" DEL /F "!ZIG_FILE!"
+    .\tigerbeetle.exe init --directory=. --cluster=0 --replica=%%i > benchmark.log 2>&1
+)
+
+for /l %%i in (0, 1, 0) do (
+    echo Starting replica %%i
+    start /B "tigerbeetle_%%i" .\tigerbeetle.exe start --directory=. --cluster=0 --addresses=3001 --replica=%%i > benchmark.log 2>&1
+)
+
+rem Wait for replicas to start, listen and connect:
+timeout /t 2
+
+echo.
+echo Benchmarking...
+zig\zig.exe run -OReleaseSafe src\benchmark.zig
+exit /b %errorlevel%

package/src/tigerbeetle/scripts/install_zig.bat

@@ -1,6 +1,6 @@
 @echo off
 
-set ZIG_RELEASE_DEFAULT=0.9.0
+set ZIG_RELEASE_DEFAULT=0.9.1
 
 :: Determine the Zig build:
 if "%~1"=="" (
@@ -39,7 +39,7 @@ if "%ZIG_RELEASE%"=="builds" (
 )
 
 :: Using variable modifiers to determine the directory and filename from the URL:
-:: %~ni Expands %i to a file name only and %~xi Expands %i to a file name extension only.
+:: %%~ni Expands %%i to a file name only and %%~xi Expands %%i to a file name extension only.
 for /f %%i in ("%ZIG_URL%") do (
     set ZIG_DIRECTORY=%%~ni
     set ZIG_TARBALL=%%~nxi

package/src/tigerbeetle/scripts/install_zig.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-ZIG_RELEASE_DEFAULT="0.9.0"
+ZIG_RELEASE_DEFAULT="0.9.1"
 
 # Default to the release build, or allow the latest dev build, or an explicit release version:
 if [ -z "$1" ]; then

package/src/tigerbeetle/scripts/vopr.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
-# Install Zig 0.8.0 if it does not already exist:
+# Install Zig if it does not already exist:
 if [ ! -d "zig" ]; then
-    scripts/install_zig.sh 0.8.0
+    scripts/install_zig.sh
     echo ""
     echo "Running the TigerBeetle VOPR for the first time..."
     echo "Visit https://www.tigerbeetle.com"

package/src/tigerbeetle/src/benchmark.zig

@@ -228,9 +228,7 @@ const TimedQueue = struct {
         if (self.batches.head_ptr()) |starting_batch| {
             log.debug("sending first batch...", .{});
             self.batch_start = now;
-            var message = self.client.get_message() orelse {
-                @panic("Client message pool has been exhausted. Cannot execute batch.");
-            };
+            const message = self.client.get_message();
             defer self.client.unref(message);
 
             std.mem.copy(
@@ -288,9 +286,7 @@ const TimedQueue = struct {
         }
 
         if (self.batches.head_ptr()) |next_batch| {
-            var message = self.client.get_message() orelse {
-                @panic("Client message pool has been exhausted.");
-            };
+            const message = self.client.get_message();
             defer self.client.unref(message);
 
             std.mem.copy(

package/src/tigerbeetle/src/cli.zig

@@ -8,6 +8,7 @@ const os = std.os;
 
 const config = @import("config.zig");
 const vsr = @import("vsr.zig");
+const IO = @import("io.zig").IO;
 
 const usage = fmt.comptimePrint(
     \\Usage:
@@ -87,18 +88,31 @@ pub const Command = union(enum) {
 
 /// Parse the command line arguments passed to the tigerbeetle binary.
 /// Exits the program with a non-zero exit code if an error is found.
-pub fn parse_args(allocator: std.mem.Allocator) Command {
+pub fn parse_args(allocator: std.mem.Allocator) !Command {
     var maybe_cluster: ?[]const u8 = null;
     var maybe_replica: ?[]const u8 = null;
     var maybe_addresses: ?[]const u8 = null;
     var maybe_directory: ?[:0]const u8 = null;
 
-    var args = std.process.args();
+    var args = try std.process.argsWithAllocator(allocator);
+    defer args.deinit();
+
+    // Keep track of the args from the ArgIterator above that were allocated
+    // then free them all at the end of the scope.
+    var args_allocated = std.ArrayList([:0]const u8).init(allocator);
+    defer {
+        for (args_allocated.items) |arg| allocator.free(arg);
+        args_allocated.deinit();
+    }
+
     // Skip argv[0] which is the name of this executable
-    _ = args.nextPosix();
+    const did_skip = args.skip();
+    assert(did_skip);
+
+    const raw_command = try (args.next(allocator) orelse
+        fatal("no command provided, expected 'start' or 'init'", .{}));
+    defer allocator.free(raw_command);
 
-    const raw_command = args.nextPosix() orelse
-        fatal("no command provided, expected 'start' or 'init'", .{});
     if (mem.eql(u8, raw_command, "-h") or mem.eql(u8, raw_command, "--help")) {
         std.io.getStdOut().writeAll(usage) catch os.exit(1);
         os.exit(0);
@@ -106,7 +120,10 @@ pub fn parse_args(allocator: std.mem.Allocator) Command {
     const command = meta.stringToEnum(meta.Tag(Command), raw_command) orelse
         fatal("unknown command '{s}', expected 'start' or 'init'", .{raw_command});
 
-    while (args.nextPosix()) |arg| {
+    while (args.next(allocator)) |parsed_arg| {
+        const arg = try parsed_arg;
+        try args_allocated.append(arg);
+
         if (mem.startsWith(u8, arg, "--cluster")) {
             maybe_cluster = parse_flag("--cluster", arg);
         } else if (mem.startsWith(u8, arg, "--replica")) {
@@ -132,7 +149,7 @@ pub fn parse_args(allocator: std.mem.Allocator) Command {
     const replica = parse_replica(raw_replica);
 
     const dir_path = maybe_directory orelse config.directory;
-    const dir_fd = os.openZ(dir_path, os.O.CLOEXEC | os.O.RDONLY, 0) catch |err|
+    const dir_fd = IO.open_dir(dir_path) catch |err|
         fatal("failed to open directory '{s}': {}", .{ dir_path, err });
 
     switch (command) {
@@ -141,11 +158,13 @@ pub fn parse_args(allocator: std.mem.Allocator) Command {
                 fatal("--addresses: supported only by 'start' command", .{});
             }
 
-            return .{ .init = .{
-                .cluster = cluster,
-                .replica = replica,
-                .dir_fd = dir_fd,
-            } };
+            return Command{
+                .init = .{
+                    .cluster = cluster,
+                    .replica = replica,
+                    .dir_fd = dir_fd,
+                },
+            };
         },
         .start => {
             const raw_addresses = maybe_addresses orelse
@@ -156,12 +175,14 @@ pub fn parse_args(allocator: std.mem.Allocator) Command {
                 fatal("--replica: value greater than length of --addresses array", .{});
             }
 
-            return .{ .start = .{
-                .cluster = cluster,
-                .replica = replica,
-                .addresses = addresses,
-                .dir_fd = dir_fd,
-            } };
+            return Command{
+                .start = .{
+                    .cluster = cluster,
+                    .replica = replica,
+                    .addresses = addresses,
+                    .dir_fd = dir_fd,
+                },
+            };
         },
     }
 }

package/src/tigerbeetle/src/config.zig

@@ -1,3 +1,6 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Whether development or production:
 pub const deployment_environment = .development;
 
@@ -86,12 +89,6 @@ pub const connections_max = replicas_max + clients_max;
 /// This impacts the amount of memory allocated at initialization by the server.
 pub const message_size_max = 1 * 1024 * 1024;
 
-/// The number of full-sized messages allocated at initialization by the message bus.
-pub const message_bus_messages_max = connections_max * 4;
-/// The number of header-sized messages allocated at initialization by the message bus.
-/// These are much smaller/cheaper and we can therefore have many of them.
-pub const message_bus_headers_max = connections_max * connection_send_queue_max * 2;
-
 /// The maximum number of Viewstamped Replication prepare messages that can be inflight at a time.
 /// This is immutable once assigned per cluster, as replicas need to know how many operations might
 /// possibly be uncommitted during a view change, and this must be constant for all replicas.
@@ -102,8 +99,15 @@ pub const pipelining_max = clients_max;
 pub const connection_delay_min_ms = 50;
 pub const connection_delay_max_ms = 1000;
 
-/// The maximum number of outgoing messages that may be queued on a connection.
-pub const connection_send_queue_max = pipelining_max;
+/// The maximum number of outgoing messages that may be queued on a replica connection.
+pub const connection_send_queue_max_replica = std.math.max(std.math.min(clients_max, 4), 2);
+
+/// The maximum number of outgoing messages that may be queued on a client connection.
+/// The client has one in-flight request, and occasionally a ping.
+pub const connection_send_queue_max_client = 2;
+
+/// The maximum number of outgoing requests that may be queued on a client (including the in-flight request).
+pub const client_request_queue_max = 32;
 
 /// The maximum number of connections in the kernel's complete connection queue pending an accept():
 /// If the backlog argument is greater than the value in `/proc/sys/net/core/somaxconn`, then it is
@@ -124,7 +128,8 @@ pub const tcp_rcvbuf = 4 * 1024 * 1024;
 /// This sets SO_SNDBUF as an alternative to the auto-tuning range in /proc/sys/net/ipv4/tcp_wmem.
 /// The value is limited by /proc/sys/net/core/wmem_max, unless the CAP_NET_ADMIN privilege exists.
 /// The kernel doubles this value to allow space for packet bookkeeping overhead.
-pub const tcp_sndbuf = 4 * 1024 * 1024;
+pub const tcp_sndbuf_replica = connection_send_queue_max_replica * message_size_max;
+pub const tcp_sndbuf_client = connection_send_queue_max_client * message_size_max;
 
 /// Whether to enable TCP keepalive:
 pub const tcp_keepalive = true;
@@ -218,3 +223,13 @@ pub const clock_synchronization_window_min_ms = 2000;
 /// This eliminates the impact of gradual clock drift on our clock offset (clock skew) measurements.
 /// If a window expires because of this then it is likely that the clock epoch will also be expired.
 pub const clock_synchronization_window_max_ms = 20000;
+
+comptime {
+    // vsr.parse_address assumes that config.address/config.port are valid.
+    _ = std.net.Address.parseIp4(address, 0) catch unreachable;
+    _ = @as(u16, port);
+
+    // Avoid latency issues from a too-large sndbuf.
+    assert(tcp_sndbuf_replica <= 4 * 1024 * 1024);
+    assert(tcp_sndbuf_client <= 4 * 1024 * 1024);
+}

package/src/tigerbeetle/src/demo.zig

@@ -53,7 +53,7 @@ pub fn request(
 
     message_bus.set_on_message(*Client, &client, Client.on_message);
 
-    var message = client.get_message() orelse unreachable;
+    var message = client.get_message();
     defer client.unref(message);
 
     const body = std.mem.asBytes(&batch);

package/src/tigerbeetle/src/io/benchmark.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const os = std.os;
 const assert = std.debug.assert;
+const log = std.log.scoped(.io_benchmark);
 
 const Time = @import("../time.zig").Time;
 const IO = @import("../io.zig").IO;
@@ -14,7 +15,7 @@ const run_duration = 1 * std.time.ns_per_s;
 
 pub fn main() !void {
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
-    const allocator = &gpa.allocator;
+    const allocator = gpa.allocator();
     defer {
         const leaks = gpa.deinit();
         assert(!leaks);
@@ -24,37 +25,34 @@ pub fn main() !void {
     defer allocator.free(buffer);
     std.mem.set(u8, buffer, 0);
 
-    var timer = Time{};
-    const started = timer.monotonic();
     var self = Context{
         .io = try IO.init(32, 0),
-        .timer = &timer,
-        .started = started,
-        .current = started,
         .tx = .{ .buffer = buffer[0 * buffer_size ..][0..buffer_size] },
         .rx = .{ .buffer = buffer[1 * buffer_size ..][0..buffer_size] },
     };
+    defer self.io.deinit();
 
+    var timer = Time{};
+    const started = timer.monotonic();
     defer {
-        self.io.deinit();
-        const elapsed_ns = self.current - started;
+        const elapsed_ns = timer.monotonic() - started;
         const transferred_mb = @intToFloat(f64, self.transferred) / 1024 / 1024;
 
-        std.debug.print("IO throughput test: took {}ms @ {d:.2} MB/s\n", .{
+        log.info("took {}ms @ {d:.2} MB/s\n", .{
             elapsed_ns / std.time.ns_per_ms,
             transferred_mb / (@intToFloat(f64, elapsed_ns) / std.time.ns_per_s),
         });
     }
 
     // Setup the server socket
-    self.server.fd = try IO.openSocket(os.AF_INET, os.SOCK_STREAM, os.IPPROTO_TCP);
+    self.server.fd = try self.io.open_socket(os.AF.INET, os.SOCK.STREAM, os.IPPROTO.TCP);
     defer os.closeSocket(self.server.fd);
 
     const address = try std.net.Address.parseIp4("127.0.0.1", 3131);
     try os.setsockopt(
         self.server.fd,
-        os.SOL_SOCKET,
-        os.SO_REUSEADDR,
+        os.SOL.SOCKET,
+        os.SO.REUSEADDR,
         &std.mem.toBytes(@as(c_int, 1)),
     );
     try os.bind(self.server.fd, &address.any, address.getOsSockLen());
@@ -70,7 +68,7 @@ pub fn main() !void {
     );
 
     // Setup the client connection
-    self.tx.socket.fd = try IO.openSocket(os.AF_INET, os.SOCK_STREAM, os.IPPROTO_TCP);
+    self.tx.socket.fd = try self.io.open_socket(os.AF.INET, os.SOCK.STREAM, os.IPPROTO.TCP);
     defer os.closeSocket(self.tx.socket.fd);
 
     self.io.connect(
@@ -82,22 +80,14 @@ pub fn main() !void {
         address,
     );
 
-    // Run the IO loop, calling either tick() or run_for_ns() at "pseudo-random"
-    // to benchmark each io-driving execution path
-    var tick: usize = 0xdeadbeef;
-    while (self.is_running()) : (tick +%= 1) {
-        if (tick % 61 == 0) {
-            const timeout_ns = tick % (10 * std.time.ns_per_ms);
-            try self.io.run_for_ns(@intCast(u63, timeout_ns));
-        } else {
-            try self.io.tick();
-        }
-    }
+    // Run the IO loop for the duration of the benchmark
+    log.info("running for {}", .{std.fmt.fmtDuration(run_duration)});
+    try self.io.run_for_ns(run_duration);
 
     // Assert that everything is connected
-    assert(self.server.fd != -1);
-    assert(self.tx.socket.fd != -1);
-    assert(self.rx.socket.fd != -1);
+    assert(self.server.fd != IO.INVALID_SOCKET);
+    assert(self.tx.socket.fd != IO.INVALID_SOCKET);
+    assert(self.rx.socket.fd != IO.INVALID_SOCKET);
 
     // Close the accepted client socket.
     // The actual client socket + server socket are closed by defer
@@ -108,14 +98,11 @@ const Context = struct {
     io: IO,
     tx: Pipe,
     rx: Pipe,
-    timer: *Time,
-    started: u64,
-    current: u64,
     server: Socket = .{},
     transferred: u64 = 0,
 
     const Socket = struct {
-        fd: os.socket_t = -1,
+        fd: os.socket_t = IO.INVALID_SOCKET,
         completion: IO.Completion = undefined,
     };
     const Pipe = struct {
@@ -124,21 +111,12 @@ const Context = struct {
         transferred: usize = 0,
     };
 
-    fn is_running(self: Context) bool {
-        // Make sure that we're connected
-        if (self.rx.socket.fd == -1) return true;
-
-        // Make sure that we haven't run too long as configured
-        const elapsed = self.current - self.started;
-        return elapsed < run_duration;
-    }
-
     fn on_accept(
         self: *Context,
         completion: *IO.Completion,
         result: IO.AcceptError!os.socket_t,
     ) void {
-        assert(self.rx.socket.fd == -1);
+        assert(self.rx.socket.fd == IO.INVALID_SOCKET);
         assert(&self.server.completion == completion);
         self.rx.socket.fd = result catch |err| std.debug.panic("accept error {}", .{err});
 
@@ -152,7 +130,9 @@ const Context = struct {
         completion: *IO.Completion,
         result: IO.ConnectError!void,
     ) void {
-        assert(self.tx.socket.fd != -1);
+        _ = result catch unreachable;
+
+        assert(self.tx.socket.fd != IO.INVALID_SOCKET);
         assert(&self.tx.socket.completion == completion);
 
         // Start sending data to the server's accepted client
@@ -161,8 +141,8 @@ const Context = struct {
     }
 
     const TransferType = enum {
-        read = 0,
-        write = 1,
+        read,
+        write,
     };
 
     fn do_transfer(
@@ -188,11 +168,6 @@ const Context = struct {
         assert(bytes <= buffer_size);
         self.transferred += bytes;
 
-        // Check in with the benchmark timer to stop sending/receiving data
-        self.current = self.timer.monotonic();
-        if (!self.is_running())
-            return;
-
         // Select which connection (tx or rx) depending on the type of transfer
         const pipe = &@field(self, pipe_name);
         pipe.transferred += bytes;