tigerbeetle-node 0.10.0 → 0.11.1

package/src/tigerbeetle/scripts/install_zig.sh

@@ -1,17 +1,17 @@
-#!/bin/bash
-set -eEuo pipefail
+#!/usr/bin/env sh
+set -eu
 
 ZIG_RELEASE_DEFAULT="0.9.1"
 # Default to the release build, or allow the latest dev build, or an explicit release version:
 ZIG_RELEASE=${1:-$ZIG_RELEASE_DEFAULT}
-if [ "$ZIG_RELEASE" == "latest" ]; then
+if [ "$ZIG_RELEASE" = "latest" ]; then
     ZIG_RELEASE="builds"
 fi
 
 # Validate the release version explicitly:
-if [[ $ZIG_RELEASE =~ ^builds$ ]]; then
+if echo "$ZIG_RELEASE" | grep -q '^builds$'; then
     echo "Installing Zig latest build..."
-elif [[ $ZIG_RELEASE =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+elif echo "$ZIG_RELEASE" | grep -q '^[0-9]\+.[0-9]\+.[0-9]\+$'; then
     echo "Installing Zig $ZIG_RELEASE release build..."
 else
     echo "Release version invalid"
@@ -19,26 +19,31 @@ else
 fi
 
 # Determine the architecture:
-if [[ $(uname -m) == 'arm64' ]] || [[ $(uname -m) == 'aarch64' ]]; then
+if [ "$(uname -m)" = 'arm64' ] || [ "$(uname -m)" = 'aarch64' ]; then
     ZIG_ARCH="aarch64"
 else
     ZIG_ARCH="x86_64"
 fi
 
 # Determine the operating system:
-if [[ "$OSTYPE" == "darwin"* ]]; then
-    ZIG_OS="macos"
-else
+if [ "$(uname)" = "Linux" ]; then
     ZIG_OS="linux"
+else
+    ZIG_OS="macos"
 fi
 
 ZIG_TARGET="zig-$ZIG_OS-$ZIG_ARCH"
 
 # Determine the build, split the JSON line on whitespace and extract the 2nd field, then remove quotes and commas:
-if command -v wget &> /dev/null; then
+if command -v wget; then
     # -4 forces `wget` to connect to ipv4 addresses, as ipv6 fails to resolve on certain distros.
     # Only A records (for ipv4) are used in DNS:
-    ZIG_URL=$(wget -4 --quiet -O - https://ziglang.org/download/index.json | grep -F "$ZIG_TARGET" | grep -F "$ZIG_RELEASE" | awk '{print $2}' | sed 's/[",]//g')
+    ipv4="-4"
+    # But Alpine doesn't support this argument
+    if [ -f /etc/alpine-release ]; then
+	ipv4=""
+    fi
+    ZIG_URL=$(wget $ipv4 --quiet -O - https://ziglang.org/download/index.json | grep -F "$ZIG_TARGET" | grep -F "$ZIG_RELEASE" | awk '{print $2}' | sed 's/[",]//g')
 else
     ZIG_URL=$(curl --silent https://ziglang.org/download/index.json | grep -F "$ZIG_TARGET" | grep -F "$ZIG_RELEASE" | awk '{print $2}' | sed 's/[",]//g')
 fi
@@ -55,12 +60,17 @@ ZIG_DIRECTORY=$(basename "$ZIG_TARBALL" .tar.xz)
 
 # Download, making sure we download to the same output document, without wget adding "-1" etc. if the file was previously partially downloaded:
 echo "Downloading $ZIG_URL..."
-if command -v wget &> /dev/null; then
+if command -v wget; then
     # -4 forces `wget` to connect to ipv4 addresses, as ipv6 fails to resolve on certain distros.
     # Only A records (for ipv4) are used in DNS:
-    wget -4 --quiet --show-progress --output-document="$ZIG_TARBALL" "$ZIG_URL"
+    ipv4="-4"
+    # But Alpine doesn't support this argument
+    if [ -f /etc/alpine-release ]; then
+	ipv4=""
+    fi
+    wget $ipv4 --quiet --output-document="$ZIG_TARBALL" "$ZIG_URL"
 else
-    curl --silent --progress-bar --output "$ZIG_TARBALL" "$ZIG_URL"
+    curl --silent --output "$ZIG_TARBALL" "$ZIG_URL"
 fi
 
 # Extract and then remove the downloaded tarball:

package/src/tigerbeetle/scripts/pre-commit.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Usage:
+# ln -s ../../scripts/pre-commit.sh .git/hooks/pre-commit
+
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+zig fmt --check .

package/src/tigerbeetle/scripts/shellcheck.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+docker run -v "$(pwd)":/wrk -w /wrk koalaman/shellcheck scripts/*.sh

package/src/tigerbeetle/scripts/tests_on_alpine.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+docker run --entrypoint sh -v "$(pwd)":/wrk -w /wrk alpine -c "
+set -e
+./scripts/install_zig.sh
+zig/zig build test
+./scripts/install.sh
+"

package/src/tigerbeetle/scripts/tests_on_ubuntu.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+docker run --entrypoint sh -v "$(pwd)":/wrk -w /wrk ubuntu -c "
+set -e
+
+apt-get update -y
+apt-get install -y curl xz-utils
+
+./scripts/install_zig.sh
+zig/zig build test
+./scripts/install.sh
+"

package/src/tigerbeetle/scripts/validate_docs.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+# This script builds the docs website for the currently checked out
+# branch.
+
+git clone https://github.com/tigerbeetledb/docs docs_website
+# Try to grab branch from Github Actions CI.
+# See also: https://docs.github.com/en/actions/learn-github-actions/environment-variables.
+BRANCH="$GITHUB_HEAD_REF"
+if [[ -z "$BRANCH" ]]; then
+    # Otherwise fall back to git rev-parse
+    BRANCH=$(git rev-parse --abbrev-ref HEAD)
+fi
+( cd docs_website && npm install && ./scripts/build.sh "$BRANCH" )
+rm -rf docs_website

package/src/tigerbeetle/src/benchmark.zig

@@ -9,10 +9,13 @@ pub const log_level: std.log.Level = .err;
 const cli = @import("cli.zig");
 const IO = @import("io.zig").IO;
 
+const util = @import("util.zig");
 const Storage = @import("storage.zig").Storage;
 const MessagePool = @import("message_pool.zig").MessagePool;
 const MessageBus = @import("message_bus.zig").MessageBusClient;
-const StateMachine = @import("state_machine.zig").StateMachineType(Storage);
+const StateMachine = @import("state_machine.zig").StateMachineType(Storage, .{
+    .message_body_size_max = config.message_body_size_max,
+});
 const RingBuffer = @import("ring_buffer.zig").RingBuffer;
 
 const vsr = @import("vsr.zig");
@@ -38,7 +41,7 @@ var accounts = [_]tb.Account{
         .user_data = 0,
         .reserved = [_]u8{0} ** 48,
         .ledger = 2,
-        .code = 0,
+        .code = 1,
         .flags = .{},
         .debits_pending = 0,
         .debits_posted = 0,
@@ -50,7 +53,7 @@ var accounts = [_]tb.Account{
         .user_data = 0,
         .reserved = [_]u8{0} ** 48,
         .ledger = 2,
-        .code = 0,
+        .code = 1,
         .flags = .{},
         .debits_pending = 0,
         .debits_posted = 0,
@@ -99,20 +102,21 @@ pub fn main() !void {
     // Pre-allocate a million transfers:
     const transfers = try allocator.alloc(tb.Transfer, transfers_max);
     defer allocator.free(transfers);
-    
+
     for (transfers) |*transfer, index| {
         transfer.* = .{
-            .id = index,
+            .id = index + 1,
             .debit_account_id = accounts[0].id,
             .credit_account_id = accounts[1].id,
-            .pending_id = 0,
             .user_data = 0,
             .reserved = 0,
-            .code = 0,
+            .pending_id = 0,
+            .timeout = 0,
             .ledger = 2,
+            .code = 1,
             .flags = .{},
             .amount = 1,
-            .timeout = 0,
+            .timestamp = 0,
         };
     }
 
@@ -207,7 +211,8 @@ const TimedQueue = struct {
             const message = self.client.get_message();
             defer self.client.unref(message);
 
-            std.mem.copy(
+            util.copy_disjoint(
+                .inexact,
                 u8,
                 message.buffer[@sizeOf(vsr.Header)..],
                 std.mem.sliceAsBytes(starting_batch.data),
@@ -238,8 +243,6 @@ const TimedQueue = struct {
             @panic("Client returned error during benchmarking.");
         };
 
-        log.debug("response={s}", .{std.mem.bytesAsSlice(tb.CreateAccountsResult, value)});
-
         const self: *TimedQueue = @intToPtr(*TimedQueue, @intCast(usize, user_data));
         const completed_batch: ?Batch = self.batches.pop();
         assert(completed_batch != null);
@@ -251,8 +254,20 @@ const TimedQueue = struct {
         });
         const latency = now - self.batch_start.?;
         switch (operation) {
-            .create_accounts => {},
+            .create_accounts => {
+                const create_accounts_results = std.mem.bytesAsSlice(tb.CreateAccountsResult, value);
+                if (create_accounts_results.len > 0) {
+                    log.err("CreateAccountsResults={any}", .{create_accounts_results});
+                    @panic("Unexpected result creating accounts.");
+                }
+            },
             .create_transfers => {
+                const create_transfers_results = std.mem.bytesAsSlice(tb.CreateTransfersResult, value);
+                if (create_transfers_results.len > 0) {
+                    log.err("CreateTransfersResults={any}", .{create_transfers_results});
+                    @panic("Unexpected result creating transfers.");
+                }
+
                 if (latency > self.transfers_latency_max) {
                     self.transfers_latency_max = latency;
                 }
@@ -264,7 +279,8 @@ const TimedQueue = struct {
             const message = self.client.get_message();
             defer self.client.unref(message);
 
-            std.mem.copy(
+            util.copy_disjoint(
+                .inexact,
                 u8,
                 message.buffer[@sizeOf(vsr.Header)..],
                 std.mem.sliceAsBytes(next_batch.data),

package/src/tigerbeetle/src/c/tb_client/context.zig

@@ -1,4 +1,22 @@
 const std = @import("std");
+const os = std.os;
+const assert = std.debug.assert;
+
+const config = @import("../../config.zig");
+const log = std.log.scoped(.tb_client_context);
+
+const util = @import("../../util.zig");
+const vsr = @import("../../vsr.zig");
+const Header = vsr.Header;
+
+const IO = @import("../../io.zig").IO;
+const message_pool = @import("../../message_pool.zig");
+
+const MessagePool = message_pool.MessagePool;
+const Message = MessagePool.Message;
+
+const Packet = @import("packet.zig").Packet;
+const Signal = @import("signal.zig").Signal;
 const ThreadType = @import("thread.zig").ThreadType;
 
 const api = @import("../tb_client.zig");
@@ -13,15 +31,64 @@ pub const ContextImplementation = struct {
     deinit_fn: fn (*ContextImplementation) void,
 };
 
+pub const Error = std.mem.Allocator.Error || error{
+    Unexpected,
+    AddressInvalid,
+    AddressLimitExceeded,
+    PacketsCountInvalid,
+    SystemResources,
+    NetworkSubsystemFailed,
+};
+
 pub fn ContextType(
-    comptime StateMachine: type,
-    comptime MessageBus: type,
+    comptime Client: type,
 ) type {
     return struct {
         const Context = @This();
-        const Thread = ThreadType(StateMachine, MessageBus);
+        const Thread = ThreadType(Context);
+
+        const UserData = extern struct {
+            self: *Context,
+            packet: *Packet,
+        };
+
+        comptime {
+            assert(@sizeOf(UserData) == @sizeOf(u128));
+        }
+
+        fn operation_event_size(op: u8) ?usize {
+            const allowed_operations = [_]Client.StateMachine.Operation{
+                .create_accounts,
+                .create_transfers,
+                .lookup_accounts,
+                .lookup_transfers,
+            };
+
+            inline for (allowed_operations) |operation| {
+                if (op == @enumToInt(operation)) {
+                    return @sizeOf(Client.StateMachine.Event(operation));
+                }
+            }
+
+            return null;
+        }
+
+        const PacketError = Client.Error || error{
+            TooMuchData,
+            InvalidOperation,
+            InvalidDataSize,
+        };
 
         allocator: std.mem.Allocator,
+        client_id: u128,
+        packets: []Packet,
+
+        addresses: []const std.net.Address,
+        io: IO,
+        message_pool: MessagePool,
+        messages_available: usize,
+        client: Client,
+
         on_completion_ctx: usize,
         on_completion_fn: tb_completion_t,
         implementation: ContextImplementation,
@@ -29,51 +96,199 @@ pub fn ContextType(
 
         pub fn init(
             allocator: std.mem.Allocator,
-            out_tb_client: *tb_client_t,
-            out_packets: *tb_packet_list_t,
             cluster_id: u32,
-            addresses_ptr: [*:0]const u8,
-            addresses_len: u32,
-            num_packets: u32,
+            addresses: []const u8,
+            packets_count: u32,
             on_completion_ctx: usize,
             on_completion_fn: tb_completion_t,
-        ) tb_status_t {
-            const context = allocator.create(Context) catch return .out_of_memory;
+        ) Error!*Context {
+            var context = try allocator.create(Context);
+            errdefer allocator.destroy(context);
+
             context.allocator = allocator;
+            context.client_id = std.crypto.random.int(u128);
+            log.debug("{}: init: initializing", .{context.client_id});
+
+            const packets_count_max = 4096;
+            if (packets_count > packets_count_max) {
+                return error.PacketsCountInvalid;
+            }
+
+            log.debug("{}: init: allocating tb_packets", .{context.client_id});
+            context.packets = try context.allocator.alloc(Packet, packets_count);
+            errdefer context.allocator.free(context.packets);
+
+            log.debug("{}: init: parsing vsr addresses: {s}", .{ context.client_id, addresses });
+            context.addresses = vsr.parse_addresses(
+                context.allocator,
+                addresses,
+                config.replicas_max,
+            ) catch |err| return switch (err) {
+                error.AddressLimitExceeded => error.AddressLimitExceeded,
+                else => error.AddressInvalid,
+            };
+            errdefer context.allocator.free(context.addresses);
+
+            log.debug("{}: init: initializing IO", .{context.client_id});
+            context.io = IO.init(32, 0) catch |err| {
+                log.err("{}: failed to initialize IO: {s}", .{
+                    context.client_id,
+                    @errorName(err),
+                });
+                return switch (err) {
+                    error.ProcessFdQuotaExceeded => error.SystemResources,
+                    error.Unexpected => error.Unexpected,
+                    else => unreachable,
+                };
+            };
+            errdefer context.io.deinit();
+
+            log.debug("{}: init: initializing MessagePool", .{context.client_id});
+            context.message_pool = try MessagePool.init(allocator, .client);
+            errdefer context.message_pool.deinit(context.allocator);
+
+            log.debug("{}: init: initializing client (cluster_id={}, addresses={any})", .{
+                context.client_id,
+                cluster_id,
+                context.addresses,
+            });
+            context.client = try Client.init(
+                allocator,
+                context.client_id,
+                cluster_id,
+                @intCast(u8, context.addresses.len),
+                &context.message_pool,
+                .{
+                    .configuration = context.addresses,
+                    .io = &context.io,
+                },
+            );
+            errdefer context.client.deinit(context.allocator);
+
+            context.messages_available = config.client_request_queue_max;
             context.on_completion_ctx = on_completion_ctx;
             context.on_completion_fn = on_completion_fn;
-
-            out_tb_client.* = api.context_to_client(&context.implementation);
             context.implementation = .{
                 .submit_fn = Context.on_submit,
                 .deinit_fn = Context.on_deinit,
             };
 
-            const addresses = @ptrCast([*]const u8, addresses_ptr)[0..addresses_len];
-            context.thread.init(
-                allocator,
-                cluster_id,
-                addresses,
-                num_packets,
-                Context.on_completion,
-            ) catch |err| {
-                allocator.destroy(context);
-                return switch (err) {
-                    error.Unexpected => .unexpected,
-                    error.OutOfMemory => .out_of_memory,
-                    error.InvalidAddress => .invalid_address,
-                    error.SystemResources => .system_resources,
-                    error.NetworkSubsystemFailed => .network_subsystem,
+            log.debug("{}: init: initializing thread", .{context.client_id});
+            try context.thread.init(context);
+            errdefer context.thread.deinit(context.allocator);
+
+            return context;
+        }
+
+        pub fn deinit(self: *Context) void {
+            self.thread.deinit();
+
+            self.client.deinit(self.allocator);
+            self.message_pool.deinit(self.allocator);
+            self.io.deinit();
+
+            self.allocator.free(self.addresses);
+            self.allocator.free(self.packets);
+            self.allocator.destroy(self);
+        }
+
+        pub fn tick(self: *Context) void {
+            self.client.tick();
+        }
+
+        pub fn run(self: *Context) void {
+            while (!self.thread.signal.is_shutdown()) {
+                self.tick();
+                self.io.run_for_ns(config.tick_ms * std.time.ns_per_ms) catch |err| {
+                    log.err("{}: IO.run() failed: {s}", .{
+                        self.client_id,
+                        @errorName(err),
+                    });
+                    @panic("IO.run() failed");
                 };
+            }
+        }
+
+        pub fn request(self: *Context, packet: *Packet) void {
+            const message = self.message_pool.get_message();
+            defer self.message_pool.unref(message);
+            self.messages_available -= 1;
+
+            // Get the size of each request structure in the packet.data:
+            const event_size: usize = operation_event_size(packet.operation) orelse {
+                return self.on_complete(packet, error.InvalidOperation);
             };
 
-            var list = tb_packet_list_t{};
-            for (context.thread.packets) |*packet| {
-                list.push(tb_packet_list_t.from(packet));
+            // Make sure the packet.data size is correct:
+            const readable = @ptrCast([*]const u8, packet.data)[0..packet.data_size];
+            if (readable.len == 0 or readable.len % event_size != 0) {
+                return self.on_complete(packet, error.InvalidDataSize);
             }
 
-            out_packets.* = list;
-            return .success;
+            // Make sure the packet.data wouldn't overflow a message:
+            const writable = message.buffer[@sizeOf(Header)..][0..config.message_body_size_max];
+            if (readable.len > writable.len) {
+                return self.on_complete(packet, error.TooMuchData);
+            }
+
+            // Write the packet data to the message:
+            util.copy_disjoint(.inexact, u8, writable, readable);
+            const wrote = readable.len;
+
+            // Submit the message for processing:
+            self.client.request(
+                @bitCast(u128, UserData{
+                    .self = self,
+                    .packet = packet,
+                }),
+                Context.on_result,
+                @intToEnum(Client.StateMachine.Operation, packet.operation),
+                message,
+                wrote,
+            );
+        }
+
+        fn on_result(
+            raw_user_data: u128,
+            op: Client.StateMachine.Operation,
+            results: Client.Error![]const u8,
+        ) void {
+            const user_data = @bitCast(UserData, raw_user_data);
+            const self = user_data.self;
+            const packet = user_data.packet;
+
+            assert(packet.operation == @enumToInt(op));
+            self.on_complete(packet, results);
+        }
+
+        fn on_complete(
+            self: *Context,
+            packet: *Packet,
+            result: PacketError![]const u8,
+        ) void {
+            self.messages_available += 1;
+            assert(self.messages_available <= config.client_request_queue_max);
+
+            // Signal to resume sending requests that was waiting for available messages.
+            if (self.messages_available == 1) self.thread.signal.notify();
+
+            const tb_client = api.context_to_client(&self.implementation);
+            const bytes = result catch |err| {
+                packet.status = switch (err) {
+                    // If there's too many requests, (re)try submitting the packet later.
+                    error.TooManyOutstandingRequests => {
+                        return self.thread.retry.push(Packet.List.from(packet));
+                    },
+                    error.TooMuchData => .too_much_data,
+                    error.InvalidOperation => .invalid_operation,
+                    error.InvalidDataSize => .invalid_data_size,
+                };
+                return (self.on_completion_fn)(self.on_completion_ctx, tb_client, packet, null, 0);
+            };
+
+            // The packet completed normally.
+            packet.status = .ok;
+            (self.on_completion_fn)(self.on_completion_ctx, tb_client, packet, bytes.ptr, @intCast(u32, bytes.len));
         }
 
         fn on_submit(implementation: *ContextImplementation, packets: *tb_packet_list_t) void {
@@ -83,21 +298,7 @@ pub fn ContextType(
 
         fn on_deinit(implementation: *ContextImplementation) void {
             const context = @fieldParentPtr(Context, "implementation", implementation);
-            context.thread.deinit();
-            context.allocator.destroy(context);
-        }
-
-        fn on_completion(thread: *Thread, packet: *tb_packet_t, result: ?[]const u8) void {
-            const context = @fieldParentPtr(Context, "thread", thread);
-            const tb_client = api.context_to_client(&context.implementation);
-
-            context.on_completion_fn(
-                context.on_completion_ctx,
-                tb_client,
-                packet,
-                if (result) |r| r.ptr else null,
-                if (result) |r| @intCast(u32, r.len) else 0,
-            );
+            context.deinit();
         }
     };
 }