tigerbeetle-node 0.8.1 → 0.9.143

package/src/tigerbeetle/scripts/vr_state_enumerate

@@ -1,46 +0,0 @@
-#!/usr/bin/env node
-
-const messages = [
-  'prepare',
-  'prepare_ok',
-  'commit',
-  'request_state_transfer',
-  'state_transfer',
-  'start_view_change',
-  'do_view_change',
-  'start_view'
-];
-
-const views = [
-  'old_view',
-  'same_view',
-  'new_view'
-];
-
-const statuses = [
-  'normal',
-  'view_change',
-  'recovering'
-];
-
-const tuples = {};
-
-statuses.forEach(
-  function(status) {
-    views.forEach(
-      function(view) {
-        messages.forEach(
-          function(message) {
-            tuples[`${message}, ${view}, ${status}`] = true;
-          }
-        );
-      }
-    );
-  }
-);
-
-Object.keys(tuples).sort().forEach(
-  function(key) {
-    console.log(key);
-  }
-);

package/src/tigerbeetle/src/benchmark.zig

@@ -1,290 +0,0 @@
-const std = @import("std");
-const builtin = @import("builtin");
-const assert = std.debug.assert;
-const config = @import("config.zig");
-
-const log = std.log;
-pub const log_level: std.log.Level = .err;
-
-const cli = @import("cli.zig");
-const IO = @import("io.zig").IO;
-
-const MessageBus = @import("message_bus.zig").MessageBusClient;
-const StateMachine = @import("state_machine.zig").StateMachine;
-const RingBuffer = @import("ring_buffer.zig").RingBuffer;
-
-const vsr = @import("vsr.zig");
-const Client = vsr.Client(StateMachine, MessageBus);
-
-const tb = @import("tigerbeetle.zig");
-
-const batches_count = 100;
-
-const transfers_per_batch: u32 = @divExact(
-    config.message_size_max - @sizeOf(vsr.Header),
-    @sizeOf(tb.Transfer),
-);
-comptime {
-    assert(transfers_per_batch >= 2041);
-}
-
-const transfers_max: u32 = batches_count * transfers_per_batch;
-
-var accounts = [_]tb.Account{
-    .{
-        .id = 1,
-        .user_data = 0,
-        .reserved = [_]u8{0} ** 48,
-        .ledger = 2,
-        .code = 0,
-        .flags = .{},
-        .debits_pending = 0,
-        .debits_posted = 0,
-        .credits_pending = 0,
-        .credits_posted = 0,
-    },
-    .{
-        .id = 2,
-        .user_data = 0,
-        .reserved = [_]u8{0} ** 48,
-        .ledger = 2,
-        .code = 0,
-        .flags = .{},
-        .debits_pending = 0,
-        .debits_posted = 0,
-        .credits_pending = 0,
-        .credits_posted = 0,
-    },
-};
-var create_transfers_latency_max: i64 = 0;
-
-pub fn main() !void {
-    const stdout = std.io.getStdOut().writer();
-    const stderr = std.io.getStdErr().writer();
-
-    if (builtin.mode != .ReleaseSafe and builtin.mode != .ReleaseFast) {
-        try stderr.print("Benchmark must be built as ReleaseSafe for reasonable results.\n", .{});
-    }
-
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-    defer arena.deinit();
-
-    const allocator = arena.allocator();
-
-    const client_id = std.crypto.random.int(u128);
-    const cluster_id: u32 = 0;
-    var address = [_]std.net.Address{try std.net.Address.parseIp4("127.0.0.1", config.port)};
-
-    var io = try IO.init(32, 0);
-    var message_bus = try MessageBus.init(allocator, cluster_id, address[0..], client_id, &io);
-    defer message_bus.deinit();
-
-    var client = try Client.init(
-        allocator,
-        client_id,
-        cluster_id,
-        @intCast(u8, address.len),
-        &message_bus,
-    );
-    defer client.deinit();
-
-    message_bus.set_on_message(*Client, &client, Client.on_message);
-
-    // Pre-allocate a million transfers:
-    const transfers = try arena.allocator().alloc(tb.Transfer, transfers_max);
-    for (transfers) |*transfer, index| {
-        transfer.* = .{
-            .id = index,
-            .debit_account_id = accounts[0].id,
-            .credit_account_id = accounts[1].id,
-            .pending_id = 0,
-            .user_data = 0,
-            .reserved = 0,
-            .code = 0,
-            .ledger = 2,
-            .flags = .{},
-            .amount = 1,
-            .timeout = 0,
-        };
-    }
-
-    try wait_for_connect(&client, &io);
-
-    var queue = TimedQueue.init(&client, &io);
-    try queue.push(.{
-        .operation = StateMachine.Operation.create_accounts,
-        .data = std.mem.sliceAsBytes(accounts[0..]),
-    });
-
-    try queue.execute();
-    assert(queue.end != null);
-    assert(queue.batches.empty());
-
-    var count: u64 = 0;
-    queue.reset();
-    while (count < transfers.len) {
-        try queue.push(.{
-            .operation = .create_transfers,
-            .data = std.mem.sliceAsBytes(transfers[count..][0..transfers_per_batch]),
-        });
-        count += transfers_per_batch;
-    }
-    assert(count == transfers_max);
-
-    try queue.execute();
-    assert(queue.end != null);
-    assert(queue.batches.empty());
-
-    var ms = queue.end.? - queue.start.?;
-
-    const result: i64 = @divFloor(@intCast(i64, transfers.len * 1000), ms);
-    try stdout.print("============================================\n", .{});
-    try stdout.print("{} transfers per second\n\n", .{result});
-    try stdout.print("max p100 latency per {} transfers = {}ms\n", .{
-        transfers_per_batch,
-        queue.transfers_latency_max,
-    });
-}
-
-const Batch = struct {
-    operation: StateMachine.Operation,
-    data: []u8,
-};
-
-const TimedQueue = struct {
-    batch_start: ?i64,
-    start: ?i64,
-    end: ?i64,
-    transfers_latency_max: i64,
-    client: *Client,
-    io: *IO,
-    batches: RingBuffer(Batch, batches_count),
-
-    pub fn init(client: *Client, io: *IO) TimedQueue {
-        var self = TimedQueue{
-            .batch_start = null,
-            .start = null,
-            .end = null,
-            .transfers_latency_max = 0,
-            .client = client,
-            .io = io,
-            .batches = .{},
-        };
-
-        return self;
-    }
-
-    pub fn reset(self: *TimedQueue) void {
-        self.batch_start = null;
-        self.start = null;
-        self.end = null;
-        self.transfers_latency_max = 0;
-    }
-
-    pub fn push(self: *TimedQueue, batch: Batch) !void {
-        try self.batches.push(batch);
-    }
-
-    pub fn execute(self: *TimedQueue) !void {
-        assert(self.start == null);
-        assert(!self.batches.empty());
-        self.reset();
-        log.debug("executing batches...", .{});
-
-        const now = std.time.milliTimestamp();
-        self.start = now;
-        if (self.batches.head_ptr()) |starting_batch| {
-            log.debug("sending first batch...", .{});
-            self.batch_start = now;
-            const message = self.client.get_message();
-            defer self.client.unref(message);
-
-            std.mem.copy(
-                u8,
-                message.buffer[@sizeOf(vsr.Header)..],
-                std.mem.sliceAsBytes(starting_batch.data),
-            );
-            self.client.request(
-                @intCast(u128, @ptrToInt(self)),
-                TimedQueue.lap,
-                starting_batch.operation,
-                message,
-                starting_batch.data.len,
-            );
-        }
-
-        while (!self.batches.empty()) {
-            self.client.tick();
-            try self.io.run_for_ns(5 * std.time.ns_per_ms);
-        }
-    }
-
-    pub fn lap(
-        user_data: u128,
-        operation: StateMachine.Operation,
-        results: Client.Error![]const u8,
-    ) void {
-        const now = std.time.milliTimestamp();
-        const value = results catch |err| {
-            log.err("Client returned error={o}", .{@errorName(err)});
-            @panic("Client returned error during benchmarking.");
-        };
-
-        log.debug("response={s}", .{std.mem.bytesAsSlice(tb.CreateAccountsResult, value)});
-
-        const self: *TimedQueue = @intToPtr(*TimedQueue, @intCast(usize, user_data));
-        const completed_batch: ?Batch = self.batches.pop();
-        assert(completed_batch != null);
-        assert(completed_batch.?.operation == operation);
-
-        log.debug("completed batch operation={} start={}", .{
-            completed_batch.?.operation,
-            self.batch_start,
-        });
-        const latency = now - self.batch_start.?;
-        switch (operation) {
-            .create_accounts => {},
-            .create_transfers => {
-                if (latency > self.transfers_latency_max) {
-                    self.transfers_latency_max = latency;
-                }
-            },
-            else => unreachable,
-        }
-
-        if (self.batches.head_ptr()) |next_batch| {
-            const message = self.client.get_message();
-            defer self.client.unref(message);
-
-            std.mem.copy(
-                u8,
-                message.buffer[@sizeOf(vsr.Header)..],
-                std.mem.sliceAsBytes(next_batch.data),
-            );
-
-            self.batch_start = std.time.milliTimestamp();
-            self.client.request(
-                @intCast(u128, @ptrToInt(self)),
-                TimedQueue.lap,
-                next_batch.operation,
-                message,
-                next_batch.data.len,
-            );
-        } else {
-            log.debug("stopping timer...", .{});
-            self.end = now;
-        }
-    }
-};
-
-fn wait_for_connect(client: *Client, io: *IO) !void {
-    var ticks: u32 = 0;
-    while (ticks < 20) : (ticks += 1) {
-        client.tick();
-        // We tick IO outside of client so that an IO instance can be shared by multiple clients:
-        // Otherwise we will hit io_uring memory restrictions too quickly.
-        try io.tick();
-
-        std.time.sleep(10 * std.time.ns_per_ms);
-    }
-}

package/src/tigerbeetle/src/cli.zig

@@ -1,244 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const fmt = std.fmt;
-const mem = std.mem;
-const meta = std.meta;
-const net = std.net;
-const os = std.os;
-
-const config = @import("config.zig");
-const vsr = @import("vsr.zig");
-const IO = @import("io.zig").IO;
-
-const usage = fmt.comptimePrint(
-    \\Usage:
-    \\
-    \\  tigerbeetle [-h | --help]
-    \\
-    \\  tigerbeetle init  [--directory=<path>] --cluster=<integer> --replica=<index>
-    \\
-    \\  tigerbeetle start [--directory=<path>] --cluster=<integer> --replica=<index> --addresses=<addresses>
-    \\
-    \\Commands:
-    \\
-    \\  init   Create a new .tigerbeetle data file. Requires the --cluster and
-    \\         --replica options. The file will be created in the path set by
-    \\         the --directory option if provided. Otherwise, it will be created in
-    \\         the default {[default_directory]s}.
-    \\
-    \\  start  Run a TigerBeetle replica as part of the cluster specified by the
-    \\         --cluster, --replica, and --addresses options. This requires an
-    \\         existing .tigerbeetle data file, either in the default
-    \\         {[default_directory]s} or the path set with --directory.
-    \\
-    \\Options:
-    \\
-    \\  -h, --help
-    \\        Print this help message and exit.
-    \\
-    \\  --directory=<path>
-    \\        Set the directory used to store .tigerbeetle data files. If this option is
-    \\        omitted, the default {[default_directory]s} will be used.
-    \\
-    \\  --cluster=<integer>
-    \\        Set the cluster ID to the provided 32-bit unsigned integer.
-    \\
-    \\  --replica=<index>
-    \\        Set the zero-based index that will be used for this replica process.
-    \\        The value of this option will be interpreted as an index into the --addresses array.
-    \\
-    \\  --addresses=<addresses>
-    \\        Set the addresses of all replicas in the cluster. Accepts a
-    \\        comma-separated list of IPv4 addresses with port numbers.
-    \\        Either the IPv4 address or port number, but not both, may be
-    \\        ommited in which case a default of {[default_address]s} or {[default_port]d}
-    \\        will be used.
-    \\
-    \\Examples:
-    \\
-    \\  tigerbeetle init --cluster=0 --replica=0 --directory=/var/lib/tigerbeetle
-    \\  tigerbeetle init --cluster=0 --replica=1 --directory=/var/lib/tigerbeetle
-    \\  tigerbeetle init --cluster=0 --replica=2 --directory=/var/lib/tigerbeetle
-    \\
-    \\  tigerbeetle start --cluster=0 --replica=0 --addresses=127.0.0.1:3003,127.0.0.1:3001,127.0.0.1:3002
-    \\  tigerbeetle start --cluster=0 --replica=1 --addresses=3003,3001,3002
-    \\  tigerbeetle start --cluster=0 --replica=2 --addresses=3003,3001,3002
-    \\
-    \\  tigerbeetle start --cluster=1 --replica=0 --addresses=192.168.0.1,192.168.0.2,192.168.0.3
-    \\
-, .{
-    .default_directory = config.directory,
-    .default_address = config.address,
-    .default_port = config.port,
-});
-
-pub const Command = union(enum) {
-    init: struct {
-        cluster: u32,
-        replica: u8,
-        dir_fd: os.fd_t,
-    },
-    start: struct {
-        cluster: u32,
-        replica: u8,
-        addresses: []net.Address,
-        dir_fd: os.fd_t,
-    },
-};
-
-/// Parse the command line arguments passed to the tigerbeetle binary.
-/// Exits the program with a non-zero exit code if an error is found.
-pub fn parse_args(allocator: std.mem.Allocator) !Command {
-    var maybe_cluster: ?[]const u8 = null;
-    var maybe_replica: ?[]const u8 = null;
-    var maybe_addresses: ?[]const u8 = null;
-    var maybe_directory: ?[:0]const u8 = null;
-
-    var args = try std.process.argsWithAllocator(allocator);
-    defer args.deinit();
-
-    // Keep track of the args from the ArgIterator above that were allocated
-    // then free them all at the end of the scope.
-    var args_allocated = std.ArrayList([:0]const u8).init(allocator);
-    defer {
-        for (args_allocated.items) |arg| allocator.free(arg);
-        args_allocated.deinit();
-    }
-
-    // Skip argv[0] which is the name of this executable
-    const did_skip = args.skip();
-    assert(did_skip);
-
-    const raw_command = try (args.next(allocator) orelse
-        fatal("no command provided, expected 'start' or 'init'", .{}));
-    defer allocator.free(raw_command);
-
-    if (mem.eql(u8, raw_command, "-h") or mem.eql(u8, raw_command, "--help")) {
-        std.io.getStdOut().writeAll(usage) catch os.exit(1);
-        os.exit(0);
-    }
-    const command = meta.stringToEnum(meta.Tag(Command), raw_command) orelse
-        fatal("unknown command '{s}', expected 'start' or 'init'", .{raw_command});
-
-    while (args.next(allocator)) |parsed_arg| {
-        const arg = try parsed_arg;
-        try args_allocated.append(arg);
-
-        if (mem.startsWith(u8, arg, "--cluster")) {
-            maybe_cluster = parse_flag("--cluster", arg);
-        } else if (mem.startsWith(u8, arg, "--replica")) {
-            maybe_replica = parse_flag("--replica", arg);
-        } else if (mem.startsWith(u8, arg, "--addresses")) {
-            maybe_addresses = parse_flag("--addresses", arg);
-        } else if (mem.startsWith(u8, arg, "--directory")) {
-            maybe_directory = parse_flag("--directory", arg);
-        } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
-            std.io.getStdOut().writeAll(usage) catch os.exit(1);
-            os.exit(0);
-        } else if (mem.startsWith(u8, arg, "--")) {
-            fatal("unexpected argument: '{s}'", .{arg});
-        } else {
-            fatal("unexpected argument: '{s}' (must start with '--')", .{arg});
-        }
-    }
-
-    const raw_cluster = maybe_cluster orelse fatal("required argument: --cluster", .{});
-    const raw_replica = maybe_replica orelse fatal("required argument: --replica", .{});
-
-    const cluster = parse_cluster(raw_cluster);
-    const replica = parse_replica(raw_replica);
-
-    const dir_path = maybe_directory orelse config.directory;
-    const dir_fd = IO.open_dir(dir_path) catch |err|
-        fatal("failed to open directory '{s}': {}", .{ dir_path, err });
-
-    switch (command) {
-        .init => {
-            if (maybe_addresses != null) {
-                fatal("--addresses: supported only by 'start' command", .{});
-            }
-
-            return Command{
-                .init = .{
-                    .cluster = cluster,
-                    .replica = replica,
-                    .dir_fd = dir_fd,
-                },
-            };
-        },
-        .start => {
-            const raw_addresses = maybe_addresses orelse
-                fatal("required argument: --addresses", .{});
-            const addresses = parse_addresses(allocator, raw_addresses);
-
-            if (replica >= addresses.len) {
-                fatal("--replica: value greater than length of --addresses array", .{});
-            }
-
-            return Command{
-                .start = .{
-                    .cluster = cluster,
-                    .replica = replica,
-                    .addresses = addresses,
-                    .dir_fd = dir_fd,
-                },
-            };
-        },
-    }
-}
-
-/// Format and print an error message followed by the usage string to stderr,
-/// then exit with an exit code of 1.
-fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn {
-    const stderr = std.io.getStdErr().writer();
-    stderr.print("error: " ++ fmt_string ++ "\n", args) catch {};
-    os.exit(1);
-}
-
-/// Parse e.g. `--cluster=1a2b3c` into `1a2b3c` with error handling.
-fn parse_flag(comptime flag: []const u8, arg: [:0]const u8) [:0]const u8 {
-    const value = arg[flag.len..];
-    if (value.len < 2) {
-        fatal("{s} argument requires a value", .{flag});
-    }
-    if (value[0] != '=') {
-        fatal("expected '=' after {s} but found '{c}'", .{ flag, value[0] });
-    }
-    return value[1..];
-}
-
-fn parse_cluster(raw_cluster: []const u8) u32 {
-    const cluster = fmt.parseUnsigned(u32, raw_cluster, 10) catch |err| switch (err) {
-        error.Overflow => fatal("--cluster: value exceeds a 32-bit unsigned integer", .{}),
-        error.InvalidCharacter => fatal("--cluster: value contains an invalid character", .{}),
-    };
-    return cluster;
-}
-
-/// Parse and allocate the addresses returning a slice into that array.
-fn parse_addresses(allocator: std.mem.Allocator, raw_addresses: []const u8) []net.Address {
-    return vsr.parse_addresses(allocator, raw_addresses) catch |err| switch (err) {
-        error.AddressHasTrailingComma => fatal("--addresses: invalid trailing comma", .{}),
-        error.AddressLimitExceeded => {
-            fatal("--addresses: too many addresses, at most {d} are allowed", .{
-                config.replicas_max,
-            });
-        },
-        error.AddressHasMoreThanOneColon => {
-            fatal("--addresses: invalid address with more than one colon", .{});
-        },
-        error.PortOverflow => fatal("--addresses: port exceeds 65535", .{}),
-        error.PortInvalid => fatal("--addresses: invalid port", .{}),
-        error.AddressInvalid => fatal("--addresses: invalid IPv4 address", .{}),
-        error.OutOfMemory => fatal("--addresses: out of memory", .{}),
-    };
-}
-
-fn parse_replica(raw_replica: []const u8) u8 {
-    comptime assert(config.replicas_max <= std.math.maxInt(u8));
-    const replica = fmt.parseUnsigned(u8, raw_replica, 10) catch |err| switch (err) {
-        error.Overflow => fatal("--replica: value exceeds an 8-bit unsigned integer", .{}),
-        error.InvalidCharacter => fatal("--replica: value contains an invalid character", .{}),
-    };
-    return replica;
-}