tigerbeetle-node 0.3.3 → 0.5.0

package/src/tigerbeetle/src/cli.zig

@@ -2,167 +2,220 @@ const std = @import("std");
 const assert = std.debug.assert;
 const fmt = std.fmt;
 const mem = std.mem;
+const meta = std.meta;
 const net = std.net;
 const os = std.os;
 
 const config = @import("config.zig");
-const vr = @import("vr.zig");
+const vsr = @import("vsr.zig");
 
 const usage = fmt.comptimePrint(
-    \\Usage: tigerbeetle [options]
+    \\Usage:
     \\
-    \\ -h, --help
+    \\  tigerbeetle [-h | --help]
+    \\
+    \\  tigerbeetle init  [--directory=<path>] --cluster=<integer> --replica=<index>
+    \\
+    \\  tigerbeetle start [--directory=<path>] --cluster=<integer> --replica=<index> --addresses=<addresses>
+    \\
+    \\Commands:
+    \\
+    \\  init   Create a new .tigerbeetle data file. Requires the --cluster and
+    \\         --replica options. The file will be created in the path set by
+    \\         the --directory option if provided. Otherwise, it will be created in
+    \\         the default {[default_directory]s}.
+    \\
+    \\  start  Run a TigerBeetle replica as part of the cluster specified by the
+    \\         --cluster, --replica, and --addresses options. This requires an
+    \\         existing .tigerbeetle data file, either in the default
+    \\         {[default_directory]s} or the path set with --directory.
+    \\
+    \\Options:
+    \\
+    \\  -h, --help
     \\        Print this help message and exit.
     \\
-    \\Required Configuration Options:
+    \\  --directory=<path>
+    \\        Set the directory used to store .tigerbeetle data files. If this option is
+    \\        omitted, the default {[default_directory]s} will be used.
+    \\
+    \\  --cluster=<integer>
+    \\        Set the cluster ID to the provided 32-bit unsigned integer.
     \\
-    \\ --cluster-id=<hex id>
-    \\        Set the cluster ID to the provided non-zero 128-bit hexadecimal number.
+    \\  --replica=<index>
+    \\        Set the zero-based index that will be used for this replica process.
+    \\        The value of this option will be interpreted as an index into the --addresses array.
     \\
-    \\ --replica-addresses=<addresses>
+    \\  --addresses=<addresses>
     \\        Set the addresses of all replicas in the cluster. Accepts a
     \\        comma-separated list of IPv4 addresses with port numbers.
     \\        Either the IPv4 address or port number, but not both, may be
     \\        ommited in which case a default of {[default_address]s} or {[default_port]d}
     \\        will be used.
     \\
-    \\ --replica-index=<index>
-    \\        Set the address in the array passed to the --replica-addresses option that
-    \\        will be used for this replica process. The value of this option is
-    \\        interpreted as a zero-based index into the array.
-    \\
     \\Examples:
     \\
-    \\ tigerbeetle --cluster-id=1a2b3c --replica-addresses=127.0.0.1:3003,127.0.0.1:3001,127.0.0.1:3002 --replica-index=0
+    \\  tigerbeetle init --cluster=0 --replica=0 --directory=/var/lib/tigerbeetle
+    \\  tigerbeetle init --cluster=0 --replica=1 --directory=/var/lib/tigerbeetle
+    \\  tigerbeetle init --cluster=0 --replica=2 --directory=/var/lib/tigerbeetle
     \\
-    \\ tigerbeetle --cluster-id=1a2b3c --replica-addresses=3003,3001,3002 --replica-index=1
+    \\  tigerbeetle start --cluster=0 --replica=0 --addresses=127.0.0.1:3003,127.0.0.1:3001,127.0.0.1:3002
+    \\  tigerbeetle start --cluster=0 --replica=1 --addresses=3003,3001,3002
+    \\  tigerbeetle start --cluster=0 --replica=2 --addresses=3003,3001,3002
     \\
-    \\ tigerbeetle --cluster-id=1a2b3c --replica-addresses=192.168.0.1,192.168.0.2,192.168.0.3 --replica-index=2
+    \\  tigerbeetle start --cluster=1 --replica=0 --addresses=192.168.0.1,192.168.0.2,192.168.0.3
     \\
 , .{
+    .default_directory = config.directory,
     .default_address = config.address,
     .default_port = config.port,
 });
 
-pub const Args = struct {
-    cluster: u128,
-    configuration: []net.Address,
-    replica: u16,
+pub const Command = union(enum) {
+    init: struct {
+        cluster: u32,
+        replica: u8,
+        dir_fd: os.fd_t,
+    },
+    start: struct {
+        cluster: u32,
+        replica: u8,
+        addresses: []net.Address,
+        dir_fd: os.fd_t,
+    },
 };
 
 /// Parse the command line arguments passed to the tigerbeetle binary.
 /// Exits the program with a non-zero exit code if an error is found.
-pub fn parse_args(allocator: *std.mem.Allocator) Args {
+pub fn parse_args(allocator: *std.mem.Allocator) Command {
     var maybe_cluster: ?[]const u8 = null;
-    var maybe_configuration: ?[]const u8 = null;
     var maybe_replica: ?[]const u8 = null;
+    var maybe_addresses: ?[]const u8 = null;
+    var maybe_directory: ?[:0]const u8 = null;
 
     var args = std.process.args();
     // Skip argv[0] which is the name of this executable
     _ = args.nextPosix();
+
+    const raw_command = args.nextPosix() orelse
+        fatal("no command provided, expected 'start' or 'init'", .{});
+    if (mem.eql(u8, raw_command, "-h") or mem.eql(u8, raw_command, "--help")) {
+        std.io.getStdOut().writeAll(usage) catch os.exit(1);
+        os.exit(0);
+    }
+    const command = meta.stringToEnum(meta.Tag(Command), raw_command) orelse
+        fatal("unknown command '{s}', expected 'start' or 'init'", .{raw_command});
+
     while (args.nextPosix()) |arg| {
-        if (mem.startsWith(u8, arg, "--cluster-id")) {
-            maybe_cluster = parse_flag("--cluster-id", arg);
-        } else if (mem.startsWith(u8, arg, "--replica-addresses")) {
-            maybe_configuration = parse_flag("--replica-addresses", arg);
-        } else if (mem.startsWith(u8, arg, "--replica-index")) {
-            maybe_replica = parse_flag("--replica-index", arg);
+        if (mem.startsWith(u8, arg, "--cluster")) {
+            maybe_cluster = parse_flag("--cluster", arg);
+        } else if (mem.startsWith(u8, arg, "--replica")) {
+            maybe_replica = parse_flag("--replica", arg);
+        } else if (mem.startsWith(u8, arg, "--addresses")) {
+            maybe_addresses = parse_flag("--addresses", arg);
+        } else if (mem.startsWith(u8, arg, "--directory")) {
+            maybe_directory = parse_flag("--directory", arg);
         } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
             std.io.getStdOut().writeAll(usage) catch os.exit(1);
             os.exit(0);
         } else {
-            print_error_exit("unexpected argument: '{s}'", .{arg});
+            fatal("unexpected argument: '{s}'", .{arg});
         }
     }
 
-    const raw_cluster = maybe_cluster orelse
-        print_error_exit("required argument: --cluster-id", .{});
-    const raw_configuration = maybe_configuration orelse
-        print_error_exit("required argument: --replica-addresses", .{});
-    const raw_replica = maybe_replica orelse
-        print_error_exit("required argument: --replica-index", .{});
+    const raw_cluster = maybe_cluster orelse fatal("required argument: --cluster", .{});
+    const raw_replica = maybe_replica orelse fatal("required argument: --replica", .{});
 
     const cluster = parse_cluster(raw_cluster);
-    const configuration = parse_configuration(allocator, raw_configuration);
-    const replica = parse_replica(raw_replica, @intCast(u16, configuration.len));
+    const replica = parse_replica(raw_replica);
 
-    return .{
-        .cluster = cluster,
-        .configuration = configuration,
-        .replica = replica,
-    };
+    const dir_path = maybe_directory orelse config.directory;
+    const dir_fd = os.openZ(dir_path, os.O_CLOEXEC | os.O_RDONLY, 0) catch |err|
+        fatal("failed to open directory '{s}': {}", .{ dir_path, err });
+
+    switch (command) {
+        .init => {
+            if (maybe_addresses != null) {
+                fatal("--addresses: supported only by 'start' command", .{});
+            }
+
+            return .{ .init = .{
+                .cluster = cluster,
+                .replica = replica,
+                .dir_fd = dir_fd,
+            } };
+        },
+        .start => {
+            const raw_addresses = maybe_addresses orelse
+                fatal("required argument: --addresses", .{});
+            const addresses = parse_addresses(allocator, raw_addresses);
+
+            if (replica >= addresses.len) {
+                fatal("--replica: value greater than length of --addresses array", .{});
+            }
+
+            return .{ .start = .{
+                .cluster = cluster,
+                .replica = replica,
+                .addresses = addresses,
+                .dir_fd = dir_fd,
+            } };
+        },
+    }
 }
 
 /// Format and print an error message followed by the usage string to stderr,
 /// then exit with an exit code of 1.
-fn print_error_exit(comptime fmt_string: []const u8, args: anytype) noreturn {
+fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn {
     const stderr = std.io.getStdErr().writer();
-    stderr.print("error: " ++ fmt_string ++ "\n\n" ++ usage, args) catch {};
+    stderr.print("error: " ++ fmt_string ++ "\n", args) catch {};
     os.exit(1);
 }
 
 /// Parse e.g. `--cluster=1a2b3c` into `1a2b3c` with error handling.
-fn parse_flag(comptime flag: []const u8, arg: []const u8) []const u8 {
+fn parse_flag(comptime flag: []const u8, arg: [:0]const u8) [:0]const u8 {
     const value = arg[flag.len..];
     if (value.len < 2) {
-        print_error_exit("{s} argument requires a value", .{flag});
+        fatal("{s} argument requires a value", .{flag});
     }
     if (value[0] != '=') {
-        print_error_exit("expected '=' after {s} but found '{c}'", .{ flag, value[0] });
+        fatal("expected '=' after {s} but found '{c}'", .{ flag, value[0] });
     }
     return value[1..];
 }
 
-fn parse_cluster(raw_cluster: []const u8) u128 {
-    const cluster = fmt.parseUnsigned(u128, raw_cluster, 16) catch |err| switch (err) {
-        error.Overflow => print_error_exit(
-            \\--cluster-id: value does not fit into a 128-bit unsigned integer
-        , .{}),
-        error.InvalidCharacter => print_error_exit(
-            \\--cluster-id: value contains an invalid character
-        , .{}),
+fn parse_cluster(raw_cluster: []const u8) u32 {
+    const cluster = fmt.parseUnsigned(u32, raw_cluster, 10) catch |err| switch (err) {
+        error.Overflow => fatal("--cluster: value exceeds a 32-bit unsigned integer", .{}),
+        error.InvalidCharacter => fatal("--cluster: value contains an invalid character", .{}),
     };
-    if (cluster == 0) {
-        print_error_exit("--cluster-id: a value of 0 is not permitted", .{});
-    }
     return cluster;
 }
 
-/// Parse and allocate the configuration returning a slice into that array.
-fn parse_configuration(allocator: *std.mem.Allocator, raw_configuration: []const u8) []net.Address {
-    return vr.parse_configuration(allocator, raw_configuration) catch |err| switch (err) {
-        error.AddressHasTrailingComma => {
-            print_error_exit("--replica-addresses: invalid trailing comma", .{});
-        },
+/// Parse and allocate the addresses returning a slice into that array.
+fn parse_addresses(allocator: *std.mem.Allocator, raw_addresses: []const u8) []net.Address {
+    return vsr.parse_addresses(allocator, raw_addresses) catch |err| switch (err) {
+        error.AddressHasTrailingComma => fatal("--addresses: invalid trailing comma", .{}),
         error.AddressLimitExceeded => {
-            print_error_exit("--replica-addresses: too many addresses, at most {d} are allowed", .{
+            fatal("--addresses: too many addresses, at most {d} are allowed", .{
                 config.replicas_max,
             });
         },
         error.AddressHasMoreThanOneColon => {
-            print_error_exit("--replica-addresses: invalid address with more than one colon", .{});
+            fatal("--addresses: invalid address with more than one colon", .{});
         },
-        error.PortOverflow => print_error_exit("--replica-addresses: port exceeds 65535", .{}),
-        error.PortInvalid => print_error_exit("--replica-addresses: invalid port", .{}),
-        error.AddressInvalid => print_error_exit("--replica-addresses: invalid IPv4 address", .{}),
-        error.OutOfMemory => print_error_exit("--replica-addresses: out of memory", .{}),
+        error.PortOverflow => fatal("--addresses: port exceeds 65535", .{}),
+        error.PortInvalid => fatal("--addresses: invalid port", .{}),
+        error.AddressInvalid => fatal("--addresses: invalid IPv4 address", .{}),
+        error.OutOfMemory => fatal("--addresses: out of memory", .{}),
     };
 }
 
-fn parse_replica(raw_replica: []const u8, configuration_len: u16) u16 {
-    comptime assert(config.replicas_max <= std.math.maxInt(u16));
-    const replica = fmt.parseUnsigned(u16, raw_replica, 10) catch |err| switch (err) {
-        error.Overflow => print_error_exit(
-            \\--replica-index: value greater than length of address array
-        , .{}),
-        error.InvalidCharacter => print_error_exit(
-            \\--replica-index: value contains an invalid character
-        , .{}),
+fn parse_replica(raw_replica: []const u8) u8 {
+    comptime assert(config.replicas_max <= std.math.maxInt(u8));
+    const replica = fmt.parseUnsigned(u8, raw_replica, 10) catch |err| switch (err) {
+        error.Overflow => fatal("--replica: value exceeds an 8-bit unsigned integer", .{}),
+        error.InvalidCharacter => fatal("--replica: value contains an invalid character", .{}),
     };
-    if (replica >= configuration_len) {
-        print_error_exit(
-            \\--replica-index: value greater than length of address array
-        , .{});
-    }
     return replica;
 }

package/src/tigerbeetle/src/config.zig

@@ -5,28 +5,35 @@ pub const deployment_environment = .development;
 pub const log_level = 6;
 
 /// The maximum number of replicas allowed in a cluster.
-pub const replicas_max = 15;
+pub const replicas_max = 6;
 
-/// The minimum number of nodes required to form quorums for leader election or replication:
-/// Majority quorums are only required across leader election and replication phases (not within).
-/// As per Flexible Paxos, provided quorum_leader_election + quorum_replication > cluster_nodes:
-/// 1. you may increase quorum_leader_election above a majority, so that
-/// 2. you can decrease quorum_replication below a majority, to optimize the common case.
+/// The maximum number of clients allowed per cluster, where each client has a unique 128-bit ID.
+/// This impacts the amount of memory allocated at initialization by the server.
+/// This determines the size of the VR client table used to cache replies to clients by client ID.
+/// Each client has one entry in the VR client table to store the latest `message_size_max` reply.
+pub const clients_max = 32;
+
+/// The minimum number of nodes required to form a quorum for replication:
+/// Majority quorums are only required across view change and replication phases (not within).
+/// As per Flexible Paxos, provided `quorum_replication + quorum_view_change > replicas`:
+/// 1. you may increase `quorum_view_change` above a majority, so that
+/// 2. you can decrease `quorum_replication` below a majority, to optimize the common case.
 /// This improves latency by reducing the number of nodes required for synchronous replication.
 /// This reduces redundancy only in the short term, asynchronous replication will still continue.
-pub const quorum_leader_election = -1;
-pub const quorum_replication = 2;
+/// The size of the replication quorum is limited to the minimum of this value and actual majority.
+/// The size of the view change quorum will then be automatically inferred from quorum_replication.
+pub const quorum_replication_max = 3;
 
-/// The default server port to listen on if not specified in `--replica-addresses`:
+/// The default server port to listen on if not specified in `--addresses`:
 pub const port = 3001;
 
-/// The default network interface address to listen on if not specified in `--replica-addresses`:
+/// The default network interface address to listen on if not specified in `--addresses`:
 /// WARNING: Binding to all interfaces with "0.0.0.0" is dangerous and opens the server to anyone.
 /// Bind to the "127.0.0.1" loopback address to accept local connections as a safe default only.
 pub const address = "127.0.0.1";
 
-/// Where journal files should be persisted:
-pub const data_directory = "/var/lib/tigerbeetle";
+/// Where data files should be persisted by default:
+pub const directory = "/var/lib/tigerbeetle";
 
 /// The maximum number of accounts to store in memory:
 /// This impacts the amount of memory allocated at initialization by the server.
@@ -54,7 +61,7 @@ pub const commits_max = transfers_max;
 /// This also enables us to detect filesystem inode corruption that would change the journal size.
 pub const journal_size_max = switch (deployment_environment) {
     .production => 128 * 1024 * 1024 * 1024,
-    else => 256 * 1024 * 1024,
+    else => 128 * 1024 * 1024,
 };
 
 /// The maximum number of batch entries in the journal file:
@@ -66,33 +73,37 @@ pub const journal_headers_max = switch (deployment_environment) {
     else => 16384,
 };
 
-/// The maximum number of connections that can be accepted and held open by the server at any time:
-pub const connections_max = 32;
+/// The maximum number of connections that can be held open by the server at any time:
+pub const connections_max = replicas_max + clients_max;
 
 /// The maximum size of a message in bytes:
 /// This is also the limit of all inflight data across multiple pipelined requests per connection.
-/// We may have one request of up to 4 MiB inflight or 4 pipelined requests of up to 1 MiB inflight.
+/// We may have one request of up to 2 MiB inflight or 2 pipelined requests of up to 1 MiB inflight.
 /// This impacts sequential disk write throughput, the larger the buffer the better.
-/// 4 MiB is 32,768 transfers, and a reasonable choice for sequential disk write throughput.
+/// 2 MiB is 16,384 transfers, and a reasonable choice for sequential disk write throughput.
 /// However, this impacts bufferbloat and head-of-line blocking latency for pipelined requests.
-/// For a 1 Gbps NIC = 125 MiB/s throughput: 4 MiB / 125 * 1000ms = 32ms for the next request.
-/// This also impacts the amount of memory allocated at initialization by the server.
-pub const message_size_max = 4 * 1024 * 1024;
+/// For a 1 Gbps NIC = 125 MiB/s throughput: 2 MiB / 125 * 1000ms = 16ms for the next request.
+/// This impacts the amount of memory allocated at initialization by the server.
+pub const message_size_max = 1 * 1024 * 1024;
 
 /// The number of full-sized messages allocated at initialization by the message bus.
 pub const message_bus_messages_max = connections_max * 4;
 /// The number of header-sized messages allocated at initialization by the message bus.
 /// These are much smaller/cheaper and we can therefore have many of them.
-pub const message_bus_headers_max = connections_max * connection_send_queue_max;
+pub const message_bus_headers_max = connections_max * connection_send_queue_max * 2;
+
+/// The maximum number of Viewstamped Replication prepare messages that can be inflight at a time.
+/// This is immutable once assigned per cluster, as replicas need to know how many operations might
+/// possibly be uncommitted during a view change, and this must be constant for all replicas.
+pub const pipelining_max = clients_max;
 
 /// The minimum and maximum amount of time in milliseconds to wait before initiating a connection.
-/// Exponential backoff and full jitter are applied within this range.
-/// For more, see: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
-pub const connection_delay_min = 50;
-pub const connection_delay_max = 1000;
+/// Exponential backoff and jitter are applied within this range.
+pub const connection_delay_min_ms = 50;
+pub const connection_delay_max_ms = 1000;
 
 /// The maximum number of outgoing messages that may be queued on a connection.
-pub const connection_send_queue_max = 3;
+pub const connection_send_queue_max = pipelining_max;
 
 /// The maximum number of connections in the kernel's complete connection queue pending an accept():
 /// If the backlog argument is greater than the value in `/proc/sys/net/core/somaxconn`, then it is
@@ -161,6 +172,49 @@ pub const sector_size = 4096;
 /// when they were never written to disk.
 pub const direct_io = true;
 
+/// The maximum number of concurrent read I/O operations to allow at once.
+pub const io_depth_read = 8;
+/// The maximum number of concurrent write I/O operations to allow at once.
+pub const io_depth_write = 8;
+
 /// The number of milliseconds between each replica tick, the basic unit of time in TigerBeetle.
 /// Used to regulate heartbeats, retries and timeouts, all specified as multiples of a tick.
 pub const tick_ms = 10;
+
+/// The conservative round-trip time at startup when there is no network knowledge.
+/// Adjusted dynamically thereafter for RTT-sensitive timeouts according to network congestion.
+/// This should be set higher rather than lower to avoid flooding the network at startup.
+pub const rtt_ticks = 300 / tick_ms;
+
+/// The multiple of round-trip time for RTT-sensitive timeouts.
+pub const rtt_multiple = 2;
+
+/// The min/max bounds of exponential backoff (and jitter) to add to RTT-sensitive timeouts.
+pub const backoff_min_ticks = 100 / tick_ms;
+pub const backoff_max_ticks = 10000 / tick_ms;
+
+/// The maximum skew between two clocks to allow when considering them to be in agreement.
+/// The principle is that no two clocks tick exactly alike but some clocks more or less agree.
+/// The maximum skew across the cluster as a whole is this value times the total number of clocks.
+/// The cluster will be unavailable if the majority of clocks are all further than this value apart.
+/// Decreasing this reduces the probability of reaching agreement on synchronized time.
+/// Increasing this reduces the accuracy of synchronized time.
+pub const clock_offset_tolerance_max_ms = 10000;
+
+/// The amount of time before the clock's synchronized epoch is expired.
+/// If the epoch is expired before it can be replaced with a new synchronized epoch, then this most
+/// likely indicates either a network partition or else too many clock faults across the cluster.
+/// A new synchronized epoch will be installed as soon as these conditions resolve.
+pub const clock_epoch_max_ms = 60000;
+
+/// The amount of time to wait for enough accurate samples before synchronizing the clock.
+/// The more samples we can take per remote clock source, the more accurate our estimation becomes.
+/// This impacts cluster startup time as the leader must first wait for synchronization to complete.
+pub const clock_synchronization_window_min_ms = 2000;
+
+/// The amount of time without agreement before the clock window is expired and a new window opened.
+/// This happens where some samples have been collected but not enough to reach agreement.
+/// The quality of samples degrades as they age so at some point we throw them away and start over.
+/// This eliminates the impact of gradual clock drift on our clock offset (clock skew) measurements.
+/// If a window expires because of this then it is likely that the clock epoch will also be expired.
+pub const clock_synchronization_window_max_ms = 20000;