tigerbeetle-node 0.4.1 → 0.5.1

package/src/tigerbeetle/src/storage.zig

@@ -1,14 +1,15 @@
 const std = @import("std");
+const builtin = @import("builtin");
 const os = std.os;
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const log = std.log.scoped(.storage);
 
 const IO = @import("io.zig").IO;
-const is_darwin = std.Target.current.isDarwin();
+const is_darwin = builtin.target.isDarwin();
 
 const config = @import("config.zig");
-const vr = @import("vr.zig");
+const vsr = @import("vsr.zig");
 
 pub const Storage = struct {
     /// See usage in Journal.write_sectors() for details.
@@ -90,7 +91,7 @@ pub const Storage = struct {
         buffer: []u8,
         offset: u64,
     ) void {
-        self.assert_alignment(buffer, offset);
+        assert_alignment(buffer, offset);
 
         read.* = .{
             .completion = undefined,
@@ -169,7 +170,7 @@ pub const Storage = struct {
                     assert(target.len > 0);
                     std.mem.set(u8, target, 0);
 
-                    // We could set `read.target_max` to `vr.sector_ceil(read.buffer.len)` here
+                    // We could set `read.target_max` to `vsr.sector_ceil(read.buffer.len)` here
                     // in order to restart our pseudo-binary search on the rest of the sectors to be
                     // read, optimistically assuming that this is the last failing sector.
                     // However, data corruption that causes EIO errors often has spacial locality.
@@ -189,7 +190,7 @@ pub const Storage = struct {
             error.Unseekable,
             error.Unexpected,
             => {
-                log.emerg(
+                log.err(
                     "impossible read: offset={} buffer.len={} error={s}",
                     .{ read.offset, read.buffer.len, @errorName(err) },
                 );
@@ -201,7 +202,7 @@ pub const Storage = struct {
             // We tried to read more than there really is available to read.
             // In other words, we thought we could read beyond the end of the file descriptor.
             // This can happen if the data file inode `size` was truncated or corrupted.
-            log.emerg(
+            log.err(
                 "short read: buffer.len={} offset={} bytes_read={}",
                 .{ read.offset, read.buffer.len, bytes_read },
             );
@@ -227,7 +228,7 @@ pub const Storage = struct {
         buffer: []const u8,
         offset: u64,
     ) void {
-        self.assert_alignment(buffer, offset);
+        assert_alignment(buffer, offset);
 
         write.* = .{
             .completion = undefined,
@@ -262,7 +263,7 @@ pub const Storage = struct {
             // TODO: It seems like it might be possible for some filesystems to return ETIMEDOUT
             // here. Consider handling this without panicking.
             else => {
-                log.emerg(
+                log.err(
                     "impossible write: offset={} buffer.len={} error={s}",
                     .{ write.offset, write.buffer.len, @errorName(err) },
                 );
@@ -295,7 +296,7 @@ pub const Storage = struct {
     /// If this is not the case, then the underlying syscall will return EINVAL.
     /// We check this only at the start of a read or write because the physical sector size may be
     /// less than our logical sector size so that partial IOs then leave us no longer aligned.
-    fn assert_alignment(self: *Storage, buffer: []const u8, offset: u64) void {
+    fn assert_alignment(buffer: []const u8, offset: u64) void {
         assert(@ptrToInt(buffer.ptr) % config.sector_size == 0);
         assert(buffer.len % config.sector_size == 0);
         assert(offset % config.sector_size == 0);
@@ -330,17 +331,17 @@ pub const Storage = struct {
         // TODO Use O_EXCL when opening as a block device to obtain a mandatory exclusive lock.
         // This is much stronger than an advisory exclusive lock, and is required on some platforms.
 
-        var flags: u32 = os.O_CLOEXEC | os.O_RDWR | os.O_DSYNC;
+        var flags: u32 = os.O.CLOEXEC | os.O.RDWR | os.O.DSYNC;
         var mode: os.mode_t = 0;
 
         // TODO Document this and investigate whether this is in fact correct to set here.
-        if (@hasDecl(os, "O_LARGEFILE")) flags |= os.O_LARGEFILE;
+        if (@hasDecl(os, "O_LARGEFILE")) flags |= os.O.LARGEFILE;
 
         var direct_io_supported = false;
         if (config.direct_io) {
             direct_io_supported = try Storage.fs_supports_direct_io(dir_fd);
             if (direct_io_supported) {
-                if (!is_darwin) flags |= os.O_DIRECT;
+                if (!is_darwin) flags |= os.O.DIRECT;
             } else if (config.deployment_environment == .development) {
                 log.warn("file system does not support Direct I/O", .{});
             } else {
@@ -352,15 +353,15 @@ pub const Storage = struct {
 
         if (must_create) {
             log.info("creating \"{s}\"...", .{relative_path});
-            flags |= os.O_CREAT;
-            flags |= os.O_EXCL;
+            flags |= os.O.CREAT;
+            flags |= os.O.EXCL;
             mode = 0o666;
         } else {
             log.info("opening \"{s}\"...", .{relative_path});
         }
 
         // This is critical as we rely on O_DSYNC for fsync() whenever we write to the file:
-        assert((flags & os.O_DSYNC) > 0);
+        assert((flags & os.O.DSYNC) > 0);
 
         // Be careful with openat(2): "If pathname is absolute, then dirfd is ignored." (man page)
         assert(!std.fs.path.isAbsolute(relative_path));
@@ -372,12 +373,12 @@ pub const Storage = struct {
 
         // On darwin, use F_NOCACHE on direct_io to disable the page cache as O_DIRECT doesn't exit.
         if (is_darwin and config.direct_io and direct_io_supported) {
-            _ = try os.fcntl(fd, os.F_NOCACHE, 1);
+            _ = try os.fcntl(fd, os.F.NOCACHE, 1);
         }
 
         // Obtain an advisory exclusive lock that works only if all processes actually use flock().
         // LOCK_NB means that we want to fail the lock without waiting if another process has it.
-        os.flock(fd, os.LOCK_EX | os.LOCK_NB) catch |err| switch (err) {
+        os.flock(fd, os.LOCK.EX | os.LOCK.NB) catch |err| switch (err) {
             error.WouldBlock => @panic("another process holds the data file lock"),
             else => return err,
         };
@@ -411,7 +412,7 @@ pub const Storage = struct {
         Storage.fallocate(fd, 0, 0, @intCast(i64, size)) catch |err| switch (err) {
             error.OperationNotSupported => {
                 log.warn("file system does not support fallocate(), an ENOSPC will panic", .{});
-                log.notice("allocating by writing to the last sector of the file instead...", .{});
+                log.info("allocating by writing to the last sector of the file instead...", .{});
 
                 const sector_size = config.sector_size;
                 const sector: [sector_size]u8 align(sector_size) = [_]u8{0} ** sector_size;
@@ -436,6 +437,8 @@ pub const Storage = struct {
             const F_ALLOCATEALL = 0x4; // allocate all or nothing
             const F_PEOFPOSMODE = 3; // use relative offset from the seek pos mode
             const F_VOLPOSMODE = 4; // use the specified volume offset
+            _ = F_VOLPOSMODE;
+
             const fstore_t = extern struct {
                 fst_flags: c_uint,
                 fst_posmode: c_int,
@@ -453,24 +456,24 @@ pub const Storage = struct {
             };
 
             // try to pre-allocate contiguous space and fall back to default non-continugous
-            var res = os.system.fcntl(fd, os.F_PREALLOCATE, @ptrToInt(&store));
-            if (os.errno(res) != 0) {
+            var res = os.system.fcntl(fd, os.F.PREALLOCATE, @ptrToInt(&store));
+            if (os.errno(res) != .SUCCESS) {
                 store.fst_flags = F_ALLOCATEALL;
-                res = os.system.fcntl(fd, os.F_PREALLOCATE, @ptrToInt(&store));
+                res = os.system.fcntl(fd, os.F.PREALLOCATE, @ptrToInt(&store));
             }
 
             switch (os.errno(res)) {
-                0 => {},
-                os.EACCES => unreachable, // F_SETLK or F_SETSIZE of F_WRITEBOOTSTRAP
-                os.EBADF => return error.FileDescriptorInvalid,
-                os.EDEADLK => unreachable, // F_SETLKW
-                os.EINTR => unreachable, // F_SETLKW
-                os.EINVAL => return error.ArgumentsInvalid, // for F_PREALLOCATE (offset invalid)
-                os.EMFILE => unreachable, // F_DUPFD or F_DUPED
-                os.ENOLCK => unreachable, // F_SETLK or F_SETLKW
-                os.EOVERFLOW => return error.FileTooBig,
-                os.ESRCH => unreachable, // F_SETOWN
-                os.EOPNOTSUPP => return error.OperationNotSupported, // not reported but need same error union
+                .SUCCESS => {},
+                .ACCES => unreachable, // F_SETLK or F_SETSIZE of F_WRITEBOOTSTRAP
+                .BADF => return error.FileDescriptorInvalid,
+                .DEADLK => unreachable, // F_SETLKW
+                .INTR => unreachable, // F_SETLKW
+                .INVAL => return error.ArgumentsInvalid, // for F_PREALLOCATE (offset invalid)
+                .MFILE => unreachable, // F_DUPFD or F_DUPED
+                .NOLCK => unreachable, // F_SETLK or F_SETLKW
+                .OVERFLOW => return error.FileTooBig,
+                .SRCH => unreachable, // F_SETOWN
+                .OPNOTSUPP => return error.OperationNotSupported, // not reported but need same error union
                 else => |errno| return os.unexpectedErrno(errno),
             }
 
@@ -484,19 +487,19 @@ pub const Storage = struct {
         while (true) {
             const rc = os.linux.fallocate(fd, mode, offset, length);
             switch (os.linux.getErrno(rc)) {
-                0 => return,
-                os.linux.EBADF => return error.FileDescriptorInvalid,
-                os.linux.EFBIG => return error.FileTooBig,
-                os.linux.EINTR => continue,
-                os.linux.EINVAL => return error.ArgumentsInvalid,
-                os.linux.EIO => return error.InputOutput,
-                os.linux.ENODEV => return error.NoDevice,
-                os.linux.ENOSPC => return error.NoSpaceLeft,
-                os.linux.ENOSYS => return error.SystemOutdated,
-                os.linux.EOPNOTSUPP => return error.OperationNotSupported,
-                os.linux.EPERM => return error.PermissionDenied,
-                os.linux.ESPIPE => return error.Unseekable,
-                os.linux.ETXTBSY => return error.FileBusy,
+                .SUCCESS => return,
+                .BADF => return error.FileDescriptorInvalid,
+                .FBIG => return error.FileTooBig,
+                .INTR => continue,
+                .INVAL => return error.ArgumentsInvalid,
+                .IO => return error.InputOutput,
+                .NODEV => return error.NoDevice,
+                .NOSPC => return error.NoSpaceLeft,
+                .NOSYS => return error.SystemOutdated,
+                .OPNOTSUPP => return error.OperationNotSupported,
+                .PERM => return error.PermissionDenied,
+                .SPIPE => return error.Unseekable,
+                .TXTBSY => return error.FileBusy,
                 else => |errno| return os.unexpectedErrno(errno),
             }
         }
@@ -509,18 +512,18 @@ pub const Storage = struct {
 
         const path = "fs_supports_direct_io";
         const dir = std.fs.Dir{ .fd = dir_fd };
-        const fd = try os.openatZ(dir_fd, path, os.O_CLOEXEC | os.O_CREAT | os.O_TRUNC, 0o666);
+        const fd = try os.openatZ(dir_fd, path, os.O.CLOEXEC | os.O.CREAT | os.O.TRUNC, 0o666);
         defer os.close(fd);
         defer dir.deleteFile(path) catch {};
 
         // F_NOCACHE on darwin is the most similar option to O_DIRECT on linux.
         if (is_darwin) {
-            _ = os.fcntl(fd, os.F_NOCACHE, 1) catch return false;
+            _ = os.fcntl(fd, os.F.NOCACHE, 1) catch return false;
             return true;
         }
 
         while (true) {
-            const res = os.system.openat(dir_fd, path, os.O_CLOEXEC | os.O_RDONLY | os.O_DIRECT, 0);
+            const res = os.system.openat(dir_fd, path, os.O.CLOEXEC | os.O.RDONLY | os.O.DIRECT, 0);
             switch (os.linux.getErrno(res)) {
                 0 => {
                     os.close(@intCast(os.fd_t, res));

package/src/tigerbeetle/src/test/cluster.zig

@@ -1,10 +1,14 @@
 const std = @import("std");
+const assert = std.debug.assert;
 const mem = std.mem;
 
 const config = @import("../config.zig");
 
 const StateChecker = @import("state_checker.zig").StateChecker;
 
+const MessagePool = @import("../message_pool.zig").MessagePool;
+const Message = MessagePool.Message;
+
 const Network = @import("network.zig").Network;
 const NetworkOptions = @import("network.zig").NetworkOptions;
 
@@ -13,9 +17,9 @@ const MessageBus = @import("message_bus.zig").MessageBus;
 const Storage = @import("storage.zig").Storage;
 const Time = @import("time.zig").Time;
 
-const vr = @import("../vr.zig");
-pub const Replica = vr.Replica(StateMachine, MessageBus, Storage, Time);
-pub const Client = vr.Client(StateMachine, MessageBus);
+const vsr = @import("../vsr.zig");
+pub const Replica = vsr.Replica(StateMachine, MessageBus, Storage, Time);
+pub const Client = vsr.Client(StateMachine, MessageBus);
 
 pub const ClusterOptions = struct {
     cluster: u32,
@@ -26,22 +30,26 @@ pub const ClusterOptions = struct {
     seed: u64,
 
     network_options: NetworkOptions,
+    storage_options: Storage.Options,
 };
 
 pub const Cluster = struct {
-    allocator: *mem.Allocator,
+    allocator: mem.Allocator,
     options: ClusterOptions,
 
     state_machines: []StateMachine,
     storages: []Storage,
+    times: []Time,
     replicas: []Replica,
 
     clients: []Client,
     network: Network,
 
+    // TODO: Initializing these fields in main() is a bit ugly
     state_checker: StateChecker = undefined,
+    on_change_state: fn (replica: *Replica) void = undefined,
 
-    pub fn create(allocator: *mem.Allocator, prng: *std.rand.Random, options: ClusterOptions) !*Cluster {
+    pub fn create(allocator: mem.Allocator, prng: std.rand.Random, options: ClusterOptions) !*Cluster {
         const cluster = try allocator.create(Cluster);
         errdefer allocator.destroy(cluster);
 
@@ -52,6 +60,9 @@ pub const Cluster = struct {
             const storages = try allocator.alloc(Storage, options.replica_count);
             errdefer allocator.free(storages);
 
+            const times = try allocator.alloc(Time, options.replica_count);
+            errdefer allocator.free(times);
+
             const replicas = try allocator.alloc(Replica, options.replica_count);
             errdefer allocator.free(replicas);
 
@@ -71,21 +82,31 @@ pub const Cluster = struct {
                 .options = options,
                 .state_machines = state_machines,
                 .storages = storages,
+                .times = times,
                 .replicas = replicas,
                 .clients = clients,
                 .network = network,
             };
         }
 
+        var buffer: [config.replicas_max]Storage.FaultyAreas = undefined;
+        const faulty_areas = Storage.generate_faulty_areas(prng, config.journal_size_max, options.replica_count, &buffer);
+
         for (cluster.replicas) |*replica, replica_index| {
-            const time: Time = .{
+            cluster.times[replica_index] = .{
                 .resolution = config.tick_ms * std.time.ns_per_ms,
                 .offset_type = .linear,
                 .offset_coefficient_A = 0,
                 .offset_coefficient_B = 0,
             };
             cluster.state_machines[replica_index] = StateMachine.init(options.seed);
-            cluster.storages[replica_index] = try Storage.init(allocator, config.journal_size_max);
+            cluster.storages[replica_index] = try Storage.init(
+                allocator,
+                config.journal_size_max,
+                options.storage_options,
+                @intCast(u8, replica_index),
+                faulty_areas[replica_index],
+            );
             const message_bus = try cluster.network.init_message_bus(
                 options.cluster,
                 .{ .replica = @intCast(u8, replica_index) },
@@ -96,7 +117,7 @@ pub const Cluster = struct {
                 options.cluster,
                 options.replica_count,
                 @intCast(u8, replica_index),
-                time,
+                &cluster.times[replica_index],
                 &cluster.storages[replica_index],
                 message_bus,
                 &cluster.state_machines[replica_index],
@@ -127,13 +148,10 @@ pub const Cluster = struct {
         for (cluster.clients) |*client| client.deinit();
         cluster.allocator.free(cluster.clients);
 
-        for (cluster.replicas) |*replica| replica.deinit();
+        for (cluster.replicas) |*replica| replica.deinit(cluster.allocator);
         cluster.allocator.free(cluster.replicas);
 
-        for (cluster.state_machines) |*state_machine| state_machine.deinit();
-        cluster.allocator.free(cluster.state_machines);
-
-        for (cluster.storages) |*storage| storage.deinit();
+        for (cluster.storages) |*storage| storage.deinit(cluster.allocator);
         cluster.allocator.free(cluster.storages);
 
         cluster.network.deinit();
@@ -141,5 +159,63 @@ pub const Cluster = struct {
         cluster.allocator.destroy(cluster);
     }
 
-    pub fn tick() void {}
+    /// Reset a replica to its initial state, simulating a random crash/panic.
+    /// Leave the persistent storage untouched, and leave any currently
+    /// inflight messages to/from the replica in the network.
+    pub fn simulate_replica_crash(cluster: *Cluster, replica_index: u8) !void {
+        const replica = &cluster.replicas[replica_index];
+        replica.deinit(cluster.allocator);
+
+        cluster.storages[replica_index].reset();
+        cluster.state_machines[replica_index] = StateMachine.init(cluster.options.seed);
+
+        // The message bus and network should be left alone, as messages
+        // may still be inflight to/from this replica. However, we should
+        // do a check to ensure that we aren't leaking any messages when
+        // deinitializing the replica above.
+        const packet_simulator = &cluster.network.packet_simulator;
+        // The same message may be used for multiple network packets, so simply counting how
+        // many packets are inflight from the replica is insufficient, we need to dedup them.
+        var messages_in_network_set = std.AutoHashMap(*Message, void).init(cluster.allocator);
+        defer messages_in_network_set.deinit();
+
+        var target: u8 = 0;
+        while (target < packet_simulator.options.node_count) : (target += 1) {
+            const path = .{ .source = replica_index, .target = target };
+            const queue = packet_simulator.path_queue(path);
+            var it = queue.iterator();
+            while (it.next()) |data| {
+                try messages_in_network_set.put(data.packet.message, {});
+            }
+        }
+
+        const messages_in_network = messages_in_network_set.count();
+
+        var messages_in_pool: usize = 0;
+        const message_bus = cluster.network.get_message_bus(.{ .replica = replica_index });
+        {
+            var it = message_bus.pool.free_list;
+            while (it) |message| : (it = message.next) messages_in_pool += 1;
+        }
+        {
+            var it = message_bus.pool.header_only_free_list;
+            while (it) |message| : (it = message.next) messages_in_pool += 1;
+        }
+
+        const total_messages = config.message_bus_messages_max + config.message_bus_headers_max;
+        assert(messages_in_network + messages_in_pool == total_messages);
+
+        replica.* = try Replica.init(
+            cluster.allocator,
+            cluster.options.cluster,
+            cluster.options.replica_count,
+            @intCast(u8, replica_index),
+            &cluster.times[replica_index],
+            &cluster.storages[replica_index],
+            message_bus,
+            &cluster.state_machines[replica_index],
+        );
+        message_bus.set_on_message(*Replica, replica, Replica.on_message);
+        replica.on_change_state = cluster.on_change_state;
+    }
 };

package/src/tigerbeetle/src/test/message_bus.zig

@@ -5,7 +5,7 @@ const config = @import("../config.zig");
 
 const MessagePool = @import("../message_pool.zig").MessagePool;
 const Message = MessagePool.Message;
-const Header = @import("../vr.zig").Header;
+const Header = @import("../vsr.zig").Header;
 
 const Network = @import("network.zig").Network;
 
@@ -25,11 +25,11 @@ pub const MessageBus = struct {
 
     /// The callback to be called when a message is received. Use set_on_message() to set
     /// with type safety for the context pointer.
-    on_message_callback: ?fn (context: ?*c_void, message: *Message) void = null,
-    on_message_context: ?*c_void = null,
+    on_message_callback: ?fn (context: ?*anyopaque, message: *Message) void = null,
+    on_message_context: ?*anyopaque = null,
 
     pub fn init(
-        allocator: *std.mem.Allocator,
+        allocator: std.mem.Allocator,
         cluster: u32,
         process: Process,
         network: *Network,
@@ -43,7 +43,7 @@ pub const MessageBus = struct {
     }
 
     /// TODO
-    pub fn deinit(bus: *MessageBus) void {}
+    pub fn deinit(_: *MessageBus) void {}
 
     pub fn set_on_message(
         bus: *MessageBus,
@@ -51,18 +51,15 @@ pub const MessageBus = struct {
         context: Context,
         comptime on_message: fn (context: Context, message: *Message) void,
     ) void {
-        assert(bus.on_message_callback == null);
-        assert(bus.on_message_context == null);
-
         bus.on_message_callback = struct {
-            fn wrapper(_context: ?*c_void, message: *Message) void {
+            fn wrapper(_context: ?*anyopaque, message: *Message) void {
                 on_message(@intToPtr(Context, @ptrToInt(_context)), message);
             }
         }.wrapper;
         bus.on_message_context = context;
     }
 
-    pub fn tick(self: *MessageBus) void {}
+    pub fn tick(_: *MessageBus) void {}
 
     pub fn get_message(bus: *MessageBus) ?*Message {
         return bus.pool.get_message();

package/src/tigerbeetle/src/test/network.zig

@@ -4,7 +4,7 @@ const mem = std.mem;
 const assert = std.debug.assert;
 
 const config = @import("../config.zig");
-const vr = @import("../vr.zig");
+const vsr = @import("../vsr.zig");
 
 const MessagePool = @import("../message_pool.zig").MessagePool;
 const Message = MessagePool.Message;
@@ -38,7 +38,7 @@ pub const Network = struct {
         target: Process,
     };
 
-    allocator: *std.mem.Allocator,
+    allocator: std.mem.Allocator,
 
     options: NetworkOptions,
     packet_simulator: PacketSimulator(Packet),
@@ -47,7 +47,7 @@ pub const Network = struct {
     processes: std.ArrayListUnmanaged(u128),
 
     pub fn init(
-        allocator: *std.mem.Allocator,
+        allocator: std.mem.Allocator,
         replica_count: u8,
         client_count: u8,
         options: NetworkOptions,
@@ -136,7 +136,7 @@ pub const Network = struct {
 
     fn deliver_message(packet: Packet, path: PacketSimulatorPath) void {
         const network = packet.network;
-        const source_bus = &network.busses.items[path.source];
+
         const target_bus = &network.busses.items[path.target];
 
         const message = target_bus.get_message() orelse {
@@ -159,7 +159,7 @@ pub const Network = struct {
         });
 
         if (message.header.command == .request or message.header.command == .prepare) {
-            const sector_ceil = vr.sector_ceil(message.header.size);
+            const sector_ceil = vsr.sector_ceil(message.header.size);
             if (message.header.size != sector_ceil) {
                 assert(message.header.size < sector_ceil);
                 assert(message.buffer.len == config.message_size_max + config.sector_size);