tigerbeetle-node 0.3.3 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tigerbeetle-node",
-  "version": "0.3.3",
+  "version": "0.5.0",
   "description": "TigerBeetle Node.js client",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -10,38 +10,38 @@
   },
   "preferUnplugged": true,
   "files": [
-    "LICENSE",
-    "README.md",
-    "package.json",
-    "tsconfig.json",
-    "yarn.lock",
     ".yarn/releases",
     ".yarnrc.yml",
+    "LICENSE",
+    "README.md",
     "dist",
     "!dist/client.node",
+    "package.json",
+    "scripts/download_node_headers.sh",
+    "scripts/postinstall.sh",
     "src/c.zig",
-    "src/index.ts",
-    "src/test.ts",
     "src/benchmark.ts",
+    "src/index.ts",
     "src/node.zig",
+    "src/test.ts",
     "src/tigerbeetle/src",
     "src/tigerbeetle/scripts",
     "src/translate.zig",
-    "scripts/download_node_headers.sh",
-    "scripts/postinstall.sh"
+    "tsconfig.json",
+    "yarn.lock"
   ],
   "engines": {
     "node": ">=14.0.0"
   },
   "scripts": {
-    "prepack": "git submodule deinit --all && git submodule update --init && yarn build",
     "benchmark": "./scripts/benchmark.sh",
     "build": "yarn build:tsc && yarn build:zig",
     "build:tsc": "tsc",
-    "build:zig": "mkdir -p dist &&  zig/zig build-lib -mcpu=baseline -OReleaseSafe -dynamic -lc -isystem build/node-$(node --version)/include/node src/node.zig -femit-bin=dist/client.node",
+    "build:zig": "mkdir -p dist && ZIG_SYSTEM_LINKER_HACK=1 zig/zig build-lib -mcpu=baseline -OReleaseSafe -dynamic -lc -isystem build/node-$(node --version)/include/node src/node.zig -fallow-shlib-undefined -femit-bin=dist/client.node",
     "clean": "rm -rf build dist node_modules src/zig-cache zig",
     "install:zig": "./src/tigerbeetle/scripts/install_zig.sh",
     "postinstall": "./scripts/postinstall.sh",
+    "prepack": "git submodule deinit --all && git submodule update --init && yarn build:tsc",
     "test": "yarn build && node dist/test"
   },
   "author": "Coil Technologies, Inc",

package/scripts/postinstall.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
 
-./src/tigerbeetle/scripts/install_zig.sh 0.8.0
+./src/tigerbeetle/scripts/install_zig.sh 0.8.1
 ./scripts/download_node_headers.sh
-mkdir -p dist &&  zig/zig build-lib -dynamic -lc -isystem build/node-$(node --version)/include/node src/node.zig -femit-bin=dist/client.node
+mkdir -p dist && ZIG_SYSTEM_LINKER_HACK=1 zig/zig build-lib -mcpu=baseline -OReleaseSafe -dynamic -lc -isystem build/node-$(node --version)/include/node src/node.zig -fallow-shlib-undefined -femit-bin=dist/client.node

package/src/benchmark.ts

@@ -20,7 +20,7 @@ const PREVIOUS_BENCHMARK = IS_RAW_REQUEST ? PREVIOUS_RAW_REQUEST_RESULT : PREVIO
 const TOLERANCE = 10 // percent that the benchmark is allowed to deviate from the previous benchmark
 
 const client = createClient({
-  cluster_id: 0x0a5ca1ab1ebee11en,
+  cluster_id: 1,
   replica_addresses: ['3001']
 })
 
@@ -185,7 +185,7 @@ const runBenchmarkRawReqeust = async () => {
   let maxCommitTransfersLatency = 0
   const start = Date.now()
 
-  for(let i = 0; i < transfers.length; i++) {
+  for (let i = 0; i < transfers.length; i++) {
     const ms1 = Date.now()
 
     const transferResults = await rawCreateTransfers(transfers[i])
@@ -265,7 +265,7 @@ const runBenchmark = async () => {
   let maxCommitTransfersLatency = 0
   const start = Date.now()
 
-  for(let i = 0; i < transfers.length; i++) {
+  for (let i = 0; i < transfers.length; i++) {
     const ms1 = Date.now()
 
     const transferResults = await client.createTransfers(transfers[i])
@@ -327,4 +327,4 @@ main().catch(error => {
   console.log(error)
 }).finally(async () => {
   await client.destroy()
-})
+})

package/src/index.ts

@@ -4,16 +4,17 @@ interface Binding {
   request: (context: Context, operation: Operation, batch: Event[], result: ResultCallback) => void
   raw_request: (context: Context, operation: Operation, raw_batch: Buffer, result: ResultCallback) => void
   tick: (context: Context) => void,
-  deinit: (context: Context) => void
+  deinit: (context: Context) => void,
+  tick_ms: number
 }
 
 interface BindingInitArgs {
-  cluster_id: bigint, // u128
+  cluster_id: number, // u32
   replica_addresses: Buffer,
 }
 
 export interface InitArgs {
-  cluster_id: bigint, // u128
+  cluster_id: number, // u32
   replica_addresses: Array<string | number>,
 }
 
@@ -129,7 +130,6 @@ export enum CommitTransferError {
   transfer_not_found,
   transfer_not_two_phase_commit,
   transfer_expired,
-  already_auto_committed,
   already_committed,
   already_committed_but_accepted,
   already_committed_but_rejected,
@@ -150,16 +150,18 @@ export type CommitTransfersError = {
 }
 
 export type AccountID = bigint // u128
+export type TransferID = bigint // u128
 
-export type Event = Account | Transfer | Commit | AccountID
-export type Result = CreateAccountsError | CreateTransfersError | CommitTransfersError | Account
+export type Event = Account | Transfer | Commit | AccountID | TransferID
+export type Result = CreateAccountsError | CreateTransfersError | CommitTransfersError | Account | Transfer
 export type ResultCallback = (error: undefined | Error, results: Result[]) => void
 
 export enum Operation {
-  CREATE_ACCOUNT = 2,
+  CREATE_ACCOUNT = 3,
   CREATE_TRANSFER,
   COMMIT_TRANSFER,
-  ACCOUNT_LOOKUP
+  ACCOUNT_LOOKUP,
+  TRANSFER_LOOKUP
 }
 
 export interface Client {
@@ -167,6 +169,7 @@ export interface Client {
   createTransfers: (batch: Transfer[]) => Promise<CreateTransfersError[]>
   commitTransfers: (batch: Commit[]) => Promise<CommitTransfersError[]>
   lookupAccounts: (batch: AccountID[]) => Promise<Account[]>
+  lookupTransfers: (batch: TransferID[]) => Promise<Transfer[]>
   request: (operation: Operation, batch: Event[], callback: ResultCallback) => void
   rawRequest: (operation: Operation, rawBatch: Buffer, callback: ResultCallback) => void
   destroy: () => void
@@ -236,6 +239,7 @@ export function createClient (args: InitArgs): Client {
       const callback = (error: undefined | Error, results: CreateAccountsError[]) => {
         if (error) {
           reject(error)
+          return
         }
         resolve(results)
       }
@@ -262,6 +266,7 @@ export function createClient (args: InitArgs): Client {
       const callback = (error: undefined | Error, results: CreateTransfersError[]) => {
         if (error) {
           reject(error)
+          return
         }
         resolve(results)
       }
@@ -288,6 +293,7 @@ export function createClient (args: InitArgs): Client {
       const callback = (error: undefined | Error, results: CommitTransfersError[]) => {
         if (error) {
           reject(error)
+          return
         }
         resolve(results)
       }
@@ -305,6 +311,7 @@ export function createClient (args: InitArgs): Client {
       const callback = (error: undefined | Error, results: Account[]) => {
         if (error) {
           reject(error)
+          return
         }
         resolve(results)
       }
@@ -317,6 +324,24 @@ export function createClient (args: InitArgs): Client {
     })
   }
 
+  const lookupTransfers = async (batch: TransferID[]): Promise<Transfer[]> => {
+    return new Promise((resolve, reject) => {
+      const callback = (error: undefined | Error, results: Transfer[]) => {
+        if (error) {
+          reject(error)
+          return
+        }
+        resolve(results)
+      }
+
+      try {
+        binding.request(context, Operation.TRANSFER_LOOKUP, batch, callback)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
   const destroy = (): void => {
     binding.deinit(context)
     if (_interval){
@@ -330,6 +355,7 @@ export function createClient (args: InitArgs): Client {
     createTransfers,
     commitTransfers,
     lookupAccounts,
+    lookupTransfers,
     request,
     rawRequest,
     destroy
@@ -337,7 +363,7 @@ export function createClient (args: InitArgs): Client {
 
   _interval = setInterval(() => {
     binding.tick(context)
-  }, 5)
+  }, binding.tick_ms)
 
   return _client
 }

package/src/node.zig

@@ -15,15 +15,15 @@ const CreateAccountsResult = tb.CreateAccountsResult;
 const CreateTransfersResult = tb.CreateTransfersResult;
 const CommitTransfersResult = tb.CommitTransfersResult;
 
-const Header = @import("tigerbeetle/src/vr.zig").Header;
-const Operation = @import("tigerbeetle/src/state_machine.zig").Operation;
+const StateMachine = @import("tigerbeetle/src/state_machine.zig").StateMachine;
+const Operation = StateMachine.Operation;
 const MessageBus = @import("tigerbeetle/src/message_bus.zig").MessageBusClient;
-const Client = @import("tigerbeetle/src/client.zig").Client;
-const ClientError = @import("tigerbeetle/src/client.zig").ClientError;
 const IO = @import("tigerbeetle/src/io.zig").IO;
 const config = @import("tigerbeetle/src/config.zig");
 
-const vr = @import("tigerbeetle/src/vr.zig");
+const vsr = @import("tigerbeetle/src/vsr.zig");
+const Header = vsr.Header;
+const Client = vsr.Client(StateMachine, MessageBus);
 
 pub const log_level: std.log.Level = .info;
 
@@ -35,6 +35,14 @@ export fn napi_register_module_v1(env: c.napi_env, exports: c.napi_value) c.napi
     translate.register_function(env, exports, "raw_request", raw_request) catch return null;
     translate.register_function(env, exports, "tick", tick) catch return null;
 
+    translate.u32_into_object(
+        env,
+        exports,
+        "tick_ms",
+        config.tick_ms,
+        "failed to add tick_ms to exports",
+    ) catch return null;
+
     const allocator = std.heap.c_allocator;
     var global = Globals.init(allocator, env) catch {
         std.log.emerg("Failed to initialise environment.\n", .{});
@@ -109,7 +117,7 @@ fn globalsCast(globals_raw: *c_void) *Globals {
 
 const Context = struct {
     io: *IO,
-    configuration: [32]std.net.Address,
+    addresses: []std.net.Address,
     message_bus: MessageBus,
     client: Client,
 
@@ -117,40 +125,41 @@ const Context = struct {
         env: c.napi_env,
         allocator: *std.mem.Allocator,
         io: *IO,
-        cluster: u128,
-        configuration_raw: []const u8,
+        cluster: u32,
+        addresses_raw: []const u8,
     ) !c.napi_value {
         const context = try allocator.create(Context);
         errdefer allocator.destroy(context);
 
         context.io = io;
 
-        const configuration = try vr.parse_configuration(allocator, configuration_raw);
-        errdefer allocator.free(configuration);
-        assert(configuration.len > 0);
-        for (configuration) |address, index| context.configuration[index] = address;
+        context.addresses = try vsr.parse_addresses(allocator, addresses_raw);
+        errdefer allocator.free(context.addresses);
+        assert(context.addresses.len > 0);
+
+        const client_id = std.crypto.random.int(u128);
 
         context.message_bus = try MessageBus.init(
             allocator,
             cluster,
-            context.configuration[0..configuration.len],
-            std.crypto.random.int(u128),
+            context.addresses,
+            client_id,
             context.io,
         );
         errdefer context.message_bus.deinit();
 
         context.client = try Client.init(
             allocator,
+            client_id,
             cluster,
-            @intCast(u16, configuration.len),
+            @intCast(u8, context.addresses.len),
             &context.message_bus,
         );
         errdefer context.client.deinit();
-        context.message_bus.process = .{ .client = &context.client };
 
-        const ret = try translate.create_external(env, context);
+        context.message_bus.set_on_message(*Client, &context.client, Client.on_message);
 
-        return ret;
+        return try translate.create_external(env, context);
     }
 };
 
@@ -204,7 +213,7 @@ fn decode_from_object(comptime T: type, env: c.napi_env, object: c.napi_value) !
             .credits_accepted = try translate.u64_from_object(env, object, "credits_accepted"),
             .timestamp = try validate_timestamp(env, object),
         },
-        u128 => try translate.u128_from_value(env, object, "Account lookup"),
+        u128 => try translate.u128_from_value(env, object, "lookup"),
         else => unreachable,
     };
 }
@@ -220,6 +229,7 @@ pub fn decode_events(
         .create_transfers => try decode_events_from_array(env, array, Transfer, output),
         .commit_transfers => try decode_events_from_array(env, array, Commit, output),
         .lookup_accounts => try decode_events_from_array(env, array, u128, output),
+        .lookup_transfers => try decode_events_from_array(env, array, u128, output),
         else => unreachable,
     };
 }
@@ -298,7 +308,7 @@ fn encode_napi_results_array(
                 );
             }
         },
-        else => {
+        Account => {
             var i: u32 = 0;
             while (i < results.len) : (i += 1) {
                 const result = results[i];
@@ -404,6 +414,105 @@ fn encode_napi_results_array(
                 );
             }
         },
+        Transfer => {
+            var i: u32 = 0;
+            while (i < results.len) : (i += 1) {
+                const result = results[i];
+                const napi_object = try translate.create_object(
+                    env,
+                    "Failed to create transfer lookup result object.",
+                );
+
+                try translate.u128_into_object(
+                    env,
+                    napi_object,
+                    "id",
+                    result.id,
+                    "Failed to set property \"id\" of transfer lookup result.",
+                );
+
+                try translate.u128_into_object(
+                    env,
+                    napi_object,
+                    "debit_account_id",
+                    result.debit_account_id,
+                    "Failed to set property \"debit_account_id\" of transfer lookup result.",
+                );
+
+                try translate.u128_into_object(
+                    env,
+                    napi_object,
+                    "credit_account_id",
+                    result.credit_account_id,
+                    "Failed to set property \"credit_account_id\" of transfer lookup result.",
+                );
+
+                try translate.u128_into_object(
+                    env,
+                    napi_object,
+                    "user_data",
+                    result.user_data,
+                    "Failed to set property \"user_data\" of transfer lookup result.",
+                );
+
+                try translate.byte_slice_into_object(
+                    env,
+                    napi_object,
+                    "reserved",
+                    &result.reserved,
+                    "Failed to set property \"reserved\" of transfer lookup result.",
+                );
+
+                try translate.u64_into_object(
+                    env,
+                    napi_object,
+                    "timeout",
+                    result.timeout,
+                    "Failed to set property \"timeout\" of transfer lookup result.",
+                );
+
+                try translate.u32_into_object(
+                    env,
+                    napi_object,
+                    "code",
+                    @intCast(u32, result.code),
+                    "Failed to set property \"code\" of transfer lookup result.",
+                );
+
+                try translate.u32_into_object(
+                    env,
+                    napi_object,
+                    "flags",
+                    @bitCast(u32, result.flags),
+                    "Failed to set property \"flags\" of transfer lookup result.",
+                );
+
+                try translate.u64_into_object(
+                    env,
+                    napi_object,
+                    "amount",
+                    result.amount,
+                    "Failed to set property \"amount\" of transfer lookup result.",
+                );
+
+                try translate.u64_into_object(
+                    env,
+                    napi_object,
+                    "timestamp",
+                    result.timestamp,
+                    "Failed to set property \"timestamp\" of transfer lookup result.",
+                );
+
+                try translate.set_array_element(
+                    env,
+                    napi_array,
+                    i,
+                    napi_object,
+                    "Failed to set element in results array.",
+                );
+            }
+        },
+        else => unreachable,
     }
 
     return napi_array;
@@ -421,8 +530,8 @@ fn init(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_value {
         "Function init() must receive 1 argument exactly.",
     ) catch return null;
 
-    const cluster = translate.u128_from_object(env, argv[0], "cluster_id") catch return null;
-    const configuration = translate.slice_from_object(
+    const cluster = translate.u32_from_object(env, argv[0], "cluster_id") catch return null;
+    const addresses = translate.slice_from_object(
         env,
         argv[0],
         "replica_addresses",
@@ -432,7 +541,8 @@ fn init(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_value {
 
     const globals_raw = translate.globals(env) catch return null;
     const globals = globalsCast(globals_raw.?);
-    const context = Context.create(env, allocator, &globals.io, cluster, configuration) catch {
+
+    const context = Context.create(env, allocator, &globals.io, cluster, addresses) catch {
         // TODO: switch on err and provide more detailed messages
         translate.throw(env, "Failed to initialize Client.") catch return null;
     };
@@ -440,7 +550,7 @@ fn init(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_value {
     return context;
 }
 
-/// This function decodes and validates an array of Node objects, one-by-one, directly into an 
+/// This function decodes and validates an array of Node objects, one-by-one, directly into an
 /// available message before requesting the client to send it.
 fn request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_value {
     var argc: usize = 4;
@@ -493,7 +603,7 @@ fn request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_valu
     return null;
 }
 
-/// The batch has already been encoded into a byte slice. This means that we only have to do one 
+/// The batch has already been encoded into a byte slice. This means that we only have to do one
 /// copy directly into an available message. No validation of the encoded data is performed except
 /// that it will fit into the message buffer.
 fn raw_request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_value {
@@ -547,7 +657,7 @@ fn raw_request(env: c.napi_env, info: c.napi_callback_info) callconv(.C) c.napi_
     return null;
 }
 
-fn create_client_error(env: c.napi_env, client_error: ClientError) !c.napi_value {
+fn create_client_error(env: c.napi_env, client_error: Client.Error) !c.napi_value {
     return switch (client_error) {
         error.TooManyOutstandingRequests => try translate.create_error(
             env,
@@ -556,7 +666,7 @@ fn create_client_error(env: c.napi_env, client_error: ClientError) !c.napi_value
     };
 }
 
-fn on_result(user_data: u128, operation: Operation, results: ClientError![]const u8) void {
+fn on_result(user_data: u128, operation: Operation, results: Client.Error![]const u8) void {
     // A reference to the user's JS callback was made in `request` or `raw_request`. This MUST be
     // cleaned up regardless of the result of this function.
     const env = @bitCast(translate.UserData, user_data).env;
@@ -581,7 +691,7 @@ fn on_result(user_data: u128, operation: Operation, results: ClientError![]const
 
     if (results) |value| {
         const napi_results = switch (operation) {
-            .reserved, .init => {
+            .reserved, .init, .register => {
                 translate.throw(env, "Reserved operation.") catch return;
             },
             .create_accounts => encode_napi_results_array(
@@ -600,6 +710,7 @@ fn on_result(user_data: u128, operation: Operation, results: ClientError![]const
                 value,
             ) catch return,
             .lookup_accounts => encode_napi_results_array(Account, env, value) catch return,
+            .lookup_transfers => encode_napi_results_array(Transfer, env, value) catch return,
         };
 
         argv[0] = globals.napi_undefined;

package/src/test.ts

@@ -1,16 +1,16 @@
 import assert, { AssertionError } from 'assert'
 import { CommitFlags,
+  createClient,
   Commit,
   Account,
-  createClient,
   Transfer,
   TransferFlags,
-  CreateTransferError,
-  CreateAccountError
+  CreateAccountError,
+  CreateTransferError
 } from '.'
 
 const client = createClient({
-  cluster_id: 0x0a5ca1ab1ebee11en,
+  cluster_id: 0,
   replica_addresses: ['3001']
 })
 
@@ -176,6 +176,20 @@ test('can create a two-phase transfer', async (): Promise<void> => {
   assert.strictEqual(accounts[1].credits_reserved, 0n)
   assert.strictEqual(accounts[1].debits_accepted, 100n)
   assert.strictEqual(accounts[1].debits_reserved, 50n)
+
+  // Lookup the transfer
+  const transfers = await client.lookupTransfers([transfer.id])
+  assert.strictEqual(transfers.length, 1)
+  assert.strictEqual(transfers[0].id, 1n)
+  assert.strictEqual(transfers[0].debit_account_id, accountB.id)
+  assert.strictEqual(transfers[0].credit_account_id, accountA.id)
+  assert.strictEqual(transfers[0].user_data, 0n)
+  assert.notStrictEqual(transfers[0].reserved, Zeroed32Bytes)
+  assert.strictEqual(transfers[0].timeout > 0, true)
+  assert.strictEqual(transfers[0].code, 1)
+  assert.strictEqual(transfers[0].flags, 2)
+  assert.strictEqual(transfers[0].amount, 50n)
+  assert.strictEqual(transfers[0].timestamp > 0, true)
 })
 
 test('can commit a two-phase transfer', async (): Promise<void> => {
@@ -312,4 +326,4 @@ async function main () {
 main().catch((error: AssertionError) => {
   console.log('operator:', error.operator)
   console.log('stack:', error.stack)
-})
+})

package/src/tigerbeetle/scripts/benchmark.sh

@@ -25,13 +25,20 @@ function onerror {
 }
 trap onerror EXIT
 
-CLUSTER_ID="--cluster-id=0a5ca1ab1ebee11e"
-REPLICA_ADDRESSES="--replica-addresses=3001"
+for I in 0
+do
+    echo "Initializing replica $I..."
+    FILE="./cluster_0000000000_replica_00${I}.tigerbeetle"
+    if [ -f $FILE ]; then
+        rm $FILE
+    fi
+    ./tigerbeetle init --directory=. --cluster=0 --replica=$I > benchmark.log 2>&1
+done
 
 for I in 0
 do
     echo "Starting replica $I..."
-    ./tigerbeetle $CLUSTER_ID $REPLICA_ADDRESSES --replica-index=$I > benchmark.log 2>&1 &
+    ./tigerbeetle start --directory=. --cluster=0 --addresses=3001 --replica=$I > benchmark.log 2>&1 &
 done
 
 # Wait for replicas to start, listen and connect:

package/src/tigerbeetle/scripts/install.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-scripts/install_zig.sh
+scripts/install_zig.sh 0.8.1
 echo "Building TigerBeetle..."
-zig/zig build -Drelease-safe
+zig/zig build -Dcpu=baseline -Drelease-safe
 mv zig-out/bin/tigerbeetle .