tigerbeetle-node 0.11.11 → 0.11.12

package/dist/.client.node.sha256

@@ -1 +1 @@
-21c6105d76e0efc68fe5cbe799d363a083a9b44359e16d7dbfbb172e381ea0c3  dist/client.node
+5d30352e917b04b1b6453bc9ba0d1c8f48673d155bee91caa8240e48130211ec  dist/client.node

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tigerbeetle-node",
-  "version": "0.11.11",
+  "version": "0.11.12",
   "description": "TigerBeetle Node.js client",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -24,9 +24,7 @@
     "src/index.ts",
     "src/node.zig",
     "src/test.ts",
-    "src/tigerbeetle/src/*.zig",
-    "src/tigerbeetle/src/{c,io,lsm,testing,vsr,state_machine}",
-    "src/tigerbeetle/scripts",
+    "src/tigerbeetle",
     "src/translate.zig",
     "tsconfig.json"
   ],
@@ -36,13 +34,12 @@
   "scripts": {
     "benchmark": "./scripts/benchmark.sh",
     "test": "node dist/test",
-    "postinstall": "npm run install_zig && npm run download_node_headers && npm run build_lib",
-    "install_zig": "sh ./src/tigerbeetle/scripts/install_zig.sh",
+    "postinstall": "npm run download_node_headers && npm run build_lib",
     "download_node_headers": "sh ./scripts/download_node_headers.sh",
     "build": "npm run build_tsc && npm run build_lib",
     "build_tsc": "./node_modules/typescript/bin/tsc",
     "build_lib": "sh ./scripts/build_lib.sh",
-    "prepack": "npm run build_tsc",
+    "prepack": "npm run build",
     "clean": "rm -rf build dist node_modules src/zig-cache zig"
   },
   "author": "TigerBeetle, Inc",

package/scripts/build_lib.sh

@@ -2,11 +2,30 @@
 
 set -e
 
+# This is a bit hacky. If the script is run inside tigerbeetle's repository,
+# copy sources over, so that we can package them into an npm package.
+#
+# If the script is run inside user's node_modules, do nothing, as we assume
+# the sources are already there.
+if [ -f "../../tigerbeetle.zig" ] && [ -d "../../clients/node" ]; then
+    rm -rf   ./src/tigerbeetle/src
+    mkdir -p ./src/tigerbeetle/src
+    cp -r ../../../scripts/ ./src/tigerbeetle/scripts/
+    cp       ../../*.zig    ./src/tigerbeetle/src/
+    for dir in io lsm testing vsr state_machine; do
+        cp -r    ../../$dir     ./src/tigerbeetle/src/$dir
+    done
+fi
+
+if ! [ -f "./zig/zig" ]; then
+    ./src/tigerbeetle/scripts/install_zig.sh
+fi
+
 # macOS 13 Ventura is not supported on Zig 0.9.x.
 # Overriding -target is one workaround Andrew suggests.
 # https://github.com/ziglang/zig/issues/10478#issuecomment-1294313967
 target=""
-if [ "$(./zig/zig targets | grep triple |cut -d '"' -f 4 | cut -d '.' -f 1,2)" = "aarch64-macos.13" ]; then
+if [ "$(./zig/zig targets | grep triple |cut -d '"' -f 4 | cut -d '.' -f 1,2 | cut -d '-' -f 2)" = "macos.13" ]; then
     target="-target native-macos.11"
 fi
 
@@ -24,11 +43,12 @@ else
     echo "Building for '$target'"
 fi
 
-mkdir -p dist
+mkdir -p ./dist
 
 # Need to do string eval-ing because of shellcheck's strict string
 # interpolation rules.
 cmd="./zig/zig build-lib \
+	--global-cache-dir ./zig/global-cache \
 	-mcpu=baseline \
 	-OReleaseSafe \
 	-dynamic \

package/src/tigerbeetle/scripts/benchmark.bat

@@ -42,9 +42,6 @@ for /l %%i in (0, 1, 0) do (
     start /B "tigerbeetle_%%i" .\tigerbeetle.exe start --addresses=3001 !ZIG_FILE! > benchmark.log 2>&1
 )
 
-rem Wait for replicas to start, listen and connect:
-timeout /t 2
-
 echo.
 echo Benchmarking...
 zig\zig.exe build benchmark -Drelease-safe

package/src/tigerbeetle/scripts/benchmark.sh

@@ -52,9 +52,6 @@ do
     ./tigerbeetle start --addresses=3001 "$FILE" >> benchmark.log 2>&1 &
 done
 
-# Wait for replicas to start, listen and connect:
-sleep 1
-
 echo ""
 echo "Benchmarking..."
 zig/zig build benchmark -Drelease-safe

package/src/tigerbeetle/scripts/scripts/benchmark.bat

@@ -0,0 +1,48 @@
+@echo off
+setlocal enabledelayedexpansion
+
+REM Install Zig if it does not already exist:
+if not exist "zig" (
+    call .\scripts\install_zig.bat
+)
+
+if "%~1" equ ":main" (
+    shift /1
+    goto main
+)
+
+cmd /d /c "%~f0" :main %*
+set ZIG_RESULT=%ERRORLEVEL%
+taskkill /F /IM tigerbeetle.exe >nul
+
+if !ZIG_RESULT! equ 0 (
+    del /f benchmark.log
+) else (
+    echo.
+    echo Error running benchmark, here are more details
+    type benchmark.log
+)
+
+echo.
+exit /b
+
+:main
+zig\zig.exe build install -Drelease-safe
+
+for /l %%i in (0, 1, 0) do (
+    echo Initializing replica %%i
+    set ZIG_FILE=.\0_%%i.tigerbeetle.benchmark
+    if exist "!ZIG_FILE!" DEL /F "!ZIG_FILE!"
+    .\tigerbeetle.exe format --cluster=0 --replica=%%i !ZIG_FILE! > benchmark.log 2>&1
+)
+
+for /l %%i in (0, 1, 0) do (
+    echo Starting replica %%i
+    set ZIG_FILE=.\0_%%i.tigerbeetle.benchmark
+    start /B "tigerbeetle_%%i" .\tigerbeetle.exe start --addresses=3001 !ZIG_FILE! > benchmark.log 2>&1
+)
+
+echo.
+echo Benchmarking...
+zig\zig.exe build benchmark -Drelease-safe
+exit /b %errorlevel%

package/src/tigerbeetle/scripts/scripts/benchmark.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -eEuo pipefail
+
+# Number of replicas to benchmark
+REPLICAS=${REPLICAS:-0}
+
+# Install Zig if it does not already exist:
+if [ ! -d "zig" ]; then
+    scripts/install_zig.sh
+fi
+
+COLOR_RED='\033[1;31m'
+COLOR_END='\033[0m'
+
+zig/zig build install -Drelease-safe
+
+function onerror {
+    if [ "$?" == "0" ]; then
+        rm benchmark.log
+    else
+        echo -e "${COLOR_RED}"
+        echo "Error running benchmark, here are more details (from benchmark.log):"
+        echo -e "${COLOR_END}"
+        cat benchmark.log
+    fi
+
+    for I in $REPLICAS
+    do
+        echo "Stopping replica $I..."
+    done
+    kill %1
+}
+trap onerror EXIT
+
+for I in $REPLICAS
+do
+    echo "Formatting replica $I..."
+
+    # Be careful to use a benchmark-specific filename so that we don't erase a real data file:
+    FILE="./0_${I}.tigerbeetle.benchmark"
+    if [ -f "$FILE" ]; then
+        rm "$FILE"
+    fi
+
+    ./tigerbeetle format --cluster=0 --replica="$I" "$FILE" > benchmark.log 2>&1
+done
+
+for I in $REPLICAS
+do
+    echo "Starting replica $I..."
+    FILE="./0_${I}.tigerbeetle.benchmark"
+    ./tigerbeetle start --addresses=3001 "$FILE" >> benchmark.log 2>&1 &
+done
+
+echo ""
+echo "Benchmarking..."
+zig/zig build benchmark -Drelease-safe
+echo ""
+
+for I in $REPLICAS
+do
+    FILE="./0_${I}.tigerbeetle.benchmark"
+    if [ -f "$FILE" ]; then
+        rm "$FILE"
+    fi
+done

package/src/tigerbeetle/scripts/scripts/confirm_image.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ -z "$1" ]]; then
+    echo "Expected a first argument with a Docker image name or id"
+    exit 2
+fi
+
+if [[ "$2" != "--want-production" ]] && [[ "$2" != "--want-debug" ]]; then
+    echo "Expected a second argument: --want-production or --want-debug".
+    exit 2
+fi
+
+check="
+set -eu
+
+apt-get update -y
+apt-get install -y binutils
+
+is_production=false
+# This getSymbolFromDwarf symbol only appears to be in the debug
+# build, not the release build. So that's the test!
+if ! [[ \$(nm -an /opt/beta-beetle/tigerbeetle | grep getSymbolFromDwarf) ]]; then
+  is_production=true
+fi
+"
+
+if [[ "$2" == '--want-production' ]]; then
+    cmd+="
+if [[ \$is_production == false ]]; then
+  echo 'Does not seem to be a production build'
+  exit 1
+fi"
+else
+    cmd+="
+if [[ \$is_production == true ]]; then
+  echo 'Does not seem to be a debug build'
+  exit 1
+fi"
+fi
+
+# We accept a passed-in arg 
+docker run --entrypoint bash "$1" -c "$check"

package/src/tigerbeetle/scripts/scripts/fuzz_loop.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+set -eu
+
+# Repeatedly runs some zig build command with different seeds and stores the output in the current directory.
+# Eg `fuzz_repeatedly.sh fuzz_lsm_forest` will run `zig build fuzz_lsm_forest -- seed $SEED > fuzz_lsm_forest_fuzz_${SEED}`
+# Use ./fuzz_unique_errors.sh to analyze the results.
+
+FUZZ_COMMAND=$1
+
+while true; do
+  SEED=$(od -A n -t u8 -N 8 /dev/urandom | xargs)
+  (zig build "$FUZZ_COMMAND" -Drelease-safe=true -- --seed "$SEED") || \
+    (zig build "$FUZZ_COMMAND" -- --seed "$SEED" 2> "fuzz_${FUZZ_COMMAND}_${SEED}") || \
+    true
+done

package/src/tigerbeetle/scripts/scripts/fuzz_unique_errors.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+set -eu
+
+# After running ./fuzz_loop.sh use this script to produce a list of unique crashes.
+# As a heuristic, we look for the first line of the stacktrace that occurs inside tigerbeetle code.
+
+grep -m 1 'tigerbeetle/src' fuzz_* | sort -u -t':' -k2,2

package/src/tigerbeetle/scripts/scripts/install.bat

@@ -0,0 +1,7 @@
+@echo off
+setlocal enabledelayedexpansion
+
+call .\scripts\install_zig.bat
+
+echo "Building TigerBeetle..."
+.\zig\zig.exe build install -Dcpu=baseline -Drelease-safe

package/src/tigerbeetle/scripts/scripts/install.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+
+if [ -z "$DEBUG" ]; then
+    debug="$DEBUG"
+fi
+
+debug="$DEBUG"
+if [ "$1" = "--debug" ]; then
+    debug="true"
+fi
+
+set -eu
+
+scripts/install_zig.sh
+if [ "$debug" = "true" ]; then
+    echo "Building Tigerbeetle debug..."
+    zig/zig build install -Dcpu=baseline
+else
+    echo "Building TigerBeetle..."
+    zig/zig build install -Dcpu=baseline -Drelease-safe
+fi

package/src/tigerbeetle/scripts/scripts/install_zig.bat

@@ -0,0 +1,113 @@
+@echo off
+
+set ZIG_RELEASE_DEFAULT=0.9.1
+
+:: Determine the Zig build:
+if "%~1"=="" (
+    set ZIG_RELEASE=%ZIG_RELEASE_DEFAULT%
+) else if "%~1"=="latest" (
+    set ZIG_RELEASE=builds
+) else (
+    set ZIG_RELEASE=%~1
+)
+
+:: Checks format of release version.
+echo.%ZIG_RELEASE% | findstr /b /r /c:"builds" /c:"^[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*">nul || (echo.Unexpected release format. && exit 1)
+
+set ZIG_OS=windows
+set ZIG_ARCH=x86_64
+
+set ZIG_TARGET=zig-%ZIG_OS%-%ZIG_ARCH%
+
+:: Determine the build, split the JSON line on whitespace and extract the 2nd field:
+for /f "tokens=2" %%a in ('curl --silent https://ziglang.org/download/index.json ^| findstr %ZIG_TARGET% ^| findstr %ZIG_RELEASE%' ) do (
+  set ZIG_URL=%%a
+)
+
+:: Then remove quotes and commas:
+for /f %%b in ("%ZIG_URL:,=%") do (
+    set ZIG_URL=%%~b
+)
+
+:: Checks the ZIG_URL variable follows the expected format.
+echo.%ZIG_URL% | findstr /b /r /c:"https://ziglang.org/builds/" /c:"https://ziglang.org/download/%ZIG_RELEASE%">nul || (echo.Unexpected release URL format. && exit 1)
+
+if "%ZIG_RELEASE%"=="builds" (
+    echo Installing Zig latest build...
+) else (
+    echo Installing Zig %ZIG_RELEASE% release build...
+)
+
+:: Using variable modifiers to determine the directory and filename from the URL:
+:: %%~ni Expands %%i to a file name only and %%~xi Expands %%i to a file name extension only.
+for /f %%i in ("%ZIG_URL%") do (
+    set ZIG_DIRECTORY=%%~ni
+    set ZIG_TARBALL=%%~nxi
+)
+
+:: Checks the ZIG_DIRECTORY variable follows the expected format.
+echo.%ZIG_DIRECTORY% | findstr /b /r /c:"zig-win64-" /c:"zig-windows-x86_64-">nul || (echo.Unexpected zip directory name format. && exit 1)
+
+:: Making sure we download to the same output document, without wget adding "-1" etc. if the file was previously partially downloaded:
+if exist %ZIG_TARBALL% (
+  del /q %ZIG_TARBALL%
+  if exist %ZIG_TARBALL% (
+    echo Failed to delete %ZIG_TARBALL%.
+    exit 1
+  )
+)
+
+echo Downloading %ZIG_URL%...
+curl --silent --progress-bar --output %ZIG_TARBALL% %ZIG_URL%
+if not exist  %ZIG_TARBALL% (
+  echo Failed to download Zig zip file.
+  exit 1
+)
+
+:: Replace any existing Zig installation so that we can install or upgrade:
+echo Removing any existing 'zig' and %ZIG_DIRECTORY% folders before extracting.
+if exist zig\ (
+  rd /s /q zig\
+  :: Ensure the directory has been deleted.
+  if exist zig\ (
+    echo The ‘zig’ directory could not be deleted.
+    exit 1
+  )
+)
+
+if exist %ZIG_DIRECTORY%\ (
+  rd /s /q %ZIG_DIRECTORY%
+  :: Ensure the directory has been deleted.
+  if exist %ZIG_DIRECTORY% (
+    echo The %ZIG_DIRECTORY% directory could not be deleted.
+    exit 1
+  )
+)
+
+:: Extract and then remove the downloaded tarball:
+echo Extracting %ZIG_TARBALL%...
+
+:: Hiding Powershell's progress bar during the extraction
+SET PS_DISABLE_PROGRESS="$ProgressPreference=[System.Management.Automation.ActionPreference]::SilentlyContinue"
+powershell -Command "%PS_DISABLE_PROGRESS%;Expand-Archive %ZIG_TARBALL% -DestinationPath ."
+
+if not exist %ZIG_TARBALL% (
+  echo Failed to extract zip file.
+  exit 1
+)
+
+echo Installing %ZIG_DIRECTORY% to 'zig' in current working directory...
+ren %ZIG_DIRECTORY% zig
+if exist %ZIG_DIRECTORY% (
+  echo Failed to rename %ZIG_DIRECTORY% to zig.
+  exit 1
+)
+
+:: Removes the zip file
+del /q %ZIG_TARBALL%
+if exist %ZIG_TARBALL% (
+  echo Failed to delete %ZIG_TARBALL% file.
+  exit 1
+)
+
+echo "Congratulations, you have successfully installed Zig version %ZIG_RELEASE%. Enjoy!"

package/src/tigerbeetle/scripts/scripts/install_zig.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env sh
+set -eu
+
+ZIG_RELEASE_DEFAULT="0.9.1"
+# Default to the release build, or allow the latest dev build, or an explicit release version:
+ZIG_RELEASE=${1:-$ZIG_RELEASE_DEFAULT}
+if [ "$ZIG_RELEASE" = "latest" ]; then
+    ZIG_RELEASE="builds"
+fi
+
+# Validate the release version explicitly:
+if echo "$ZIG_RELEASE" | grep -q '^builds$'; then
+    echo "Installing Zig latest build..."
+elif echo "$ZIG_RELEASE" | grep -q '^[0-9]\+.[0-9]\+.[0-9]\+$'; then
+    echo "Installing Zig $ZIG_RELEASE release build..."
+else
+    echo "Release version invalid"
+    exit 1
+fi
+
+# Determine the architecture:
+if [ "$(uname -m)" = 'arm64' ] || [ "$(uname -m)" = 'aarch64' ]; then
+    ZIG_ARCH="aarch64"
+else
+    ZIG_ARCH="x86_64"
+fi
+
+# Determine the operating system:
+if [ "$(uname)" = "Linux" ]; then
+    ZIG_OS="linux"
+else
+    ZIG_OS="macos"
+fi
+
+ZIG_TARGET="zig-$ZIG_OS-$ZIG_ARCH"
+
+# Determine the build, split the JSON line on whitespace and extract the 2nd field, then remove quotes and commas:
+if command -v wget; then
+    # -4 forces `wget` to connect to ipv4 addresses, as ipv6 fails to resolve on certain distros.
+    # Only A records (for ipv4) are used in DNS:
+    ipv4="-4"
+    # But Alpine doesn't support this argument
+    if [ -f /etc/alpine-release ]; then
+	ipv4=""
+    fi
+    ZIG_URL=$(wget $ipv4 --quiet -O - https://ziglang.org/download/index.json | grep -F "$ZIG_TARGET" | grep -F "$ZIG_RELEASE" | awk '{print $2}' | sed 's/[",]//g')
+else
+    ZIG_URL=$(curl --silent https://ziglang.org/download/index.json | grep -F "$ZIG_TARGET" | grep -F "$ZIG_RELEASE" | awk '{print $2}' | sed 's/[",]//g')
+fi
+
+# Ensure that the release is actually hosted on the ziglang.org website:
+if [ -z "$ZIG_URL" ]; then
+    echo "Release not found on ziglang.org"
+    exit 1
+fi
+
+# Work out the filename from the URL, as well as the directory without the ".tar.xz" file extension:
+ZIG_TARBALL=$(basename "$ZIG_URL")
+ZIG_DIRECTORY=$(basename "$ZIG_TARBALL" .tar.xz)
+
+# Download, making sure we download to the same output document, without wget adding "-1" etc. if the file was previously partially downloaded:
+echo "Downloading $ZIG_URL..."
+if command -v wget; then
+    # -4 forces `wget` to connect to ipv4 addresses, as ipv6 fails to resolve on certain distros.
+    # Only A records (for ipv4) are used in DNS:
+    ipv4="-4"
+    # But Alpine doesn't support this argument
+    if [ -f /etc/alpine-release ]; then
+	ipv4=""
+    fi
+    wget $ipv4 --quiet --output-document="$ZIG_TARBALL" "$ZIG_URL"
+else
+    curl --silent --output "$ZIG_TARBALL" "$ZIG_URL"
+fi
+
+# Extract and then remove the downloaded tarball:
+echo "Extracting $ZIG_TARBALL..."
+tar -xf "$ZIG_TARBALL"
+rm "$ZIG_TARBALL"
+
+# Replace any existing Zig installation so that we can install or upgrade:
+echo "Installing $ZIG_DIRECTORY to 'zig' in current working directory..."
+rm -rf zig
+mv "$ZIG_DIRECTORY" zig
+
+# It's up to the user to add this to their path if they want to:
+ZIG_BIN="$(pwd)/zig/zig"
+
+ZIG_VERSION=$($ZIG_BIN version)
+echo "Congratulations, you have successfully installed Zig $ZIG_VERSION to $ZIG_BIN. Enjoy!"