tigerbeetle-node 0.11.0 → 0.11.2

package/src/tigerbeetle/src/benchmarks/perf.zig

@@ -1,299 +0,0 @@
-// Copyright (c) 2015-2021, Zig contributors
-// Backported from https://github.com/ziglang/zig/blob/master/lib/std/os/linux.zig
-// TODO Remove this file once we upgrade to Zig 0.9.0.
-const std = @import("std");
-const pid_t = std.os.linux.pid_t;
-const fd_t = std.os.linux.fd_t;
-
-pub fn perf_event_open(
-    attr: *perf_event_attr,
-    pid: pid_t,
-    cpu: i32,
-    group_fd: fd_t,
-    flags: usize,
-) !fd_t {
-    const rc = perf_event_open_internal(attr, pid, cpu, group_fd, flags);
-    const errno = std.os.errno(rc);
-    if (errno != 0) {
-        std.log.err("perf_event_open_internal errno={}", .{errno});
-        return error.Unexpected;
-    }
-    return @intCast(fd_t, rc);
-}
-
-fn perf_event_open_internal(
-    attr: *perf_event_attr,
-    pid: pid_t,
-    cpu: i32,
-    group_fd: fd_t,
-    flags: usize,
-) usize {
-    return std.os.linux.syscall5(
-        .perf_event_open,
-        @ptrToInt(attr),
-        @bitCast(usize, @as(isize, pid)),
-        @bitCast(usize, @as(isize, cpu)),
-        @bitCast(usize, @as(isize, group_fd)),
-        flags,
-    );
-}
-
-pub const perf_event_attr = extern struct {
-    /// Major type: hardware/software/tracepoint/etc.
-    type: PERF.TYPE = undefined,
-    /// Size of the attr structure, for fwd/bwd compat.
-    size: u32 = @sizeOf(perf_event_attr),
-    /// Type specific configuration information.
-    config: u64 = 0,
-
-    sample_period_or_freq: u64 = 0,
-    sample_type: u64 = 0,
-    read_format: u64 = 0,
-
-    flags: packed struct {
-        /// off by default
-        disabled: bool = false,
-        /// children inherit it
-        inherit: bool = false,
-        /// must always be on PMU
-        pinned: bool = false,
-        /// only group on PMU
-        exclusive: bool = false,
-        /// don't count user
-        exclude_user: bool = false,
-        /// ditto kernel
-        exclude_kernel: bool = false,
-        /// ditto hypervisor
-        exclude_hv: bool = false,
-        /// don't count when idle
-        exclude_idle: bool = false,
-        /// include mmap data
-        mmap: bool = false,
-        /// include comm data
-        comm: bool = false,
-        /// use freq, not period
-        freq: bool = false,
-        /// per task counts
-        inherit_stat: bool = false,
-        /// next exec enables
-        enable_on_exec: bool = false,
-        /// trace fork/exit
-        task: bool = false,
-        /// wakeup_watermark
-        watermark: bool = false,
-        /// precise_ip:
-        ///
-        ///  0 - SAMPLE_IP can have arbitrary skid
-        ///  1 - SAMPLE_IP must have constant skid
-        ///  2 - SAMPLE_IP requested to have 0 skid
-        ///  3 - SAMPLE_IP must have 0 skid
-        ///
-        ///  See also PERF_RECORD_MISC_EXACT_IP
-        /// skid constraint
-        precise_ip: u2 = 0,
-        /// non-exec mmap data
-        mmap_data: bool = false,
-        /// sample_type all events
-        sample_id_all: bool = false,
-
-        /// don't count in host
-        exclude_host: bool = false,
-        /// don't count in guest
-        exclude_guest: bool = false,
-
-        /// exclude kernel callchains
-        exclude_callchain_kernel: bool = false,
-        /// exclude user callchains
-        exclude_callchain_user: bool = false,
-        /// include mmap with inode data
-        mmap2: bool = false,
-        /// flag comm events that are due to an exec
-        comm_exec: bool = false,
-        /// use @clockid for time fields
-        use_clockid: bool = false,
-        /// context switch data
-        context_switch: bool = false,
-        /// Write ring buffer from end to beginning
-        write_backward: bool = false,
-        /// include namespaces data
-        namespaces: bool = false,
-
-        __reserved_1: u35 = 0,
-    } = .{},
-    /// wakeup every n events, or
-    /// bytes before wakeup
-    wakeup_events_or_watermark: u32 = 0,
-
-    bp_type: u32 = 0,
-
-    /// This field is also used for:
-    /// bp_addr
-    /// kprobe_func for perf_kprobe
-    /// uprobe_path for perf_uprobe
-    config1: u64 = 0,
-    /// This field is also used for:
-    /// bp_len
-    /// kprobe_addr when kprobe_func == null
-    /// probe_offset for perf_[k,u]probe
-    config2: u64 = 0,
-
-    /// enum perf_branch_sample_type
-    branch_sample_type: u64 = 0,
-
-    /// Defines set of user regs to dump on samples.
-    /// See asm/perf_regs.h for details.
-    sample_regs_user: u64 = 0,
-
-    /// Defines size of the user stack to dump on samples.
-    sample_stack_user: u32 = 0,
-
-    clockid: i32 = 0,
-    /// Defines set of regs to dump for each sample
-    /// state captured on:
-    ///  - precise = 0: PMU interrupt
-    ///  - precise > 0: sampled instruction
-    ///
-    /// See asm/perf_regs.h for details.
-    sample_regs_intr: u64 = 0,
-
-    /// Wakeup watermark for AUX area
-    aux_watermark: u32 = 0,
-    sample_max_stack: u16 = 0,
-    /// Align to u64
-    __reserved_2: u16 = 0,
-};
-
-pub const PERF = struct {
-    pub const TYPE = enum(u32) {
-        HARDWARE,
-        SOFTWARE,
-        TRACEPOINT,
-        HW_CACHE,
-        RAW,
-        BREAKPOINT,
-        MAX,
-    };
-
-    pub const COUNT = struct {
-        pub const HW = enum(u32) {
-            CPU_CYCLES,
-            INSTRUCTIONS,
-            CACHE_REFERENCES,
-            CACHE_MISSES,
-            BRANCH_INSTRUCTIONS,
-            BRANCH_MISSES,
-            BUS_CYCLES,
-            STALLED_CYCLES_FRONTEND,
-            STALLED_CYCLES_BACKEND,
-            REF_CPU_CYCLES,
-            MAX,
-
-            pub const CACHE = enum(u32) {
-                L1D,
-                L1I,
-                LL,
-                DTLB,
-                ITLB,
-                BPU,
-                NODE,
-                MAX,
-
-                pub const OP = enum(u32) {
-                    READ,
-                    WRITE,
-                    PREFETCH,
-                    MAX,
-                };
-
-                pub const RESULT = enum(u32) {
-                    ACCESS,
-                    MISS,
-                    MAX,
-                };
-            };
-        };
-
-        pub const SW = enum(u32) {
-            CPU_CLOCK,
-            TASK_CLOCK,
-            PAGE_FAULTS,
-            CONTEXT_SWITCHES,
-            CPU_MIGRATIONS,
-            PAGE_FAULTS_MIN,
-            PAGE_FAULTS_MAJ,
-            ALIGNMENT_FAULTS,
-            EMULATION_FAULTS,
-            DUMMY,
-            BPF_OUTPUT,
-            MAX,
-        };
-    };
-
-    pub const SAMPLE = struct {
-        pub const IP = 1;
-        pub const TID = 2;
-        pub const TIME = 4;
-        pub const ADDR = 8;
-        pub const READ = 16;
-        pub const CALLCHAIN = 32;
-        pub const ID = 64;
-        pub const CPU = 128;
-        pub const PERIOD = 256;
-        pub const STREAM_ID = 512;
-        pub const RAW = 1024;
-        pub const BRANCH_STACK = 2048;
-        pub const REGS_USER = 4096;
-        pub const STACK_USER = 8192;
-        pub const WEIGHT = 16384;
-        pub const DATA_SRC = 32768;
-        pub const IDENTIFIER = 65536;
-        pub const TRANSACTION = 131072;
-        pub const REGS_INTR = 262144;
-        pub const PHYS_ADDR = 524288;
-        pub const MAX = 1048576;
-
-        pub const BRANCH = struct {
-            pub const USER = 1 << 0;
-            pub const KERNEL = 1 << 1;
-            pub const HV = 1 << 2;
-            pub const ANY = 1 << 3;
-            pub const ANY_CALL = 1 << 4;
-            pub const ANY_RETURN = 1 << 5;
-            pub const IND_CALL = 1 << 6;
-            pub const ABORT_TX = 1 << 7;
-            pub const IN_TX = 1 << 8;
-            pub const NO_TX = 1 << 9;
-            pub const COND = 1 << 10;
-            pub const CALL_STACK = 1 << 11;
-            pub const IND_JUMP = 1 << 12;
-            pub const CALL = 1 << 13;
-            pub const NO_FLAGS = 1 << 14;
-            pub const NO_CYCLES = 1 << 15;
-            pub const TYPE_SAVE = 1 << 16;
-            pub const MAX = 1 << 17;
-        };
-    };
-
-    pub const FLAG = struct {
-        pub const FD_NO_GROUP = 1 << 0;
-        pub const FD_OUTPUT = 1 << 1;
-        pub const PID_CGROUP = 1 << 2;
-        pub const FD_CLOEXEC = 1 << 3;
-    };
-
-    pub const EVENT_IOC = struct {
-        pub const ENABLE = 9216;
-        pub const DISABLE = 9217;
-        pub const REFRESH = 9218;
-        pub const RESET = 9219;
-        pub const PERIOD = 1074275332;
-        pub const SET_OUTPUT = 9221;
-        pub const SET_FILTER = 1074275334;
-        pub const SET_BPF = 1074013192;
-        pub const PAUSE_OUTPUT = 1074013193;
-        pub const QUERY_BPF = 3221758986;
-        pub const MODIFY_ATTRIBUTES = 1074275339;
-    };
-
-    pub const IOC_FLAG_GROUP = 1;
-};

package/src/tigerbeetle/src/vopr_hub/README.md

@@ -1,58 +0,0 @@
-# The VOPR Hub
-
-*The VOPR Hub collects the seeds for crash, liveness and correctness reports from the VOPR simulators running permanently in CI or on local dev machines. The VOPR Hub then checks out that commit, replays the seed locally in Debug mode, collects the logs, and automatically creates a GitHub issue, also deduplicating against existing issues if necessary.*
-
-## The VOPR
-
-*The Viewstamped Operation Replicator* provides deterministic simulation testing for TigerBeetle. It tests that clusters of TigerBeetle replicas and clients interact correctly according to TigerBeetle's Viewstamped Replication consensus protocol, even under the pressure of simulated network and storage faults.
-
-The VOPR has an optional `--send` flag that enables it to send bug reports to the VOPR Hub. This flag can only be used when any local code changes have been committed and pushed. The reason for this is that for the hub to replay a failing seed it needs to run that seed on the same commit to get the same result.
-
-If the VOPR discovers a failing seed it creates a bug report in the format of a fixed length byte array.
-
-* 16 bytes contain the first half of a SHA256 hash of the remainder of the message, which is used both as a checksum, and to reject clearly misdirected or random network packets.
-* 1 byte indicates the type of bug detected (correctness, liveness, or crash).
-* 8 bytes are reserved for the seed.
-* The final 20 bytes contain the hash of the git commit to which the seed applies.
-
-## Hub Logic
-
-The hub listens for bug reports sent by any VOPR via TCP.
-
-### Setup
-The VOPR Hub must run a VOPR in a separate tigerbeetle directory to prevent it from checking out a commit that could change the hub itself.
-
-The VOPR which the hub runs must be run inside its own tigerbeetle directory in order to parse its output correctly.
-
-To run the VOPR Hub, Zig must be installed and five environmental variables are required:
-1. TIGERBEETLE_DIRECTORY the location of the VOPR's tigerbeetle directory
-2. ISSUE_DIRECTORY where issues are stored on disk
-3. DEVELOPER_TOKEN for access to GitHub
-4. VOPR_HUB_ADDRESS for the IP address to listen on for incoming messages
-5. REPOSITORY_URL to post the GitHub issue
-
-### Validation
-
-When the hub receives a message it first validates it immediately. Messages are expected to be exactly 45 bytes in length. The hub hashes the last 29 bytes of the message and ensures that the first half of the SHA256 hash matches the first 16 bytes of the message. If the hash is correct then the hub checks that the first byte (representing the bug type) is between 1 and 3. After the 64-bit unsigned seed, the remaining 20 bytes are the GitHub commit hash, which must all decode to valid hex characters.
-
-Once validated, the message is decoded and added to a queue for processing.
-
-### Replies to the VOPR
-
-Once the message is determined to be valid then a reply of "1" is sent back to the VOPR and the connection is closed. If it's invalid, the connection is simply closed (following the principle of “Don't talk to strangers”) and no further processing is applied.
-
-### Message Processing
-
-When the hub replays a seed it will save the logs to disk. This way each issue can be tracked to see if it has already been submitted. For correctness bugs (bug 1) and liveness bugs (bug 2) the format of the file name is `bug_seed_commit`. Correctness and liveness bugs can be deduped immediately by checking for their file name on disk. Crash bugs (bug 3) do not include the seed in their file name but do have an additional field which is the hash of the stack trace of the issue (`bug_commit_stacktracehash`) since multiple seeds may all trip the same assertion, leading to the same stack trace. Therefore, crash bugs can only be deduped after the seed has been replayed and the logs have been generated.
-
-If no duplicate issue has been found then the hub will replay the seed in `Debug` mode and capture the logs. In order to do this it must first checkout the correct git commit. This step requires that the reported commit is available in the `tigerbeetle` repository.
-
-### Create an Issue
-
-Once the simulation has completed, the stack trace is extracted and parsed to remove the local directory structure and any local memory addresses. This ensures that it is not only deterministic across machines, but also anonymized. Now the stack trace can be hashed and used to dedupe any crash bugs that may have already been reported. While crash bugs include a hash of the stack trace in their filename, to deduplicate assertion crashes for the same call graph, we do not do this for correctness bugs, since these are always detected by the same set of panics in the simulator, at the same call site, but there may be different causes for them.
-
-A copy of the issue is written to disk and a GitHub issue is also automatically generated. The issue contains the bug type, seed, commit hash, parameters of the VOPR, stack trace (if there is one), and debug logs.
-
-If the VOPR Hub replays a seed and it passes unexpectedly then, to err on the safe side, a GitHub issue will still be created with a warning explaining that the seed passed.
-
-The VOPR Hub may only have up to 6 GitHub issues open at any time. These issues are identified by the `seed` label that is attached to them as they are created. If there are 6 open issues and the VOPR Hub finds an additional issue then it is simply logged locally on the machine.

package/src/tigerbeetle/src/vopr_hub/SETUP.md

@@ -1,199 +0,0 @@
-# VOPR Server Setup
-
-*The VOPR Server is a dedicated testing machine. It continually runs VOPR simulators (performing deterministic fuzz testing) as well as another program, called the VOPR Hub (that listens out for bug reports). When a VOPR finds a bug, it creates a report and sends it to the VOPR Hub. When a report comes in the hub has an additional, dedicated VOPR to use to rerun the test. The hub then automatically creates a GitHub issue with the collected debug logs and stack trace.*
-
-*To ensure the latest code is always being tested, the VOPRs restart after running only a few seeds. Before restarting, the scheduler is responsible for fetching the latest code and checking out the relevant commit for that VOPR. At least one VOPR will always run on main. Other VOPRs can be assigned to test pull requests with the `vopr` label. The latest pull requests are always favored if there are too many to test.*
-
-*Follow these instructions to set up the VOPR Hub, along with several full-time VOPRs.*
-
-## Server Setup
-
-Install Go:
-```bash
-sudo add-apt-repository ppa:longsleep/golang-backports
-sudo apt update
-sudo apt install golang-go
-```
-
-Add two users, namely `voprrunner` and `voprhub`. This creates *separation of concerns* between the two functions of the server.
-
-The `voprrunner` will continuously run the VOPR and send any bugs to the VOPR Hub.
-
-This will require setting passwords for the respective users:
-
-```bash
-sudo adduser voprhub
-sudo adduser voprrunner
-sudo usermod -aG sudo voprhub
-sudo usermod -aG sudo voprrunner
-```
-
-## Set Up the VOPR Hub Component
-
-Become the `voprhub` user:
-```bash
-su - voprhub
-```
-
-Clone `tigerbeetle`:
-```bash
-git clone https://github.com/tigerbeetledb/tigerbeetle.git
-```
-
-Install Zig:
-```bash
-cd ./tigerbeetle
-./scripts/install_zig.sh
-cd ../
-```
-
-Create a second `tigerbeetle` directory here inside the hub directory which will run the VOPR Hub. The initial `tigerbeetle` directory will be needed to replay any seeds that the hub receives.
-```bash
-mkdir hub
-cp -r tigerbeetle hub/tigerbeetle
-```
-
-Create a `systemd` service unit file for the hub:
-```bash
-sudo nano /etc/systemd/system/voprhub.service
-```
-The file should contain the following (including an actual IP address and developer token with access to public repositories):
-```
-[Unit]
-
-Description=Continously runs the VOPR Hub.
-
-[Service]
-
-User=voprhub
-WorkingDirectory=/home/voprhub/hub/tigerbeetle/src/vopr_hub
-Environment="REPOSITORY_URL=https://api.github.com/repos/tigerbeetledb/tigerbeetle"
-Environment="TIGERBEETLE_DIRECTORY=/home/voprhub/tigerbeetle"
-Environment="VOPR_HUB_ADDRESS=<address>"
-Environment="ISSUE_DIRECTORY=/home/voprhub"
-Environment="DEVELOPER_TOKEN=******"
-ExecStart=go run main.go
-Restart=on-success
-
-[Install]
-
-WantedBy=multi-user.target
-```
-
-Start the VOPR Hub service:
-```bash
-systemctl start voprhub.service
-# Check that we have liftoff.
-systemctl status voprhub.service
-# View logs e.g.
-journalctl -f -n 100 -u voprhub.service
-```
-
-Go back to the `root` user:
-```bash
-exit
-```
-
-## Set Up the VOPR Component
-
-Become the `voprrunner` user:
-```bash
-su - voprrunner
-```
-
-Create a script that will be used by the service to fetch the latest code and run the VOPR:
-```bash
-sudo nano vopr_runner.sh
-```
-
-The file should contain the following (including an actual IP address and developer token with access to public repositories):
-```
-#!/usr/bin/env bash
-set -e
-
-# Checkout the correct branch
-export TIGERBEETLE_DIRECTORY="/home/voprrunner/tigerbeetle"
-export REPOSITORY_URL="https://api.github.com/repos/tigerbeetledb/tigerbeetle"
-export DEVELOPER_TOKEN="******"
-export NUM_VOPRS="4"
-export CURRENT_VOPR=$1
-go run ./src/vopr_hub/scheduler/main.go
-
-# Run the VOPR a few times before we go back to the top:
-zig/zig run ./src/vopr.zig -- --send="<address>" --simulations=5
-```
-
-Ensure the script is executable:
-```bash
-sudo chmod +x vopr_runner.sh
-```
-
-Create four `tigerbeetle` directories here.
-
-Note that the number of directories corresponds to the number of service instances that will run.
-
-Ideally, this number should be increased/decreased to be two less than the number of CPU cores available. The simulators burn CPU, and so this allocation leaves a core available for the rest of the system, plus a core for the hub itself.
-```bash
-git clone https://github.com/tigerbeetledb/tigerbeetle.git
-# Install Zig:
-cd ./tigerbeetle
-./scripts/install_zig.sh
-cd ../
-# Copy this directory to get four tigerbeetle directories.
-cp -r tigerbeetle tigerbeetle0 # Repeat with incrementing values for the other instances. E.g. 1, 2,3.
-# Now we can remove the original directory.
-sudo rm -r tigerbeetle
-```
-
-Create a `systemd` service unit file.
-
-Naming the file `vopr@.service` means that it acts as a template that can reuse the same file to run different services that each target their own directories.
-```bash
-sudo nano /etc/systemd/system/vopr@.service
-```
-
-The file should contain the following:
-```
-[Unit]
-Description=Continously runs the VOPR.
-PartOf=vopr.target
-
-[Service]
-
-User=voprrunner
-WorkingDirectory=/home/voprrunner/tigerbeetle%i
-ExecStart=/home/voprrunner/vopr_runner.sh %i
-Restart=on-success
-
-[Install]
-
-WantedBy=multi-user.target
-```
-
-Create a target file to manage all instances, called `vopr.target`.
-
-Dependencies must be listed under `Wants` instead of `Requires` because requiring the services will cause them all to restart whenever one terminates.
-```bash
-sudo nano /etc/systemd/system/vopr.target
-```
-
-The file should contain the following:
-```
-[Unit]
-Description=Runs all VOPR services.
-Wants=vopr@0.service vopr@1.service vopr@2.service vopr@3.service
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Start all services:
-```bash
-systemctl start vopr.target
-# Check it's VOPR'izing:
-systemctl status vopr.target
-# Check that the individual VOPR'raptors have started up:
-systemctl status vopr@0.service
-# View logs e.g.
-journalctl -f -n 100 -u vopr@0.service
-```

package/src/tigerbeetle/src/vopr_hub/go.mod

@@ -1,3 +0,0 @@
-module github.com/coilhq/tigerbeetle
-
-go 1.17