zbench-js 0.1.0

package/LICENSE.md

@@ -0,0 +1,157 @@
+# GNU LESSER GENERAL PUBLIC LICENSE
+
+Version 3, 29 June 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+This version of the GNU Lesser General Public License incorporates the
+terms and conditions of version 3 of the GNU General Public License,
+supplemented by the additional permissions listed below.
+
+## 0. Additional Definitions.
+
+As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the
+GNU General Public License.
+
+"The Library" refers to a covered work governed by this License, other
+than an Application or a Combined Work as defined below.
+
+An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+## 1. Exception to Section 3 of the GNU GPL.
+
+You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+## 2. Conveying Modified Versions.
+
+If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+-   a) under this License, provided that you make a good faith effort
+    to ensure that, in the event an Application does not supply the
+    function or data, the facility still operates, and performs
+    whatever part of its purpose remains meaningful, or
+-   b) under the GNU GPL, with none of the additional permissions of
+    this License applicable to that copy.
+
+## 3. Object Code Incorporating Material from Library Header Files.
+
+The object code form of an Application may incorporate material from a
+header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+-   a) Give prominent notice with each copy of the object code that
+    the Library is used in it and that the Library and its use are
+    covered by this License.
+-   b) Accompany the object code with a copy of the GNU GPL and this
+    license document.
+
+## 4. Combined Works.
+
+You may convey a Combined Work under terms of your choice that, taken
+together, effectively do not restrict modification of the portions of
+the Library contained in the Combined Work and reverse engineering for
+debugging such modifications, if you also do each of the following:
+
+-   a) Give prominent notice with each copy of the Combined Work that
+    the Library is used in it and that the Library and its use are
+    covered by this License.
+-   b) Accompany the Combined Work with a copy of the GNU GPL and this
+    license document.
+-   c) For a Combined Work that displays copyright notices during
+    execution, include the copyright notice for the Library among
+    these notices, as well as a reference directing the user to the
+    copies of the GNU GPL and this license document.
+-   d) Do one of the following:
+    -   0) Convey the Minimal Corresponding Source under the terms of
+        this License, and the Corresponding Application Code in a form
+        suitable for, and under terms that permit, the user to
+        recombine or relink the Application with a modified version of
+        the Linked Version to produce a modified Combined Work, in the
+        manner specified by section 6 of the GNU GPL for conveying
+        Corresponding Source.
+    -   1) Use a suitable shared library mechanism for linking with
+        the Library. A suitable mechanism is one that (a) uses at run
+        time a copy of the Library already present on the user's
+        computer system, and (b) will operate properly with a modified
+        version of the Library that is interface-compatible with the
+        Linked Version.
+-   e) Provide Installation Information, but only if you would
+    otherwise be required to provide such information under section 6
+    of the GNU GPL, and only to the extent that such information is
+    necessary to install and execute a modified version of the
+    Combined Work produced by recombining or relinking the Application
+    with a modified version of the Linked Version. (If you use option
+    4d0, the Installation Information must accompany the Minimal
+    Corresponding Source and Corresponding Application Code. If you
+    use option 4d1, you must provide the Installation Information in
+    the manner specified by section 6 of the GNU GPL for conveying
+    Corresponding Source.)
+
+## 5. Combined Libraries.
+
+You may place library facilities that are a work based on the Library
+side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+-   a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities, conveyed under the terms of this License.
+-   b) Give prominent notice with the combined library that part of it
+    is a work based on the Library, and explaining where to find the
+    accompanying uncombined form of the same work.
+
+## 6. Revised Versions of the GNU Lesser General Public License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Library
+as you received it specifies that a certain numbered version of the
+GNU Lesser General Public License "or any later version" applies to
+it, you have the option of following the terms and conditions either
+of that published version or of any later version published by the
+Free Software Foundation. If the Library as you received it does not
+specify a version number of the GNU Lesser General Public License, you
+may choose any version of the GNU Lesser General Public License ever
+published by the Free Software Foundation.
+
+If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

package/README.md

@@ -0,0 +1,195 @@
+# zbench
+
+Reproducible, data-driven performance testing.
+
+`node:test` is the wrong tool for benchmarks: it reports pass/fail, not throughput, and it has no
+notion of a matrix, a baseline, or noise. zbench is the other tool. You describe the matrix in JSON,
+write a `test` function, and it handles iteration, measurement, machine gating, process isolation,
+and comparison across git references.
+
+```
+npm i -D zbench-js
+```
+
+## The shape of a suite
+
+A config file lists tests. Each test names a module and the configurations to run it under.
+
+`tests/perf/config.json`:
+
+```json
+{
+	"tests": [
+		{
+			"path": "unpack.ts",
+			"name": "Unpack a zip of random data",
+			"measure": {
+				"bytes": { "throughput": "s", "unit": "MB" },
+				"entries": { "cost": true, "aggregate_cost": true, "unit": "entry" }
+			},
+			"configurations": [
+				{ "name": "1000x 128KB", "value": { "size": 131072, "entries": 1000 } },
+				{ "value": { "size": 131072, "entries": 2000 } },
+				{ "cpu": 1, "value": { "size": 524288, "entries": 1000 } },
+				{ "mem": 1, "value": { "size": 131072, "entries": 16000 } }
+			]
+		}
+	]
+}
+```
+
+`tests/perf/unpack.ts`:
+
+```ts
+interface Config {
+	size: number;
+	entries: number;
+}
+
+export function setup(config: Config) {
+	return buildArchive(config.entries, config.size);
+}
+
+export async function test(config: Config, archive: Archive) {
+	const unpacked = await unpack(archive);
+	return { bytes: unpacked.byteLength, entries: unpacked.count };
+}
+```
+
+Then `npx zbench`.
+
+The config file is discovered at `tests/perf/config.json`, `tests/perf.json`,
+`tests/perf.config.json`, or `zbench.json`, searching upward from the working directory. Test paths
+resolve relative to it.
+
+## What a test module exports
+
+Only `test` is required.
+
+| Export     | When it runs           | Timed                           |
+| ---------- | ---------------------- | ------------------------------- |
+| `setup`    | once per configuration | separately, reported as `setup` |
+| `before`   | before every iteration | no                              |
+| `test`     | the operation          | **yes**                         |
+| `after`    | after every iteration  | no                              |
+| `teardown` | once per configuration | no                              |
+
+`setup` returns state, which every other hook receives as its second argument. `before` is where
+you restore anything the test consumes, so each iteration starts from the same place.
+
+`test` receives the configuration's `value` merged with the active flags. It may return an object of
+amounts — how much work it actually did — which overrides the amounts taken from the configuration.
+Return them whenever the real amount is not known until the test runs.
+
+Node strips types rather than resolving them, so a test that imports a sibling `.ts` file has to
+name the real extension (`./fixtures.ts`). Set `allowImportingTsExtensions` in the tsconfig that
+covers your tests.
+
+## Measurements
+
+Every test reports `setup`, `total`, `avg`, `ops/s`, and `±` (relative standard deviation — the
+noise floor) without asking. `measure` adds columns derived from the quantities the test works in.
+
+| Key                    | Reports                                     | Example    |
+| ---------------------- | ------------------------------------------- | ---------- |
+| `throughput`           | units per timespan, per configuration       | `MB/s`     |
+| `cost`                 | timespan per unit, per configuration        | `ms/entry` |
+| `aggregate_throughput` | units per timespan, across the whole matrix | `MB/s`     |
+| `aggregate_cost`       | timespan per unit, across the whole matrix  | `ms/entry` |
+
+Each takes `"ns"`, `"us"`, `"ms"`, `"s"`, or `true` to pick whichever timespan keeps the number
+readable. The timespan is settled once per matrix, so a column has one heading.
+
+`unit` names the quantity in the column heading and defaults to the key. Byte units (`B`, `KB`,
+`MB`, `GB`, `TB`, `KiB`, `MiB`, ...) also scale the amount, so a test that counts bytes and declares
+`"unit": "MB"` reports MB/s with no further arithmetic. `scale` sets the multiplier explicitly and
+overrides that.
+
+## Flags
+
+Flags are alternative code paths rather than points in the matrix: the whole matrix runs once per
+combination, and each combination gets its own table. Declare them once and let tests draw from the
+pool by name:
+
+```json
+{
+	"flags": { "lazy": [true, false] },
+	"tests": [{ "path": "mount.ts", "flags": ["lazy"], "configurations": [] }]
+}
+```
+
+A test can also declare its own inline: `"flags": { "lazy": [true, false] }`. Flag values are merged
+into the configuration the test receives. `-f lazy=true` restricts a run to one value.
+
+## Machine requirements
+
+A configuration can say what it needs:
+
+```json
+{ "cpu": 2, "mem": 1, "value": { "entries": 32000 } }
+```
+
+Both are logarithmic — level `n + 1` is about twice level `n`. zbench calibrates the machine at
+startup and reports rows it cannot afford as `N/A`, so the same config file runs everywhere without
+timing out a laptop. Override with `--cpu` / `--mem` (or `ZBENCH_CPU` / `ZBENCH_MEM`), or ignore the
+gating entirely with `-a`.
+
+## Comparing states
+
+```
+zbench -R v3.2.1 -R main -R HEAD
+```
+
+Each reference is checked out into a cached worktree under `.zbench/`, built, and run. The first is
+the baseline; every other column shows its value and a speedup factor that is `> 1` when better,
+whichever direction the column improves in. A change is only colored when it is larger than both
+`--threshold` (default 1%) and the two runs' combined noise — so a green number means the change
+outran the variance, and a gray one means it did not.
+
+Use `.` for the working tree as it is, without a checkout or a build.
+
+The tests themselves always come from the working tree and are copied into each worktree, so an old
+reference is benchmarked with today's tests. `--build` sets the command that makes a worktree
+runnable (default `npm install --no-audit --no-fund && npm run build`), `--rebuild` forces it, and
+`--clean` removes the cached worktrees.
+
+## Isolation and concurrency
+
+Each matrix runs in a fresh process by default, so one configuration cannot warm up or poison the
+next, and a reference's own build is what gets loaded. `--no-isolate` runs everything in-process for
+debugging or profiling.
+
+`-J` sets how many matrices are timed at once, defaulting to half the number of hardware threads. Concurrent
+matrices contend for the machine, which inflates absolute numbers and widens `±`; the comparison
+factors hold up because every reference runs under the same contention, but pass `-J 1` when the
+absolute numbers are the point.
+
+A matrix that crashes or outruns `--timeout` (default 300s) is reported as `N/A` rather than
+bringing down the run, so the references that did finish are still compared. This matters when the
+thing you are looking for is a quadratic: the configuration that takes 30 ms on the fix can take ten
+minutes on the commit before it.
+
+## Options
+
+```
+  -c, --config <path>     Config file. Discovered by default.
+  -n, --iterations <n>    Timed runs per configuration.
+  -w, --warmup <n>        Untimed runs before the timed ones.
+  -R, --ref <ref>         Benchmark a git reference. Repeatable; the first one is the baseline.
+  -f, --flag <name=json>  Restrict a flag to one value. Repeatable.
+  -t, --threshold <pct>   Smallest change worth coloring. [1]
+  -T, --timeout <s>       Seconds a matrix may take before it is killed and reported N/A. [300]
+  -a, --all               Run every configuration, ignoring cpu/mem requirements.
+  -J, --jobs <n>          Matrices to time at once.
+  -l, --list              List what would run, then exit.
+  -j, --json <path>       Write the raw results as JSON.
+      --cpu <n>           Override the detected CPU level.
+      --mem <n>           Override the detected memory level.
+      --no-isolate        Run in this process instead of one child per matrix.
+      --build <cmd>       Command that makes a reference's worktree runnable.
+      --rebuild           Rebuild reference worktrees even when they are up to date.
+      --clean             Remove cached reference worktrees, then exit.
+  -q, --quiet             Only print results.
+```
+
+Positional arguments filter tests by name or path.

package/dist/capabilities.d.ts

@@ -0,0 +1,15 @@
+/**
+ * What the current machine can be asked to run.
+ * A configuration is skipped when it asks for more than these.
+ */
+export interface Capabilities {
+    cpu: number;
+    mem: number;
+}
+/**
+ * Measure what this machine can handle. Levels are logarithmic, so `n + 1` is about twice `n`.
+ * The result is cached, since calibration costs real time.
+ */
+export declare function capabilities(): Capabilities;
+/** Resolve the budget from overrides, the environment, and finally calibration. */
+export declare function resolveCapabilities(overrides?: Partial<Capabilities>): Capabilities;

package/dist/capabilities.js

@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { totalmem } from 'node:os';
+import { performance } from 'node:perf_hooks';
+/** Milliseconds the calibration loop takes on the machine level 0 is defined against. */
+const cpuReference = 40;
+/** Memory a level 0 machine is assumed to have. */
+const memReference = 4 * 2 ** 30;
+function calibrate() {
+    const run = () => {
+        let x = 0;
+        for (let i = 0; i < 5e6; i++)
+            x = (x + Math.imul(i, 2654435761)) >>> 0;
+        return x;
+    };
+    run(); // let the JIT settle before the timed pass
+    const start = performance.now();
+    run();
+    return performance.now() - start;
+}
+let detected;
+/**
+ * Measure what this machine can handle. Levels are logarithmic, so `n + 1` is about twice `n`.
+ * The result is cached, since calibration costs real time.
+ */
+export function capabilities() {
+    detected ??= {
+        cpu: Math.max(0, Math.round(Math.log2(cpuReference / calibrate()))),
+        mem: Math.max(0, Math.floor(Math.log2(totalmem() / memReference))),
+    };
+    return detected;
+}
+/** Resolve the budget from overrides, the environment, and finally calibration. */
+export function resolveCapabilities(overrides = {}) {
+    const env = (name) => {
+        const value = process.env[name];
+        if (value === undefined)
+            return undefined;
+        const parsed = Number(value);
+        return Number.isFinite(parsed) ? parsed : undefined;
+    };
+    const cpu = overrides.cpu ?? env('ZBENCH_CPU');
+    const mem = overrides.mem ?? env('ZBENCH_MEM');
+    if (cpu !== undefined && mem !== undefined)
+        return { cpu, mem };
+    const auto = capabilities();
+    return { cpu: cpu ?? auto.cpu, mem: mem ?? auto.mem };
+}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};