zbench-js 0.1.0

package/dist/report.js

@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import * as io from 'ioium/node';
+import { styleText } from 'node:util';
+import { columns, matrices, timing } from './measure.js';
+import { duration, sig } from './units.js';
+const dim = (text) => styleText('gray', text);
+const bold = (text) => styleText('bold', text);
+/** `io.table` declares an `indent` option but does not apply it, so the first column carries it. */
+const indent = '  ';
+/** Heading for one test, with its file path when that isn't already the name. */
+function heading(test) {
+    return bold(test.name) + (test.name == test.path ? '' : dim('  ' + test.path));
+}
+/**
+ * What a cell shows when there is no measurement.
+ * The reason is kept out of the cell so one failure cannot stretch every column in the table.
+ */
+function na(result) {
+    return result.error ? styleText('red', 'N/A') : dim('N/A');
+}
+/** Why some rows have no numbers, one line per distinct reason. */
+function notes(cases) {
+    const byReason = new Map();
+    for (const result of cases) {
+        const reason = result.skipped ?? result.error?.split('\n')[0].trim();
+        if (!reason)
+            continue;
+        if (!byReason.has(reason))
+            byReason.set(reason, []);
+        byReason.get(reason).push(result.configuration);
+    }
+    return [...byReason].map(([reason, where]) => `${indent}${dim('N/A: ' + where.join(', ') + ' — ' + reason)}`);
+}
+/** Print one matrix as a table, followed by its aggregate measurements. */
+export function reportMatrix(matrix) {
+    if (matrix.label)
+        console.log('  ' + dim(matrix.label));
+    const all = columns(matrix);
+    const perRow = all.filter(c => !c.aggregate);
+    const aggregates = all.filter(c => c.aggregate);
+    io.table([
+        { name: indent + 'configuration', text: result => indent + result.configuration, grow: 0 },
+        { name: 'setup', text: result => (result.setup ? duration(result.setup) : ''), padStart: true, grow: 0 },
+        {
+            name: 'total',
+            text: result => (result.samples.length ? duration(timing(result).total) : na(result)),
+            padStart: true,
+            grow: 0,
+        },
+        ...perRow.map(column => ({
+            name: column.label,
+            text: (result) => {
+                const value = column.value(result);
+                return value === null ? na(result) : column.format(value);
+            },
+            padStart: true,
+            grow: 0,
+        })),
+        {
+            name: '±',
+            text: (result) => (result.samples.length ? sig(timing(result).rsd, 2) + '%' : ''),
+            padStart: true,
+            grow: 0,
+        },
+    ], { formatHead: dim }, matrix.cases);
+    for (const note of notes(matrix.cases))
+        console.log(note);
+    if (!aggregates.length)
+        return;
+    const totals = aggregates
+        .map(column => {
+        const value = column.total?.(matrix.cases) ?? null;
+        return value === null ? null : `${column.format(value)} ${column.label}`;
+    })
+        .filter(text => text !== null);
+    if (totals.length)
+        console.log('  ' + dim('aggregate') + '  ' + totals.join(dim(', ')));
+}
+/** Print every test's tables for a single state. */
+export function reportRun(suite, cases) {
+    for (const test of suite.tests) {
+        const found = matrices(test, cases);
+        if (!found.length)
+            continue;
+        console.log();
+        console.log(heading(test));
+        for (const matrix of found)
+            reportMatrix(matrix);
+    }
+}
+/** `1.35x` when better, `0.74x` when worse, colored only when the change beats the noise. */
+function factorText(row, index) {
+    const { factor, significant, better } = row.deltas[index];
+    if (factor === null)
+        return '';
+    const text = sig(factor, 3) + 'x';
+    if (!significant)
+        return dim(text);
+    return styleText(better ? 'green' : 'red', text);
+}
+/** Print the delta tables that follow each state's own tables. */
+export function reportComparison(comparisons, refs) {
+    const names = refs.map(ref => ref.ref);
+    let heading = null;
+    for (const comparison of comparisons) {
+        const rows = comparison.aggregate ? [comparison.aggregate] : comparison.rows;
+        if (!rows.length)
+            continue;
+        const label = comparison.test.name + (comparison.label ? dim('  ' + comparison.label) : '');
+        if (label != heading) {
+            console.log();
+            console.log(bold((heading = label)));
+        }
+        console.log('  ' + dim(comparison.column.label) + (comparison.aggregate ? dim(' (aggregate)') : ''));
+        io.table([
+            { name: indent + 'configuration', text: row => indent + row.configuration, grow: 0 },
+            {
+                name: names[0],
+                text: row => row.cells[0].value === null ? dim('N/A') : comparison.column.format(row.cells[0].value),
+                padStart: true,
+                grow: 0,
+            },
+            ...names.slice(1).flatMap((name, i) => [
+                {
+                    name,
+                    text: (row) => {
+                        const cell = row.cells[i + 1];
+                        return cell.value === null ? dim('N/A') : comparison.column.format(cell.value);
+                    },
+                    padStart: true,
+                    grow: 0,
+                },
+                { name: '', text: (row) => factorText(row, i + 1), padStart: true, grow: 0 },
+            ]),
+        ], { formatHead: dim }, rows);
+    }
+}
+/** A one-line note about what the numbers were produced under. */
+export function reportEnvironment(cpu, mem, iterations, warmup, jobs) {
+    console.log(dim(`${process.version} on ${process.platform}/${process.arch}`
+        + `  cpu=${cpu} mem=${mem}  ${iterations} iterations, ${warmup} warmup`
+        + (jobs > 1 ? `, ${jobs} at a time` : '')));
+}
+/** List the tests and configurations that would run, without running them. */
+export function reportList(suite) {
+    for (const test of suite.tests) {
+        console.log();
+        console.log(heading(test));
+        for (const config of test.configurations) {
+            const needs = [config.cpu && `cpu ${config.cpu}`, config.mem && `mem ${config.mem}`].filter(Boolean);
+            console.log('  ' + config.name + (needs.length ? dim('  needs ' + needs.join(', ')) : ''));
+        }
+        for (const [name, values] of Object.entries(test.flags))
+            console.log('  ' + dim(`flag ${name}: ${values.map(v => JSON.stringify(v)).join(', ')}`));
+    }
+}

package/dist/runner.d.ts

@@ -0,0 +1,40 @@
+import { type Test } from './config.js';
+import type { Amounts, TestModule } from './types.js';
+/** The measurements taken for one test at one point in its matrix. */
+export interface CaseResult {
+    /** The test's id, i.e. its path as written in the config */
+    test: string;
+    flags: Record<string, unknown>;
+    configuration: string;
+    value: Record<string, number>;
+    /** Milliseconds `setup` took */
+    setup: number;
+    /** Milliseconds each timed iteration took */
+    samples: number[];
+    /** Raw amounts summed over every timed iteration, keyed by quantity */
+    amounts: Amounts;
+    /** Present when the configuration was not run */
+    skipped?: string;
+    /** Present when the test threw */
+    error?: string;
+}
+export interface RunOptions {
+    /** Overrides the test's own iteration count */
+    iterations?: number;
+    warmup?: number;
+    /** The machine's budget. Configurations asking for more are skipped. */
+    cpu?: number;
+    mem?: number;
+    /** Run every configuration, whatever the budget says */
+    all?: boolean;
+    /** Restricts a flag to a subset of its declared values */
+    flags?: Record<string, unknown[]>;
+    /** Called before each configuration so callers can show progress */
+    onCase?(test: Test, flags: Record<string, unknown>, configuration: string, index: number, total: number): void;
+}
+/** `import()` the test module and check that it is one. */
+export declare function loadTest(test: Test): Promise<TestModule>;
+/** Whether a configuration fits in the budget, and why not when it doesn't. */
+export declare function skipReason(configuration: Test['configurations'][number], options: RunOptions): string | undefined;
+/** Run every configuration of a test, once per flag combination. */
+export declare function runTest(test: Test, options?: RunOptions): Promise<CaseResult[]>;

package/dist/runner.js

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { performance } from 'node:perf_hooks';
+import { pathToFileURL } from 'node:url';
+import { flagCombinations } from './config.js';
+/** `import()` the test module and check that it is one. */
+export async function loadTest(test) {
+    const module = (await import(pathToFileURL(test.file).href));
+    if (typeof module.test != 'function')
+        throw new Error(`${test.path} does not export a \`test\` function`);
+    return module;
+}
+/**
+ * Time one configuration.
+ * `setup`/`teardown` run once, `before`/`after` run around each iteration, and only `test` is timed.
+ */
+async function runCase(module, test, flags, configuration, options) {
+    const config = { ...flags, ...configuration.value };
+    const iterations = options.iterations ?? test.iterations;
+    const warmup = options.warmup ?? test.warmup;
+    const result = {
+        test: test.id,
+        flags,
+        configuration: configuration.name,
+        value: configuration.value,
+        setup: 0,
+        samples: [],
+        amounts: {},
+    };
+    // By key, not by metric: a key with both a per-configuration and an aggregate metric is one quantity
+    const quantities = [...new Set(test.metrics.map(metric => metric.key))];
+    const setupStart = performance.now();
+    let state;
+    try {
+        state = await module.setup?.(config);
+    }
+    catch (e) {
+        result.error = `setup: ${e?.stack ?? e}`;
+        return result;
+    }
+    result.setup = performance.now() - setupStart;
+    try {
+        for (let i = 0; i < warmup + iterations; i++) {
+            const timed = i >= warmup;
+            await module.before?.(config, state);
+            globalThis.gc?.();
+            const start = performance.now();
+            const amounts = await module.test(config, state);
+            const elapsed = performance.now() - start;
+            await module.after?.(config, state);
+            if (!timed)
+                continue;
+            result.samples.push(elapsed);
+            for (const key of quantities) {
+                const amount = amounts?.[key] ?? configuration.value[key];
+                if (amount === undefined)
+                    continue;
+                result.amounts[key] = (result.amounts[key] ?? 0) + amount;
+            }
+        }
+    }
+    catch (e) {
+        result.error = String(e?.stack ?? e);
+    }
+    finally {
+        try {
+            await module.teardown?.(config, state);
+        }
+        catch (e) {
+            result.error ??= `teardown: ${e?.stack ?? e}`;
+        }
+    }
+    return result;
+}
+/** Whether a configuration fits in the budget, and why not when it doesn't. */
+export function skipReason(configuration, options) {
+    if (options.all)
+        return undefined;
+    if (options.cpu !== undefined && configuration.cpu > options.cpu)
+        return `needs cpu ${configuration.cpu}`;
+    if (options.mem !== undefined && configuration.mem > options.mem)
+        return `needs mem ${configuration.mem}`;
+    return undefined;
+}
+/** Narrow a test's flags to the values the caller asked for, dropping the rest. */
+function selectFlags(test, options) {
+    const flags = {};
+    for (const [name, values] of Object.entries(test.flags)) {
+        const wanted = options.flags?.[name];
+        flags[name] = wanted ? values.filter(v => wanted.some(w => Object.is(w, v))) : values;
+    }
+    return flagCombinations(flags);
+}
+/** Run every configuration of a test, once per flag combination. */
+export async function runTest(test, options = {}) {
+    const module = await loadTest(test);
+    const combinations = selectFlags(test, options);
+    const results = [];
+    const total = combinations.length * test.configurations.length;
+    let index = 0;
+    for (const flags of combinations) {
+        for (const configuration of test.configurations) {
+            options.onCase?.(test, flags, configuration.name, index++, total);
+            const skipped = skipReason(configuration, options);
+            if (skipped) {
+                results.push({
+                    test: test.id,
+                    flags,
+                    configuration: configuration.name,
+                    value: configuration.value,
+                    setup: 0,
+                    samples: [],
+                    amounts: {},
+                    skipped,
+                });
+                continue;
+            }
+            results.push(await runCase(module, test, flags, configuration, options));
+        }
+    }
+    return results;
+}

package/dist/stats.d.ts

@@ -0,0 +1,19 @@
+/** Summary of a set of timing samples, in whatever unit the samples were taken in. */
+export interface Stats {
+    n: number;
+    total: number;
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    /** Sample standard deviation */
+    stddev: number;
+    /** Standard deviation as a percentage of the mean. This is the noise floor for comparisons. */
+    rsd: number;
+}
+export declare function stats(samples: number[]): Stats;
+/**
+ * The combined noise of two independent measurements, as a percentage.
+ * A change smaller than this is indistinguishable from run-to-run variance.
+ */
+export declare function combinedNoise(a: number, b: number): number;

package/dist/stats.js

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+export function stats(samples) {
+    const n = samples.length;
+    if (!n)
+        return { n, total: 0, mean: 0, median: 0, min: 0, max: 0, stddev: 0, rsd: 0 };
+    const total = samples.reduce((sum, v) => sum + v, 0);
+    const mean = total / n;
+    const sorted = [...samples].sort((a, b) => a - b);
+    const mid = n >> 1;
+    // Bessel's correction: a single sample has no spread to estimate
+    const variance = n < 2 ? 0 : samples.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1);
+    const stddev = Math.sqrt(variance);
+    return {
+        n,
+        total,
+        mean,
+        median: n % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2,
+        min: sorted[0],
+        max: sorted[n - 1],
+        stddev,
+        rsd: mean ? (stddev / mean) * 100 : 0,
+    };
+}
+/**
+ * The combined noise of two independent measurements, as a percentage.
+ * A change smaller than this is indistinguishable from run-to-run variance.
+ */
+export function combinedNoise(a, b) {
+    return Math.sqrt(a ** 2 + b ** 2);
+}

package/dist/types.d.ts

@@ -0,0 +1,25 @@
+/**
+ * How much work one iteration did, keyed by quantity.
+ * Returning this from `test` overrides the amounts taken from the configuration,
+ * which matters whenever the real amount is not known until the test runs.
+ */
+export type Amounts = Record<string, number>;
+/**
+ * What a test file exports.
+ * Only `test` is required; everything else fills in the gaps around it.
+ *
+ * @typeParam C The configuration, i.e. one entry's `value` merged with the active flags
+ * @typeParam S Whatever `setup` hands to the rest of the lifecycle
+ */
+export interface TestModule<C = any, S = any> {
+    /** Run once per configuration, before any iteration. Its time is reported separately and never counted as the test's. */
+    setup?(config: C, ...args: never[]): S | Promise<S>;
+    /** Run before each iteration, untimed. Use it to restore state the test consumes. */
+    before?(config: C, state: S): unknown;
+    /** The timed operation. Return {@link Amounts} to report what it actually processed. */
+    test(config: C, state: S): Amounts | void | Promise<Amounts | void>;
+    /** Run after each iteration, untimed. */
+    after?(config: C, state: S): unknown;
+    /** Run once per configuration, after every iteration. */
+    teardown?(config: C, state: S): unknown;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+export {};

package/dist/units.d.ts

@@ -0,0 +1,44 @@
+/** A unit of time a measurement can be expressed in. */
+export type Timespan = 'ns' | 'us' | 'ms' | 's';
+/** Milliseconds in each timespan, since every duration is measured in milliseconds. */
+export declare const timespans: {
+    readonly ns: 0.000001;
+    readonly us: 0.001;
+    readonly ms: 1;
+    readonly s: 1000;
+};
+/** Timespans from shortest to longest. */
+export declare const timespanOrder: readonly ["ns", "us", "ms", "s"];
+export declare function isTimespan(value: unknown): value is Timespan;
+/**
+ * Units whose amounts are counted in bytes, so `unit: "MB"` on a byte count reports MB/s without a `scale`.
+ * Both decimal and binary prefixes are recognized; an explicit `scale` overrides this.
+ */
+export declare const byteUnits: {
+    readonly B: 1;
+    readonly kB: 1000;
+    readonly KB: 1000;
+    readonly MB: 1000000;
+    readonly GB: 1000000000;
+    readonly TB: 1000000000000;
+    readonly KiB: number;
+    readonly MiB: number;
+    readonly GiB: number;
+    readonly TiB: number;
+};
+/** The multiplier that converts a raw amount into `unit`. */
+export declare function unitScale(unit: string): number;
+/**
+ * Round to `digits` significant figures, then render without trailing zeroes.
+ * Exponential notation is kept for values too small to read otherwise.
+ */
+export declare function sig(value: number, digits?: number): string;
+/** Render a duration in milliseconds using whichever timespan keeps it readable. */
+export declare function duration(ms: number): string;
+/**
+ * Pick the timespan that makes `amount` per timespan readable.
+ * Throughput grows with the timespan, so the search runs from longest to shortest.
+ */
+export declare function inferThroughputSpan(amount: number, ms: number): Timespan;
+/** Pick the timespan that makes the time per unit readable. Cost shrinks with the timespan. */
+export declare function inferCostSpan(amount: number, ms: number): Timespan;

package/dist/units.js

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+/** Milliseconds in each timespan, since every duration is measured in milliseconds. */
+export const timespans = { ns: 1e-6, us: 1e-3, ms: 1, s: 1e3 };
+/** Timespans from shortest to longest. */
+export const timespanOrder = ['ns', 'us', 'ms', 's'];
+export function isTimespan(value) {
+    return typeof value == 'string' && value in timespans;
+}
+/**
+ * Units whose amounts are counted in bytes, so `unit: "MB"` on a byte count reports MB/s without a `scale`.
+ * Both decimal and binary prefixes are recognized; an explicit `scale` overrides this.
+ */
+export const byteUnits = {
+    B: 1,
+    kB: 1e3,
+    KB: 1e3,
+    MB: 1e6,
+    GB: 1e9,
+    TB: 1e12,
+    KiB: 2 ** 10,
+    MiB: 2 ** 20,
+    GiB: 2 ** 30,
+    TiB: 2 ** 40,
+};
+/** The multiplier that converts a raw amount into `unit`. */
+export function unitScale(unit) {
+    return unit in byteUnits ? 1 / byteUnits[unit] : 1;
+}
+/**
+ * Round to `digits` significant figures, then render without trailing zeroes.
+ * Exponential notation is kept for values too small to read otherwise.
+ */
+export function sig(value, digits = 4) {
+    if (!Number.isFinite(value))
+        return '-';
+    if (value == 0)
+        return '0';
+    return Number(value.toPrecision(digits)).toString();
+}
+/** Render a duration in milliseconds using whichever timespan keeps it readable. */
+export function duration(ms) {
+    if (!Number.isFinite(ms))
+        return '-';
+    if (ms == 0)
+        return '0';
+    const span = pick(timespanOrder, s => ms / timespans[s]);
+    return sig(ms / timespans[span]) + span;
+}
+/**
+ * Choose the first candidate whose value lands in `[1, 1000)`.
+ * When none does, the closest one is used, so the result is always readable rather than empty.
+ */
+function pick(candidates, value) {
+    let best = candidates[0];
+    let distance = Infinity;
+    for (const candidate of candidates) {
+        const v = Math.abs(value(candidate));
+        if (v >= 1 && v < 1000)
+            return candidate;
+        const from1 = v > 0 && Number.isFinite(v) ? Math.abs(Math.log10(v)) : Infinity;
+        if (from1 < distance) {
+            distance = from1;
+            best = candidate;
+        }
+    }
+    return best;
+}
+/**
+ * Pick the timespan that makes `amount` per timespan readable.
+ * Throughput grows with the timespan, so the search runs from longest to shortest.
+ */
+export function inferThroughputSpan(amount, ms) {
+    return pick([...timespanOrder].reverse(), span => (amount * timespans[span]) / ms);
+}
+/** Pick the timespan that makes the time per unit readable. Cost shrinks with the timespan. */
+export function inferCostSpan(amount, ms) {
+    return pick(timespanOrder, span => ms / timespans[span] / amount);
+}

package/package.json

@@ -0,0 +1,76 @@
+{
+	"name": "zbench-js",
+	"version": "0.1.0",
+	"description": "Data-driven, reproducible performance testing",
+	"funding": {
+		"type": "individual",
+		"url": "https://github.com/sponsors/james-pre"
+	},
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"type": "module",
+	"keywords": [
+		"benchmark",
+		"performance",
+		"testing"
+	],
+	"exports": {
+		".": "./dist/index.js",
+		"./*": "./dist/*.js"
+	},
+	"bin": {
+		"zbench": "dist/cli.js"
+	},
+	"files": [
+		"dist",
+		"LICENSE.md"
+	],
+	"publishConfig": {
+		"access": "public",
+		"provenance": true
+	},
+	"scripts": {
+		"format:check": "prettier --check .",
+		"format": "prettier --write .",
+		"lint": "tsc --noEmit && eslint src",
+		"build": "tsc -b",
+		"test": "node --test test/**/*.test.ts",
+		"prepublishOnly": "npx tsc"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/james-pre/zbench.git"
+	},
+	"author": "James Prevett <jp@jamespre.dev> (https://jamespre.dev)",
+	"license": "LGPL-3.0-or-later",
+	"bugs": {
+		"url": "https://github.com/james-pre/zbench/issues"
+	},
+	"homepage": "https://github.com/james-pre/zbench#readme",
+	"devDependencies": {
+		"@eslint/js": "^10.0.1",
+		"@types/node": "^26.0.0",
+		"eslint": "^10.1.0",
+		"globals": "^17.8.0",
+		"prettier": "^3.2.5",
+		"typedoc": "^0.28.18",
+		"typescript": "^6.0.0",
+		"typescript-eslint": "^8.58.0"
+	},
+	"dependencies": {
+		"ioium": "^1.8.0",
+		"utilium": "^3.0.0"
+	},
+	"engines": {
+		"node": ">=22.18.0"
+	},
+	"prettier": {
+		"singleQuote": true,
+		"useTabs": true,
+		"trailingComma": "es5",
+		"tabWidth": 4,
+		"printWidth": 120,
+		"arrowParens": "avoid",
+		"experimentalOperatorPosition": "start"
+	}
+}