zbench-js 0.1.0

package/dist/cli.js

@@ -0,0 +1,278 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import * as io from 'ioium/node';
+import { writeFile } from 'node:fs/promises';
+import { join, relative, resolve } from 'node:path';
+import { parseArgs, styleText } from 'node:util';
+import { resolveCapabilities } from './capabilities.js';
+import { compare } from './compare.js';
+import { findConfig, flagCombinations, flagLabel, loadSuite } from './config.js';
+import { runIsolated, runWorker } from './isolate.js';
+import { cleanRefs, prepareRefs, repoRoot } from './refs.js';
+import { reportComparison, reportEnvironment, reportList, reportRun } from './report.js';
+import { mapPool } from './pool.js';
+import { runTest } from './runner.js';
+const usage = `Usage: zbench [options] [filter...]
+
+Run the performance tests named by the config file. Filters match a test's name or path.
+
+Options:
+  -c, --config <path>     Config file. Discovered by default.
+  -n, --iterations <n>    Timed runs per configuration.
+  -w, --warmup <n>        Untimed runs before the timed ones.
+  -R, --ref <ref>         Benchmark a git reference. Repeatable; the first one is the baseline.
+                          Use "." for the working tree as it is.
+  -f, --flag <name=json>  Restrict a flag to one value. Repeatable.
+  -t, --threshold <pct>   Smallest change worth coloring. [1]
+  -T, --timeout <s>       Seconds a matrix may take before it is killed and reported N/A. [300]
+                          0 waits forever. A reference with a pathological regression can
+                          otherwise hold the whole comparison open.
+  -a, --all               Run every configuration, ignoring cpu/mem requirements.
+  -J, --jobs <n>          Matrices to time at once. [one per hardware thread, 1 with --no-isolate]
+                          Concurrent matrices contend for the machine, which widens the ± column;
+                          pass -J 1 for the quietest numbers.
+  -l, --list              List what would run, then exit.
+  -j, --json <path>       Write the raw results as JSON.
+      --cpu <n>           Override the detected CPU level.
+      --mem <n>           Override the detected memory level.
+      --no-isolate        Run in this process instead of one child per matrix.
+      --build <cmd>       Command that makes a reference's worktree runnable.
+      --rebuild           Rebuild reference worktrees even when they are up to date.
+      --clean             Remove cached reference worktrees, then exit.
+  -q, --quiet             Only print results.
+  -h, --help              Show this message.
+`;
+const { values: opts, positionals: filters } = parseArgs({
+    options: {
+        config: { short: 'c', type: 'string' },
+        iterations: { short: 'n', type: 'string' },
+        warmup: { short: 'w', type: 'string' },
+        ref: { short: 'R', type: 'string', multiple: true, default: [] },
+        flag: { short: 'f', type: 'string', multiple: true, default: [] },
+        threshold: { short: 't', type: 'string' },
+        timeout: { short: 'T', type: 'string' },
+        all: { short: 'a', type: 'boolean' },
+        jobs: { short: 'J', type: 'string' },
+        list: { short: 'l', type: 'boolean' },
+        json: { short: 'j', type: 'string' },
+        cpu: { type: 'string' },
+        mem: { type: 'string' },
+        isolate: { type: 'boolean', default: true },
+        build: { type: 'string' },
+        rebuild: { type: 'boolean' },
+        clean: { type: 'boolean' },
+        quiet: { short: 'q', type: 'boolean' },
+        help: { short: 'h', type: 'boolean' },
+        worker: { type: 'string' },
+    },
+    allowPositionals: true,
+});
+if (opts.help) {
+    console.log(usage);
+    process.exit(0);
+}
+if (opts.worker) {
+    await runWorker(opts.worker);
+    process.exit(0);
+}
+function fail(message) {
+    io.error(message);
+    process.exit(1);
+}
+function number(value, name) {
+    if (value === undefined)
+        return undefined;
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed))
+        fail(`--${name} expects a number, got ${JSON.stringify(value)}`);
+    return parsed;
+}
+/** Parse `-f name=json`, falling back to the raw string when it isn't valid JSON. */
+function parseFlagOverrides(entries) {
+    const overrides = {};
+    for (const entry of entries) {
+        const at = entry.indexOf('=');
+        if (at < 0)
+            fail(`--flag expects name=value, got ${JSON.stringify(entry)}`);
+        const name = entry.slice(0, at);
+        const raw = entry.slice(at + 1);
+        let value;
+        try {
+            value = JSON.parse(raw);
+        }
+        catch {
+            value = raw;
+        }
+        (overrides[name] ??= []).push(value);
+    }
+    return overrides;
+}
+const configPath = opts.config ? resolve(opts.config) : findConfig();
+if (!configPath)
+    fail('no config file found; pass --config or add one at tests/perf/config.json');
+let suite;
+try {
+    suite = loadSuite(configPath);
+}
+catch (e) {
+    fail(e.message);
+}
+if (filters.length) {
+    const matches = (test) => filters.some(f => test.name.toLowerCase().includes(f.toLowerCase()) || test.path.includes(f));
+    suite.tests = suite.tests.filter(matches);
+    if (!suite.tests.length)
+        fail(`no test matches ${filters.map(f => JSON.stringify(f)).join(', ')}`);
+}
+if (opts.list) {
+    reportList(suite);
+    process.exit(0);
+}
+const cache = join(await repoRoot().catch(() => process.cwd()), '.zbench');
+if (opts.clean) {
+    await cleanRefs(cache);
+    io.info('removed cached worktrees');
+    process.exit(0);
+}
+const flagOverrides = parseFlagOverrides(opts.flag);
+const { cpu, mem } = resolveCapabilities({ cpu: number(opts.cpu, 'cpu'), mem: number(opts.mem, 'mem') });
+const threshold = number(opts.threshold, 'threshold') ?? 1;
+const timeout = number(opts.timeout, 'timeout') ?? 300;
+const iterations = number(opts.iterations, 'iterations');
+const warmup = number(opts.warmup, 'warmup');
+const cli = resolve(import.meta.dirname, 'cli.js');
+const refNames = opts.ref.length ? opts.ref : ['.'];
+// Concurrent work in one process would interleave the tests' own awaits into each other's timings
+const jobs = Math.max(1, number(opts.jobs, 'jobs') ?? (opts.isolate ? Math.round((navigator.hardwareConcurrency || 0) / 2) || 1 : 1));
+if (jobs > 1 && !opts.isolate)
+    fail('--jobs above 1 needs process isolation, so it cannot be used with --no-isolate');
+/** Overwrite the progress line, if there is one to overwrite. */
+const progress = {
+    active: false,
+    show(message) {
+        if (opts.quiet || !process.stdout.isTTY)
+            return;
+        this.clear();
+        process.stdout.write(styleText('gray', message));
+        this.active = true;
+    },
+    clear() {
+        if (!this.active)
+            return;
+        process.stdout.clearLine(0);
+        process.stdout.cursorTo(0);
+        this.active = false;
+    },
+};
+/** Every (test, flag combination) pair that will run, i.e. one table each. */
+function* planned(suite) {
+    for (const test of suite.tests) {
+        const narrowed = Object.fromEntries(Object.entries(test.flags).map(([name, values]) => [
+            name,
+            flagOverrides[name] ? values.filter(v => flagOverrides[name].some(w => Object.is(w, v))) : values,
+        ]));
+        for (const flags of flagCombinations(narrowed))
+            yield [test, flags];
+    }
+}
+async function runRef(ref, repo) {
+    const config = ref.sha ? join(ref.dir, relative(repo, suite.file)) : suite.file;
+    const matrices = [...planned(suite)];
+    let done = 0;
+    const results = await mapPool(matrices, jobs, async ([test, flags]) => {
+        const label = [test.name, flagLabel(flags)].filter(Boolean).join(' ');
+        progress.show(`[${done + 1}/${matrices.length}] ${label}${ref.sha ? ` @ ${ref.name}` : ''}...`);
+        try {
+            return opts.isolate
+                ? await runIsolated({ config, test: test.id, flags, iterations, warmup, cpu, mem, all: opts.all }, ref.dir, cli, timeout)
+                : await runTest(test, {
+                    iterations,
+                    warmup,
+                    cpu,
+                    mem,
+                    all: opts.all,
+                    flags: Object.fromEntries(Object.entries(flags).map(([k, v]) => [k, [v]])),
+                });
+        }
+        catch (e) {
+            // A matrix that cannot run is one reference's problem, not the whole comparison's:
+            // report it as N/A and let the references that did run still be compared
+            return test.configurations.map((configuration) => ({
+                test: test.id,
+                flags,
+                configuration: configuration.name,
+                value: configuration.value,
+                setup: 0,
+                samples: [],
+                amounts: {},
+                error: String(e?.message ?? e),
+            }));
+        }
+        finally {
+            done++;
+        }
+    });
+    progress.clear();
+    return results.flat();
+}
+if (!opts.quiet)
+    reportEnvironment(cpu, mem, iterations ?? 5, warmup ?? 1, jobs);
+const repo = await repoRoot().catch(() => process.cwd());
+let prepared;
+try {
+    prepared = await prepareRefs(refNames, {
+        cache,
+        build: opts.build ?? suite.build,
+        root: suite.root,
+        rebuild: opts.rebuild,
+        onStep: message => progress.show(message + '...'),
+    });
+}
+catch (e) {
+    progress.clear();
+    fail(`could not prepare references: ${e.stderr || e.message}`);
+}
+progress.clear();
+const refs = [];
+for (const ref of prepared) {
+    if (prepared.length > 1 && !opts.quiet) {
+        console.log();
+        console.log(styleText(['bold', 'underline'], ref.name) + (ref.sha ? styleText('gray', '  ' + ref.sha.slice(0, 8)) : ''));
+    }
+    const cases = await runRef(ref, repo);
+    reportRun(suite, cases);
+    refs.push({ ref: ref.name, cases });
+}
+if (refs.length > 1) {
+    console.log();
+    console.log(styleText(['bold', 'underline'], `Change vs ${refs[0].ref}`));
+    reportComparison(compare(suite, refs, threshold), refs);
+}
+if (opts.json) {
+    await writeFile(opts.json, JSON.stringify({
+        node: process.version,
+        platform: `${process.platform}/${process.arch}`,
+        capabilities: { cpu, mem },
+        refs,
+    }, null, '\t'));
+}
+const failed = refs.flatMap(r => r.cases).filter(c => c.error);
+if (failed.length) {
+    // One failing code path usually fails in every configuration, so report each distinct message once
+    const byMessage = new Map();
+    for (const c of failed) {
+        const message = c.error.split('\n')[0].trim();
+        const where = [c.test, flagLabel(c.flags), c.configuration].filter(Boolean).join(' · ');
+        if (!byMessage.has(message))
+            byMessage.set(message, []);
+        byMessage.get(message).push(where);
+    }
+    console.log();
+    for (const [message, where] of byMessage) {
+        io.error(`${message}  (${where.length} configuration${where.length == 1 ? '' : 's'})`);
+        for (const at of where.slice(0, 4))
+            console.error(styleText('gray', '    ' + at));
+        if (where.length > 4)
+            console.error(styleText('gray', `    ...and ${where.length - 4} more`));
+    }
+    process.exit(1);
+}

package/dist/compare.d.ts

@@ -0,0 +1,51 @@
+import { type Column } from './measure.js';
+import type { CaseResult } from './runner.js';
+import type { Suite, Test } from './config.js';
+/** Results for one of the states being compared. */
+export interface RefResults {
+    /** How the state was named on the command line, e.g. `main` or `v3.2.1` */
+    ref: string;
+    cases: CaseResult[];
+}
+/** One value alongside how noisy the run that produced it was. */
+export interface Cell {
+    value: number | null;
+    /** Relative standard deviation of the timings, as a percentage */
+    noise: number;
+    missing: boolean;
+}
+/** How one state's value relates to the baseline's. */
+export interface Delta {
+    /** Speedup: always `> 1` when better, whichever direction the column improves in */
+    factor: number | null;
+    /** Percentage change of the raw value, signed */
+    change: number | null;
+    /** Whether the change stands out from run-to-run variance */
+    significant: boolean;
+    better: boolean;
+}
+export interface ComparisonRow {
+    configuration: string;
+    /** Indexed the same as the states being compared, baseline first */
+    cells: Cell[];
+    deltas: Delta[];
+}
+export interface Comparison {
+    test: Test;
+    /** Flag combination, empty when the test has no flags */
+    label: string;
+    column: Column;
+    rows: ComparisonRow[];
+    /** Present for aggregate metrics, which have one value per state rather than one per row */
+    aggregate?: ComparisonRow;
+}
+/**
+ * Compare a cell against the baseline.
+ * A change is only called out when it is bigger than both `threshold` and the two runs' combined noise.
+ */
+export declare function delta(baseline: Cell, other: Cell, higherIsBetter: boolean, threshold: number): Delta;
+/**
+ * Line up every state's results against the baseline, one comparison per matrix column.
+ * `ops/s` is left out because it is just the reciprocal of `avg`.
+ */
+export declare function compare(suite: Suite, refs: RefResults[], threshold: number): Comparison[];

package/dist/compare.js

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { columns, matrices, timing } from './measure.js';
+import { combinedNoise } from './stats.js';
+function key(result) {
+    return `${result.test}\0${JSON.stringify(result.flags)}\0${result.configuration}`;
+}
+function cell(column, result) {
+    if (!result || result.skipped || result.error)
+        return { value: null, noise: 0, missing: true };
+    return { value: column.value(result), noise: timing(result).rsd, missing: false };
+}
+/**
+ * Compare a cell against the baseline.
+ * A change is only called out when it is bigger than both `threshold` and the two runs' combined noise.
+ */
+export function delta(baseline, other, higherIsBetter, threshold) {
+    if (baseline.value === null || other.value === null || !baseline.value)
+        return { factor: null, change: null, significant: false, better: false };
+    const factor = higherIsBetter ? other.value / baseline.value : baseline.value / other.value;
+    const change = ((other.value - baseline.value) / baseline.value) * 100;
+    const noise = combinedNoise(baseline.noise, other.noise);
+    return {
+        factor,
+        change,
+        significant: Math.abs(change) > Math.max(threshold, noise),
+        better: factor > 1,
+    };
+}
+/**
+ * Line up every state's results against the baseline, one comparison per matrix column.
+ * `ops/s` is left out because it is just the reciprocal of `avg`.
+ */
+export function compare(suite, refs, threshold) {
+    const [baseline, ...others] = refs;
+    if (!baseline || !others.length)
+        return [];
+    const indexes = refs.map(ref => new Map(ref.cases.map(result => [key(result), result])));
+    const comparisons = [];
+    for (const test of suite.tests) {
+        for (const matrix of matrices(test, baseline.cases)) {
+            for (const column of columns(matrix).filter(c => c.label != 'ops/s')) {
+                const rows = matrix.cases.map(result => {
+                    const cells = indexes.map(index => cell(column, index.get(key(result))));
+                    return {
+                        configuration: result.configuration,
+                        cells,
+                        deltas: cells.map(c => delta(cells[0], c, column.higherIsBetter, threshold)),
+                    };
+                });
+                comparisons.push({
+                    test,
+                    label: matrix.label,
+                    column,
+                    rows: column.aggregate ? [] : rows,
+                    aggregate: column.aggregate ? aggregateRow(matrix, column, refs, threshold) : undefined,
+                });
+            }
+        }
+    }
+    return comparisons;
+}
+/** The matrix-wide value for each state, compared the same way a row is. */
+function aggregateRow(matrix, column, refs, threshold) {
+    const flags = JSON.stringify(matrix.flags);
+    const cells = refs.map((ref) => {
+        const cases = ref.cases.filter(r => r.test == matrix.test.id && JSON.stringify(r.flags) == flags);
+        const total = column.total?.(cases) ?? null;
+        const noise = cases.filter(c => c.samples.length).map(c => timing(c).rsd);
+        return {
+            value: total,
+            noise: noise.length ? noise.reduce((sum, v) => sum + v, 0) / noise.length : 0,
+            missing: total === null,
+        };
+    });
+    return {
+        configuration: 'aggregate',
+        cells,
+        deltas: cells.map(c => delta(cells[0], c, column.higherIsBetter, threshold)),
+    };
+}

package/dist/config.d.ts

@@ -0,0 +1,110 @@
+import { type Timespan } from './units.js';
+/** A measurement is either derived at an inferred timespan (`true`) or at an explicit one. */
+export type MeasureValue = boolean | Timespan;
+/**
+ * How to turn one of a test's quantities into a reported measurement.
+ * The quantity's amount per iteration comes from what `test` returns, falling back to the configuration value.
+ */
+export interface MeasureSpec {
+    /** `<unit>`s per `<timespan>` for each configuration, e.g. `MB/s` */
+    throughput?: MeasureValue;
+    /** `<unit>`s per `<timespan>` across every configuration in the matrix */
+    aggregate_throughput?: MeasureValue;
+    /** `<timespan>` per `<unit>` for each configuration, e.g. `ms/entry` */
+    cost?: MeasureValue;
+    /** `<timespan>` per `<unit>` across every configuration in the matrix */
+    aggregate_cost?: MeasureValue;
+    /** What the amount is counted in. Defaults to the quantity's key. */
+    unit?: string;
+    /**
+     * Multiplier from the raw amount to `unit`.
+     * Defaults to 1, except for byte units (`MB`, `MiB`, ...), where amounts are taken to be bytes.
+     */
+    scale?: number;
+}
+/** One point in a test's matrix. */
+export interface ConfigurationSpec {
+    /** Defaults to the configuration's values, e.g. `size=128, entries=1000` */
+    name?: string;
+    /**
+     * Minimum CPU level needed to run this configuration.
+     * Levels are logarithmic: `n + 1` costs about twice as much as `n`.
+     */
+    cpu?: number;
+    /** Minimum memory level needed to run this configuration. Also logarithmic. */
+    mem?: number;
+    /** Passed to the test, and the source of measurement amounts. */
+    value: Record<string, number>;
+}
+export interface TestSpec {
+    /** Resolved relative to the config file */
+    path: string;
+    /** Defaults to `path` */
+    name?: string;
+    measure?: Record<string, MeasureSpec>;
+    configurations: ConfigurationSpec[];
+    /** Flags from the suite's pool to apply, or flags declared inline. The matrix runs once per combination. */
+    flags?: string[] | Record<string, unknown[]>;
+    iterations?: number;
+    warmup?: number;
+}
+export interface SuiteSpec {
+    /** Timed runs per configuration. @default 5 */
+    iterations?: number;
+    /** Untimed runs before the timed ones. @default 1 */
+    warmup?: number;
+    /** Flag pool that tests can draw from by name. */
+    flags?: Record<string, unknown[]>;
+    /** Shell command that makes a git reference's worktree runnable. */
+    build?: string;
+    /** Directory copied into a reference's worktree so every reference runs today's tests. Defaults to the config file's directory. */
+    root?: string;
+    tests: TestSpec[];
+}
+/** Config files are looked for at these paths, in order, relative to the working directory. */
+export declare const configPaths: string[];
+export declare const defaultBuild = "npm install --no-audit --no-fund && npm run build";
+/** A single reported column, derived from one `MeasureSpec` entry. */
+export interface Metric {
+    /** The quantity this is derived from */
+    key: string;
+    kind: 'throughput' | 'cost';
+    /** Whether this collapses the whole matrix into one number */
+    aggregate: boolean;
+    /** `null` means infer per matrix */
+    span: Timespan | null;
+    scale: number;
+    unit: string;
+    higherIsBetter: boolean;
+}
+/** A test with its defaults filled in and its measurements flattened into metrics. */
+export interface Test {
+    id: string;
+    name: string;
+    /** Absolute path to the module */
+    file: string;
+    /** Path as written, for display */
+    path: string;
+    metrics: Metric[];
+    configurations: Required<Pick<ConfigurationSpec, 'name' | 'cpu' | 'mem' | 'value'>>[];
+    flags: Record<string, unknown[]>;
+    iterations: number;
+    warmup: number;
+}
+export interface Suite {
+    /** Absolute path to the config file */
+    file: string;
+    /** Absolute path to the directory copied into reference worktrees */
+    root: string;
+    build: string;
+    tests: Test[];
+}
+/** Find the config file, starting from `from` and walking up to the filesystem root. */
+export declare function findConfig(from?: string): string | null;
+/** Parse and validate an already-loaded suite. `file` is used to resolve test paths. */
+export declare function parseSuite(data: unknown, file: string): Suite;
+export declare function loadSuite(file: string): Suite;
+/** Every combination of the given flags, in declaration order. Always at least one (possibly empty) combination. */
+export declare function flagCombinations(flags: Record<string, unknown[]>): Record<string, unknown>[];
+/** A stable, human-readable label for a flag combination. Empty when there are no flags. */
+export declare function flagLabel(flags: Record<string, unknown>): string;