zbench-js 0.1.1 → 0.2.0

package/dist/isolate.js

@@ -1,74 +1,43 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-import { spawn } from 'node:child_process';
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { loadSuite } from './config.js';
-import { runTest } from './runner.js';
-/** Run one matrix and write the results where the parent expects them. */
-export async function runWorker(specPath) {
-    const spec = JSON.parse(await readFile(specPath, 'utf-8'));
-    const suite = loadSuite(spec.config);
-    const test = suite.tests.find(t => t.id == spec.test);
-    if (!test)
-        throw new Error(`no test with id ${JSON.stringify(spec.test)} in ${spec.config}`);
-    const results = await runTest(test, {
-        iterations: spec.iterations,
-        warmup: spec.warmup,
-        cpu: spec.cpu,
-        mem: spec.mem,
-        all: spec.all,
-        flags: Object.fromEntries(Object.entries(spec.flags).map(([k, v]) => [k, [v]])),
-    });
-    await writeFile(spec.out, JSON.stringify(results));
-}
+import { Worker } from 'node:worker_threads';
+/** Resolved next to this module, so it works from `dist` and from the sources alike. */
+const entry = join(import.meta.dirname, import.meta.filename.endsWith('.ts') ? './worker.ts' : './worker.js');
 /**
- * Run one matrix in a fresh process.
- * Every matrix gets its own heap and its own module graph, which is what keeps one configuration
- * from warming up (or poisoning) the next, and what lets a git reference's own build be loaded.
+ * Run one matrix in a fresh worker thread.
  */
-export async function runIsolated(spec, cwd, cli, 
+export async function runIsolated(spec, 
 /** Seconds before the worker is killed. 0 waits forever. */
 timeout = 0) {
-    const dir = await mkdtemp(join(tmpdir(), 'zbench-'));
-    const specPath = join(dir, 'spec.json');
-    const out = join(dir, 'results.json');
+    const worker = new Worker(entry, { workerData: spec, stdout: true, stderr: true });
+    let output = '';
+    let timedOut = false;
+    const collect = (chunk) => (output += chunk.toString());
+    worker.stdout.on('data', collect);
+    worker.stderr.on('data', collect);
+    const timer = timeout
+        ? setTimeout(() => {
+            timedOut = true;
+            void worker.terminate();
+        }, timeout * 1000)
+        : undefined;
     try {
-        await writeFile(specPath, JSON.stringify({ ...spec, out }));
-        // Output is buffered rather than inherited: with several workers running it would interleave,
-        // and it is only worth reading when one of them fails
-        let output = '';
-        let timedOut = false;
-        const code = await new Promise((resolve, reject) => {
-            const extra = [];
-            if (!process.features.typescript)
-                extra.push('--disable-warning=ExperimentalWarning');
-            const child = spawn(process.execPath, [...process.execArgv, ...extra, cli, '--worker', specPath], {
-                cwd,
-                stdio: ['ignore', 'pipe', 'pipe'],
-            });
-            const collect = (chunk) => (output += chunk.toString());
-            child.stdout.on('data', collect);
-            child.stderr.on('data', collect);
-            const timer = timeout
-                ? setTimeout(() => {
-                    timedOut = true;
-                    child.kill('SIGKILL');
-                }, timeout * 1000)
-                : undefined;
-            child.on('error', reject);
-            child.on('close', c => {
-                clearTimeout(timer);
-                resolve(c ?? 1);
-            });
+        const { promise, resolve, reject } = Promise.withResolvers();
+        worker.on('message', (results) => {
+            resolve(results);
+            // The results are in hand, so nothing the test left running is worth waiting on
+            void worker.terminate();
+        });
+        worker.on('error', reject);
+        worker.on('exit', code => {
+            if (timedOut)
+                reject(new Error(`timed out after ${timeout}s`));
+            else
+                reject(new Error(`worker exited with code ${code}\n${output.trim()}`));
         });
-        if (timedOut)
-            throw new Error(`timed out after ${timeout}s`);
-        if (code != 0)
-            throw new Error(`worker exited with code ${code}\n${output.trim()}`);
-        return JSON.parse(await readFile(out, 'utf-8'));
+        return promise;
     }
     finally {
-        await rm(dir, { recursive: true, force: true });
+        clearTimeout(timer);
     }
 }

package/dist/measure.d.ts

@@ -1,5 +1,5 @@
 import { type Metric, type Test } from './config.js';
-import type { CaseResult } from './runner.js';
+import type { CaseResult } from './types.js';
 import { type Stats } from './stats.js';
 import { type Timespan } from './units.js';
 /** One test's results for one flag combination: the unit a table is printed for. */
@@ -51,6 +51,8 @@ export interface Column {
     total?(cases: CaseResult[]): number | null;
     /** Aggregate columns are reported once per matrix instead of once per row */
     aggregate: boolean;
+    /** Whether the metric behind this column asked to be the one comparisons report. Never set on the built-ins. */
+    compare: boolean;
 }
 /**
  * The columns a matrix reports: the built-in timings, then whatever the test declared.

package/dist/measure.js

@@ -88,6 +88,7 @@ export function columns(matrix) {
             format: duration,
             value: result => timing(result).mean || null,
             aggregate: false,
+            compare: false,
         },
         {
             label: 'ops/s',
@@ -95,6 +96,7 @@ export function columns(matrix) {
             format: sig,
             value: opsPerSecond,
             aggregate: false,
+            compare: false,
         },
     ];
     for (const metric of resolveMetrics(matrix)) {
@@ -105,6 +107,7 @@ export function columns(matrix) {
             value: result => caseValue(metric, result),
             total: cases => aggregateValue(metric, { ...matrix, cases }),
             aggregate: metric.aggregate,
+            compare: metric.compare,
         });
     }
     return list;

package/dist/refs.d.ts

@@ -18,18 +18,11 @@ export interface RefOptions {
     root: string;
     /** Build even when the worktree is already at the right commit */
     rebuild?: boolean;
+    /** Called as preparation moves from one step to the next, so a caller can report where it is. */
     onStep?(message: string): void;
 }
 /** The root of the repository containing `from`. */
 export declare function repoRoot(from?: string): Promise<string>;
-/**
- * Check out every reference and make it runnable.
- *
- * Checkout is serial because `git worktree add` takes a repository-wide lock, but building is not,
- * and building is where nearly all of the time goes. Nothing here is timed, so the concurrency
- * costs the measurements nothing.
- */
-export declare function prepareRefs(names: string[], options: RefOptions): Promise<Ref[]>;
 /** Check out one reference into a reusable worktree, build it, and stage the tests inside it. */
 export declare function prepareRef(name: string, options: RefOptions): Promise<Ref>;
 /** Remove every cached worktree. */

package/dist/refs.js

@@ -39,55 +39,48 @@ async function copyTests(repo, dir, root) {
     });
 }
 /**
- * Check out every reference and make it runnable.
- *
- * Checkout is serial because `git worktree add` takes a repository-wide lock, but building is not,
- * and building is where nearly all of the time goes. Nothing here is timed, so the concurrency
- * costs the measurements nothing.
+ * `git worktree add` takes a repository-wide lock, so two references being prepared at once would
+ * race over it. Checkouts queue up behind this; building, which is where nearly all of the time
+ * goes, does not. Nothing here is timed, so the overlap costs the measurements nothing.
  */
-export async function prepareRefs(names, options) {
+let checkouts = Promise.resolve();
+function serially(run) {
+    // Settled either way, so one reference failing to check out does not wedge the queue
+    const result = checkouts.then(run, run);
+    checkouts = result.catch(() => { });
+    return result;
+}
+/** Check out one reference into a reusable worktree, build it, and stage the tests inside it. */
+export async function prepareRef(name, options) {
     const repo = await repoRoot();
-    const staged = [];
-    for (const name of names) {
-        if (workingRefs.includes(name)) {
-            staged.push({ name, dir: repo, sha: null, build: false });
-            continue;
-        }
-        const sha = await git(repo, 'rev-parse', name + '^{commit}');
-        const dir = join(options.cache, slug(name));
-        const stamp = join(dir, '.zbench-ref');
-        const current = existsSync(stamp) ? (await readFile(stamp, 'utf-8')).trim() : null;
-        if (current == sha && !options.rebuild) {
-            staged.push({ name, dir, sha, build: false });
-            continue;
-        }
-        if (existsSync(dir)) {
-            await git(repo, 'worktree', 'remove', '--force', dir).catch(() => { });
-            await rm(dir, { recursive: true, force: true });
-        }
-        await mkdir(options.cache, { recursive: true });
-        options.onStep?.(`checking out ${name} (${sha.slice(0, 8)})`);
-        await git(repo, 'worktree', 'add', '--detach', dir, sha);
-        staged.push({ name, dir, sha, build: true });
+    // The working tree runs where it is, so there is nothing to check out, copy, or build
+    if (workingRefs.includes(name))
+        return { name, dir: repo, sha: null };
+    const sha = await git(repo, 'rev-parse', name + '^{commit}');
+    const dir = join(options.cache, slug(name));
+    const stamp = join(dir, '.zbench-ref');
+    const current = existsSync(stamp) ? (await readFile(stamp, 'utf-8')).trim() : null;
+    const build = current != sha || !!options.rebuild;
+    if (build) {
+        await serially(async () => {
+            options.onStep?.(`checking out ${sha.slice(0, 8)}`);
+            if (existsSync(dir)) {
+                await git(repo, 'worktree', 'remove', '--force', dir).catch(() => { });
+                await rm(dir, { recursive: true, force: true });
+            }
+            await mkdir(options.cache, { recursive: true });
+            await git(repo, 'worktree', 'add', '--detach', dir, sha);
+        });
     }
-    const building = staged.filter(ref => ref.build).map(ref => ref.name);
-    if (building.length)
-        options.onStep?.(`building ${building.join(', ')}`);
-    await Promise.all(staged.map(async ({ dir, sha, build }) => {
-        if (sha === null)
-            return;
-        await copyTests(repo, dir, options.root);
-        if (!build)
-            return;
+    // Even a cached worktree gets today's tests, since the ones it was built with are a release behind
+    options.onStep?.('staging tests');
+    await copyTests(repo, dir, options.root);
+    if (build) {
+        options.onStep?.('building');
         await execFile('sh', ['-c', options.build], { cwd: dir, maxBuffer: 1 << 26 });
         await writeFile(join(dir, '.zbench-ref'), sha + '\n');
-    }));
-    return staged.map(({ name, dir, sha }) => ({ name, dir, sha }));
-}
-/** Check out one reference into a reusable worktree, build it, and stage the tests inside it. */
-export async function prepareRef(name, options) {
-    const [ref] = await prepareRefs([name], options);
-    return ref;
+    }
+    return { name, dir, sha };
 }
 /** Remove every cached worktree. */
 export async function cleanRefs(cache) {

package/dist/report.d.ts

@@ -1,13 +1,19 @@
 import type { Comparison, RefResults } from './compare.js';
 import type { Suite } from './config.js';
 import { type Matrix } from './measure.js';
-import type { CaseResult } from './runner.js';
+import type { CaseResult } from './types.js';
 /** Print one matrix as a table, followed by its aggregate measurements. */
 export declare function reportMatrix(matrix: Matrix): void;
 /** Print every test's tables for a single state. */
 export declare function reportRun(suite: Suite, cases: CaseResult[]): void;
+export interface ComparisonOptions {
+    /** Also report matrices where nothing beat the threshold */
+    unchanged?: boolean;
+    /** Also report matrices where fewer than two states produced numbers */
+    partial?: boolean;
+}
 /** Print the delta tables that follow each state's own tables. */
-export declare function reportComparison(comparisons: Comparison[], refs: RefResults[]): void;
+export declare function reportComparison(comparisons: Comparison[], refs: RefResults[], options?: ComparisonOptions): void;
 /** A one-line note about what the numbers were produced under. */
 export declare function reportEnvironment(cpu: number, mem: number, iterations: number, warmup: number, jobs: number): void;
 /** List the tests and configurations that would run, without running them. */

package/dist/report.js

@@ -98,42 +98,75 @@ function factorText(row, index) {
         return dim(text);
     return styleText(better ? 'green' : 'red', text);
 }
+/**
+ * One table holding every column of a matrix, grouped by state.
+ * Each state gets a value per column and a single factor, since the columns of a matrix are
+ * proportional and would otherwise repeat the same speedup.
+ */
+function comparisonTable(names, cols, rows) {
+    if (!cols.length || !rows.length)
+        return;
+    io.table([
+        { name: indent + 'configuration', text: row => indent + row.configuration, grow: 0 },
+        ...names.flatMap((name, r) => [
+            { name, text: () => '', grow: 0 },
+            ...cols.map((column, c) => ({
+                name: column.label,
+                text: (row) => {
+                    const { value } = row.cells[r][c];
+                    return value === null ? dim('N/A') : column.format(value);
+                },
+                padStart: true,
+                grow: 0,
+            })),
+            ...(r == 0
+                ? []
+                : [{ name: '', text: (row) => factorText(row, r), padStart: true, grow: 0 }]),
+        ]),
+    ], { formatHead: dim }, rows);
+}
 /** Print the delta tables that follow each state's own tables. */
-export function reportComparison(comparisons, refs) {
+export function reportComparison(comparisons, refs, options = {}) {
     const names = refs.map(ref => ref.ref);
     let heading = null;
+    let unchanged = 0;
+    let partial = 0;
+    let shown = 0;
     for (const comparison of comparisons) {
-        const rows = comparison.aggregate ? [comparison.aggregate] : comparison.rows;
-        if (!rows.length)
+        if (comparison.reported < 2) {
+            if (!options.partial) {
+                partial++;
+                continue;
+            }
+        }
+        else if (!comparison.changed && !options.unchanged) {
+            unchanged++;
             continue;
-        const label = comparison.test.name + (comparison.label ? dim('  ' + comparison.label) : '');
-        if (label != heading) {
-            console.log();
-            console.log(bold((heading = label)));
         }
-        console.log('  ' + dim(comparison.column.label) + (comparison.aggregate ? dim(' (aggregate)') : ''));
-        io.table([
-            { name: indent + 'configuration', text: row => indent + row.configuration, grow: 0 },
-            {
-                name: names[0],
-                text: row => row.cells[0].value === null ? dim('N/A') : comparison.column.format(row.cells[0].value),
-                padStart: true,
-                grow: 0,
-            },
-            ...names.slice(1).flatMap((name, i) => [
-                {
-                    name,
-                    text: (row) => {
-                        const cell = row.cells[i + 1];
-                        return cell.value === null ? dim('N/A') : comparison.column.format(cell.value);
-                    },
-                    padStart: true,
-                    grow: 0,
-                },
-                { name: '', text: (row) => factorText(row, i + 1), padStart: true, grow: 0 },
-            ]),
-        ], { formatHead: dim }, rows);
+        if (!comparison.rows.length && !comparison.aggregate)
+            continue;
+        console.log();
+        if (comparison.test != heading)
+            console.log(bold((heading = comparison.test).name));
+        if (comparison.label)
+            console.log('  ' + dim(comparison.label));
+        comparisonTable(names, comparison.columns, comparison.rows);
+        if (comparison.aggregate) {
+            if (comparison.columns.length)
+                console.log();
+            comparisonTable(names, comparison.aggregateColumns, [comparison.aggregate]);
+        }
+        shown++;
     }
+    const hidden = [
+        unchanged && `${unchanged} unchanged (--unchanged)`,
+        partial && `${partial} reported by one reference (--partial)`,
+    ].filter(Boolean);
+    if (!hidden.length)
+        return;
+    if (!shown)
+        console.log();
+    console.log(indent + dim(`${hidden.join(', ')}`));
 }
 /** A one-line note about what the numbers were produced under. */
 export function reportEnvironment(cpu, mem, iterations, warmup, jobs) {

package/dist/types.d.ts

@@ -23,3 +23,21 @@ export interface TestModule<C = any, S = any> {
     /** Run once per configuration, after every iteration. */
     teardown?(config: C, state: S): unknown;
 }
+/** The measurements taken for one test at one point in its matrix. */
+export interface CaseResult {
+    /** The test's id, i.e. its path as written in the config */
+    test: string;
+    flags: Record<string, unknown>;
+    configuration: string;
+    value: Record<string, number>;
+    /** Milliseconds `setup` took */
+    setup: number;
+    /** Milliseconds each timed iteration took */
+    samples: number[];
+    /** Raw amounts summed over every timed iteration, keyed by quantity */
+    amounts: Amounts;
+    /** Present when the configuration was not run */
+    skipped?: string;
+    /** Present when the test threw */
+    error?: string;
+}

package/dist/units.d.ts

@@ -1,5 +1,7 @@
+/** Timespans from shortest to longest. */
+export declare const timespanOrder: readonly ["ns", "us", "ms", "s"];
 /** A unit of time a measurement can be expressed in. */
-export type Timespan = 'ns' | 'us' | 'ms' | 's';
+export type Timespan = (typeof timespanOrder)[number];
 /** Milliseconds in each timespan, since every duration is measured in milliseconds. */
 export declare const timespans: {
     readonly ns: 0.000001;
@@ -7,8 +9,6 @@ export declare const timespans: {
     readonly ms: 1;
     readonly s: 1000;
 };
-/** Timespans from shortest to longest. */
-export declare const timespanOrder: readonly ["ns", "us", "ms", "s"];
 export declare function isTimespan(value: unknown): value is Timespan;
 /**
  * Units whose amounts are counted in bytes, so `unit: "MB"` on a byte count reports MB/s without a `scale`.

package/dist/units.js

@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-/** Milliseconds in each timespan, since every duration is measured in milliseconds. */
-export const timespans = { ns: 1e-6, us: 1e-3, ms: 1, s: 1e3 };
 /** Timespans from shortest to longest. */
 export const timespanOrder = ['ns', 'us', 'ms', 's'];
+/** Milliseconds in each timespan, since every duration is measured in milliseconds. */
+export const timespans = { ns: 1e-6, us: 1e-3, ms: 1, s: 1e3 };
 export function isTimespan(value) {
     return typeof value == 'string' && value in timespans;
 }

package/dist/worker.d.ts

@@ -0,0 +1,3 @@
+import { type Test } from './config.js';
+/** Whether a configuration fits in the budget, and why not when it doesn't. */
+export declare function skipReason(configuration: Test['configurations'][number]): string | undefined;

package/dist/worker.js

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { parentPort, workerData } from 'node:worker_threads';
+import { flagCombinations, loadSuite } from './config.js';
+if (!parentPort)
+    throw new Error('zbench/worker is only meant to be run as a worker thread');
+const config = workerData;
+const suite = loadSuite(config.config);
+const test = suite.tests.find(t => t.id == config.test);
+if (!test)
+    throw new Error(`no test with id ${JSON.stringify(config.test)} in ${config.config}`);
+const flagsConfig = Object.fromEntries(Object.entries(config.flags).map(([k, v]) => [k, [v]]));
+/** Narrow a test's flags to the values the caller asked for, dropping the rest. */
+function selectFlags(test) {
+    const flags = {};
+    for (const [name, values] of Object.entries(test.flags)) {
+        const wanted = flagsConfig?.[name];
+        flags[name] = wanted ? values.filter(v => wanted.some(w => Object.is(w, v))) : values;
+    }
+    return flagCombinations(flags);
+}
+/** Whether a configuration fits in the budget, and why not when it doesn't. */
+export function skipReason(configuration) {
+    if (config.all)
+        return undefined;
+    if (config.cpu !== undefined && configuration.cpu > config.cpu)
+        return `needs cpu ${configuration.cpu}`;
+    if (config.mem !== undefined && configuration.mem > config.mem)
+        return `needs mem ${configuration.mem}`;
+    return undefined;
+}
+const module = (await import(test.file));
+if (typeof module.test != 'function')
+    throw new Error(`${test.path} does not export a \`test\` function`);
+const combinations = selectFlags(test);
+const results = [];
+const total = combinations.length * test.configurations.length;
+let index = 0;
+for (const flags of combinations) {
+    for (const configuration of test.configurations) {
+        config.onCase?.(test, flags, configuration.name, index++, total);
+        const skipped = skipReason(configuration);
+        if (skipped) {
+            results.push({
+                test: test.id,
+                flags,
+                configuration: configuration.name,
+                value: configuration.value,
+                setup: 0,
+                samples: [],
+                amounts: {},
+                skipped,
+            });
+            continue;
+        }
+        const testConfig = { ...flags, ...configuration.value };
+        const iterations = config.iterations ?? test.iterations;
+        const warmup = config.warmup ?? test.warmup;
+        const result = {
+            test: test.id,
+            flags,
+            configuration: configuration.name,
+            value: configuration.value,
+            setup: 0,
+            samples: [],
+            amounts: {},
+        };
+        // By key, not by metric: a key with both a per-configuration and an aggregate metric is one quantity
+        const quantities = [...new Set(test.metrics.map(metric => metric.key))];
+        const setupStart = performance.now();
+        let state;
+        try {
+            state = await module.setup?.(testConfig);
+        }
+        catch (e) {
+            result.error = `setup: ${e?.stack ?? e}`;
+            results.push(result);
+            continue;
+        }
+        result.setup = performance.now() - setupStart;
+        try {
+            for (let i = 0; i < warmup + iterations; i++) {
+                const timed = i >= warmup;
+                await module.before?.(testConfig, state);
+                globalThis.gc?.();
+                const start = performance.now();
+                const amounts = await module.test(testConfig, state);
+                const elapsed = performance.now() - start;
+                await module.after?.(testConfig, state);
+                if (!timed)
+                    continue;
+                result.samples.push(elapsed);
+                for (const key of quantities) {
+                    const amount = amounts?.[key] ?? configuration.value[key];
+                    if (amount === undefined)
+                        continue;
+                    result.amounts[key] = (result.amounts[key] ?? 0) + amount;
+                }
+            }
+        }
+        catch (e) {
+            result.error = String(e?.stack ?? e);
+        }
+        finally {
+            try {
+                await module.teardown?.(testConfig, state);
+            }
+            catch (e) {
+                result.error ??= `teardown: ${e?.stack ?? e}`;
+            }
+        }
+        results.push(result);
+    }
+}
+parentPort.postMessage(results);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "zbench-js",
-	"version": "0.1.1",
+	"version": "0.2.0",
 	"description": "Data-driven, reproducible performance testing",
 	"funding": {
 		"type": "individual",
@@ -58,8 +58,9 @@
 		"typescript-eslint": "^8.58.0"
 	},
 	"dependencies": {
-		"ioium": "^1.8.0",
-		"utilium": "^3.0.0"
+		"ioium": "^1.9.0",
+		"utilium": "^3.0.0",
+		"zod": "^4.5.4"
 	},
 	"engines": {
 		"node": ">=22.18.0"

package/dist/pool.d.ts

@@ -1,5 +0,0 @@
-/**
- * Map over `items` with at most `limit` running at once, keeping the results in input order.
- * A `limit` of 1 is a plain serial loop.
- */
-export declare function mapPool<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;

package/dist/pool.js

@@ -1,15 +0,0 @@
-// SPDX-License-Identifier: LGPL-3.0-or-later
-/**
- * Map over `items` with at most `limit` running at once, keeping the results in input order.
- * A `limit` of 1 is a plain serial loop.
- */
-export async function mapPool(items, limit, fn) {
-    const results = new Array(items.length);
-    let next = 0;
-    const worker = async () => {
-        for (let i = next++; i < items.length; i = next++)
-            results[i] = await fn(items[i], i);
-    };
-    await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
-    return results;
-}

package/dist/runner.d.ts

@@ -1,40 +0,0 @@
-import { type Test } from './config.js';
-import type { Amounts, TestModule } from './types.js';
-/** The measurements taken for one test at one point in its matrix. */
-export interface CaseResult {
-    /** The test's id, i.e. its path as written in the config */
-    test: string;
-    flags: Record<string, unknown>;
-    configuration: string;
-    value: Record<string, number>;
-    /** Milliseconds `setup` took */
-    setup: number;
-    /** Milliseconds each timed iteration took */
-    samples: number[];
-    /** Raw amounts summed over every timed iteration, keyed by quantity */
-    amounts: Amounts;
-    /** Present when the configuration was not run */
-    skipped?: string;
-    /** Present when the test threw */
-    error?: string;
-}
-export interface RunOptions {
-    /** Overrides the test's own iteration count */
-    iterations?: number;
-    warmup?: number;
-    /** The machine's budget. Configurations asking for more are skipped. */
-    cpu?: number;
-    mem?: number;
-    /** Run every configuration, whatever the budget says */
-    all?: boolean;
-    /** Restricts a flag to a subset of its declared values */
-    flags?: Record<string, unknown[]>;
-    /** Called before each configuration so callers can show progress */
-    onCase?(test: Test, flags: Record<string, unknown>, configuration: string, index: number, total: number): void;
-}
-/** `import()` the test module and check that it is one. */
-export declare function loadTest(test: Test): Promise<TestModule>;
-/** Whether a configuration fits in the budget, and why not when it doesn't. */
-export declare function skipReason(configuration: Test['configurations'][number], options: RunOptions): string | undefined;
-/** Run every configuration of a test, once per flag combination. */
-export declare function runTest(test: Test, options?: RunOptions): Promise<CaseResult[]>;