zbench-js 0.1.1 → 0.2.0

package/README.md

@@ -105,6 +105,11 @@ readable. The timespan is settled once per matrix, so a column has one heading.
 `"unit": "MB"` reports MB/s with no further arithmetic. `scale` sets the multiplier explicitly and
 overrides that.
 
+`compare` narrows what a comparison reports. A test's measurements are all proportional to its
+timings, so every one of them would show the same speedup; marking a single measurement with
+`"compare": true` keeps the comparison table down to the numbers worth reading, and makes that
+measurement the one the factor is computed from.
+
 ## Flags
 
 Flags are alternative code paths rather than points in the matrix: the whole matrix runs once per
@@ -141,11 +146,21 @@ zbench -R v3.2.1 -R main -R HEAD
 ```
 
 Each reference is checked out into a cached worktree under `.zbench/`, built, and run. The first is
-the baseline; every other column shows its value and a speedup factor that is `> 1` when better,
+the baseline; every other reference gets its columns and a speedup factor that is `> 1` when better,
 whichever direction the column improves in. A change is only colored when it is larger than both
 `--threshold` (default 1%) and the two runs' combined noise — so a green number means the change
 outran the variance, and a gray one means it did not.
 
+A matrix is one table, grouped by reference, with a column per measurement and a single factor for
+each reference. The measurements of a matrix are proportional, so the factor is taken from the first
+column rather than repeated across all of them; `"compare": true` on a measurement picks which one
+that is and drops the rest.
+
+Only the matrices worth reading are printed. A matrix where nothing beat the threshold is left out
+unless you pass `--unchanged`, and one that fewer than two references could report — the rest being
+`N/A` — is left out unless you pass `--partial`; either way the count of what was hidden is printed.
+`--no-runs` drops each reference's own tables and reports nothing but the changes.
+
 Use `.` for the working tree as it is, without a checkout or a build.
 
 The tests themselves always come from the working tree and are copied into each worktree, so an old
@@ -155,14 +170,18 @@ runnable (default `npm install --no-audit --no-fund && npm run build`), `--rebui
 
 ## Isolation and concurrency
 
-Each matrix runs in a fresh process by default, so one configuration cannot warm up or poison the
-next, and a reference's own build is what gets loaded. `--no-isolate` runs everything in-process for
-debugging or profiling.
+Every matrix runs in a fresh worker thread, so one configuration cannot warm up or poison the next,
+and a reference's own build is what gets loaded.
+
+`-J` sets how many matrices are timed at once, defaulting to half the number of hardware threads.
+Concurrent matrices contend for the machine, which inflates absolute numbers and widens `±`;
+the comparison factors hold up because every reference runs under the same contention,
+but pass `-J 1` when the absolute numbers matter more.
 
-`-J` sets how many matrices are timed at once, defaulting to half the number of hardware threads. Concurrent
-matrices contend for the machine, which inflates absolute numbers and widens `±`; the comparison
-factors hold up because every reference runs under the same contention, but pass `-J 1` when the
-absolute numbers are the point.
+References do not overlap: one is finished before the next one starts, so a reference's matrices all
+meet the same contention rather than some of them sharing the machine with the reference they are
+being compared against. `--overlap` lifts that and lets `-J` fill from every reference at once,
+which finishes sooner at the cost of a noisier comparison.
 
 A matrix that crashes or outruns `--timeout` (default 300s) is reported as `N/A` rather than
 bringing down the run, so the references that did finish are still compared. This matters when the
@@ -183,9 +202,12 @@ minutes on the commit before it.
   -J, --jobs <n>          Matrices to time at once.
   -l, --list              List what would run, then exit.
   -j, --json <path>       Write the raw results as JSON.
+      --overlap           Let matrices from different references run at the same time.
+      --no-runs           When comparing, skip each reference's own tables and only report changes.
+      --unchanged         Also report matrices where nothing beat the threshold.
+      --partial           Also report matrices where fewer than two references produced numbers.
       --cpu <n>           Override the detected CPU level.
       --mem <n>           Override the detected memory level.
-      --no-isolate        Run in this process instead of one child per matrix.
       --build <cmd>       Command that makes a reference's worktree runnable.
       --rebuild           Rebuild reference worktrees even when they are up to date.
       --clean             Remove cached reference worktrees, then exit.

package/dist/cli.js

@@ -1,17 +1,16 @@
 #!/usr/bin/env node
-// SPDX-License-Identifier: LGPL-3.0-or-later
 import * as io from 'ioium/node';
 import { writeFile } from 'node:fs/promises';
 import { join, relative, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
 import { parseArgs, styleText } from 'node:util';
 import { resolveCapabilities } from './capabilities.js';
 import { compare } from './compare.js';
 import { findConfig, flagCombinations, flagLabel, loadSuite } from './config.js';
-import { runIsolated, runWorker } from './isolate.js';
-import { cleanRefs, prepareRefs, repoRoot } from './refs.js';
+import { runIsolated } from './isolate.js';
+import { cleanRefs, prepareRef, repoRoot, workingRefs } from './refs.js';
 import { reportComparison, reportEnvironment, reportList, reportRun } from './report.js';
-import { mapPool } from './pool.js';
-import { runTest } from './runner.js';
+import { duration } from './units.js';
 const usage = `Usage: zbench [options] [filter...]
 
 Run the performance tests named by the config file. Filters match a test's name or path.
@@ -28,14 +27,19 @@ Options:
                           0 waits forever. A reference with a pathological regression can
                           otherwise hold the whole comparison open.
   -a, --all               Run every configuration, ignoring cpu/mem requirements.
-  -J, --jobs <n>          Matrices to time at once. [one per hardware thread, 1 with --no-isolate]
+  -J, --jobs <n>          Matrices to time at once. [one per two hardware threads]
                           Concurrent matrices contend for the machine, which widens the ± column;
                           pass -J 1 for the quietest numbers.
   -l, --list              List what would run, then exit.
   -j, --json <path>       Write the raw results as JSON.
+      --overlap           Let matrices from different references run at the same time. By default
+                          a reference is finished before the next one starts, so every matrix of a
+                          reference meets the same contention.
+      --no-runs           When comparing, skip each reference's own tables and only report changes.
+      --unchanged         Also report matrices where nothing beat the threshold.
+      --partial           Also report matrices where fewer than two references produced numbers.
       --cpu <n>           Override the detected CPU level.
       --mem <n>           Override the detected memory level.
-      --no-isolate        Run in this process instead of one child per matrix.
       --build <cmd>       Command that makes a reference's worktree runnable.
       --rebuild           Rebuild reference worktrees even when they are up to date.
       --clean             Remove cached reference worktrees, then exit.
@@ -55,36 +59,40 @@ const { values: opts, positionals: filters } = parseArgs({
         jobs: { short: 'J', type: 'string' },
         list: { short: 'l', type: 'boolean' },
         json: { short: 'j', type: 'string' },
+        overlap: { type: 'boolean' },
+        runs: { type: 'boolean', default: true },
+        unchanged: { type: 'boolean' },
+        partial: { type: 'boolean' },
         cpu: { type: 'string' },
         mem: { type: 'string' },
-        isolate: { type: 'boolean', default: true },
         build: { type: 'string' },
         rebuild: { type: 'boolean' },
         clean: { type: 'boolean' },
         quiet: { short: 'q', type: 'boolean' },
         help: { short: 'h', type: 'boolean' },
-        worker: { type: 'string' },
     },
     allowPositionals: true,
+    allowNegative: true,
 });
 if (opts.help) {
     console.log(usage);
     process.exit(0);
 }
-if (opts.worker) {
-    await runWorker(opts.worker);
-    process.exit(0);
-}
-function fail(message) {
-    io.error(message);
-    process.exit(1);
+const dim = (text) => styleText('gray', text);
+const ignore = () => { };
+// Results go to the console directly, so silencing ioium leaves them in place while the job lines,
+// the progress line and the incidental notices go away
+if (opts.quiet) {
+    io.useOutput({ debug: ignore, log: ignore, info: ignore, warn: console.warn, error: console.error });
+    io.jobs.useDraw(ignore);
+    io.jobs.useClear(ignore);
 }
 function number(value, name) {
     if (value === undefined)
         return undefined;
     const parsed = Number(value);
     if (!Number.isFinite(parsed))
-        fail(`--${name} expects a number, got ${JSON.stringify(value)}`);
+        io.exit(`--${name} expects a number, got ${JSON.stringify(value)}`);
     return parsed;
 }
 /** Parse `-f name=json`, falling back to the raw string when it isn't valid JSON. */
@@ -93,7 +101,7 @@ function parseFlagOverrides(entries) {
     for (const entry of entries) {
         const at = entry.indexOf('=');
         if (at < 0)
-            fail(`--flag expects name=value, got ${JSON.stringify(entry)}`);
+            io.exit(`--flag expects name=value, got ${JSON.stringify(entry)}`);
         const name = entry.slice(0, at);
         const raw = entry.slice(at + 1);
         let value;
@@ -109,19 +117,19 @@ function parseFlagOverrides(entries) {
 }
 const configPath = opts.config ? resolve(opts.config) : findConfig();
 if (!configPath)
-    fail('no config file found; pass --config or add one at tests/perf/config.json');
+    io.exit('no config file found; pass --config or add one at tests/perf/config.json');
 let suite;
 try {
     suite = loadSuite(configPath);
 }
 catch (e) {
-    fail(e.message);
+    io.exit(e);
 }
 if (filters.length) {
     const matches = (test) => filters.some(f => test.name.toLowerCase().includes(f.toLowerCase()) || test.path.includes(f));
     suite.tests = suite.tests.filter(matches);
     if (!suite.tests.length)
-        fail(`no test matches ${filters.map(f => JSON.stringify(f)).join(', ')}`);
+        io.exit(`no test matches ${filters.map(f => JSON.stringify(f)).join(', ')}`);
 }
 if (opts.list) {
     reportList(suite);
@@ -139,30 +147,8 @@ const threshold = number(opts.threshold, 'threshold') ?? 1;
 const timeout = number(opts.timeout, 'timeout') ?? 300;
 const iterations = number(opts.iterations, 'iterations');
 const warmup = number(opts.warmup, 'warmup');
-const cli = resolve(import.meta.dirname, 'cli.js');
 const refNames = opts.ref.length ? opts.ref : ['.'];
-// Concurrent work in one process would interleave the tests' own awaits into each other's timings
-const jobs = Math.max(1, number(opts.jobs, 'jobs') ?? (opts.isolate ? Math.round((navigator.hardwareConcurrency || 0) / 2) || 1 : 1));
-if (jobs > 1 && !opts.isolate)
-    fail('--jobs above 1 needs process isolation, so it cannot be used with --no-isolate');
-/** Overwrite the progress line, if there is one to overwrite. */
-const progress = {
-    active: false,
-    show(message) {
-        if (opts.quiet || !process.stdout.isTTY)
-            return;
-        this.clear();
-        process.stdout.write(styleText('gray', message));
-        this.active = true;
-    },
-    clear() {
-        if (!this.active)
-            return;
-        process.stdout.clearLine(0);
-        process.stdout.cursorTo(0);
-        this.active = false;
-    },
-};
+const jobs = Math.max(1, number(opts.jobs, 'jobs') ?? (Math.round((navigator.hardwareConcurrency || 0) / 2) || 1));
 /** Every (test, flag combination) pair that will run, i.e. one table each. */
 function* planned(suite) {
     for (const test of suite.tests) {
@@ -174,78 +160,101 @@ function* planned(suite) {
             yield [test, flags];
     }
 }
-async function runRef(ref, repo) {
-    const config = ref.sha ? join(ref.dir, relative(repo, suite.file)) : suite.file;
-    const matrices = [...planned(suite)];
-    let done = 0;
-    const results = await mapPool(matrices, jobs, async ([test, flags]) => {
-        const label = [test.name, flagLabel(flags)].filter(Boolean).join(' ');
-        progress.show(`[${done + 1}/${matrices.length}] ${label}${ref.sha ? ` @ ${ref.name}` : ''}...`);
+if (!opts.quiet)
+    reportEnvironment(cpu, mem, iterations ?? 5, warmup ?? 1, jobs);
+const repo = await repoRoot().catch(() => process.cwd());
+const refOptions = { cache, build: opts.build ?? suite.build, root: suite.root, rebuild: opts.rebuild };
+const prepared = new Array(refNames.length);
+// The working tree is used where it is, so the usual case has nothing to report and no job to run
+if (refNames.every(name => workingRefs.includes(name))) {
+    prepared.splice(0, refNames.length, ...(await Promise.all(refNames.map(name => prepareRef(name, refOptions)))));
+}
+else {
+    const failures = [];
+    await io.jobs.runWithData({
+        concurrency: refNames.length,
+        jobStartText: dim('preparing'),
+        name: ({ name }) => name,
+        async run({ name, index }, progress) {
+            try {
+                prepared[index] = await prepareRef(name, { ...refOptions, onStep: m => progress(dim(m)) });
+                return { status: 'succeeded', text: dim('ready') };
+            }
+            catch (e) {
+                const text = String(e?.stderr || e?.message || e).trim();
+                const reason = text.split('\n').filter(Boolean).at(-1) ?? 'failed';
+                failures.push(`${name}: ${reason}`);
+                return { status: 'failed', text: reason };
+            }
+        },
+    }, refNames.map((name, index) => ({ name, index })));
+    // A reference that will not build has no numbers at all, so there is nothing left to compare
+    if (failures.length)
+        io.exit(`could not prepare references:\n  ${failures.join('\n  ')}`);
+}
+const plan = [...planned(suite)];
+const queue = [];
+// Every reference's matrices go into one queue, so a reference that finishes early does not leave
+// the machine idle while a slow one is still running
+for (const ref of prepared)
+    for (const [test, flags] of plan)
+        queue.push({ index: queue.length, ref, test, flags });
+const results = new Array(queue.length);
+const comparing = prepared.length > 1;
+await io.jobs.runWithData({
+    concurrency: jobs,
+    jobStartText: 'running',
+    name: job => [job.test.name, flagLabel(job.flags)].filter(Boolean).join(' ')
+        + (comparing ? dim(' @ ' + job.ref.name) : ''),
+    group: opts.overlap ? undefined : job => job.ref,
+    async run(job) {
+        const config = job.ref.sha ? join(job.ref.dir, relative(repo, suite.file)) : suite.file;
+        const started = performance.now();
         try {
-            return opts.isolate
-                ? await runIsolated({ config, test: test.id, flags, iterations, warmup, cpu, mem, all: opts.all }, ref.dir, cli, timeout)
-                : await runTest(test, {
-                    iterations,
-                    warmup,
-                    cpu,
-                    mem,
-                    all: opts.all,
-                    flags: Object.fromEntries(Object.entries(flags).map(([k, v]) => [k, [v]])),
-                });
+            results[job.index] = await runIsolated({ config, test: job.test.id, flags: job.flags, iterations, warmup, cpu, mem, all: opts.all }, timeout);
+            return { status: 'succeeded', text: styleText('blue', duration(performance.now() - started)) };
         }
         catch (e) {
+            const error = String(e?.message ?? e);
             // A matrix that cannot run is one reference's problem, not the whole comparison's:
             // report it as N/A and let the references that did run still be compared
-            return test.configurations.map((configuration) => ({
-                test: test.id,
-                flags,
+            results[job.index] = job.test.configurations.map((configuration) => ({
+                test: job.test.id,
+                flags: job.flags,
                 configuration: configuration.name,
                 value: configuration.value,
                 setup: 0,
                 samples: [],
                 amounts: {},
-                error: String(e?.message ?? e),
+                error,
             }));
+            return { status: 'failed', text: error.split('\n')[0] };
         }
-        finally {
-            done++;
+    },
+}, queue);
+// The queue is reference-major, so each reference owns one contiguous run of the results
+const refs = prepared.map((ref, i) => ({
+    ref: ref.name,
+    cases: results.slice(i * plan.length, (i + 1) * plan.length).flat(),
+}));
+// With one reference there is no comparison to fall back on, so its own tables are the whole report
+if (opts.runs || !comparing) {
+    for (const [i, ref] of refs.entries()) {
+        if (comparing && !opts.quiet) {
+            const { sha } = prepared[i];
+            console.log();
+            console.log(styleText(['bold', 'underline'], ref.ref) + (sha ? dim('  ' + sha.slice(0, 8)) : ''));
         }
-    });
-    progress.clear();
-    return results.flat();
-}
-if (!opts.quiet)
-    reportEnvironment(cpu, mem, iterations ?? 5, warmup ?? 1, jobs);
-const repo = await repoRoot().catch(() => process.cwd());
-let prepared;
-try {
-    prepared = await prepareRefs(refNames, {
-        cache,
-        build: opts.build ?? suite.build,
-        root: suite.root,
-        rebuild: opts.rebuild,
-        onStep: message => progress.show(message + '...'),
-    });
-}
-catch (e) {
-    progress.clear();
-    fail(`could not prepare references: ${e.stderr || e.message}`);
-}
-progress.clear();
-const refs = [];
-for (const ref of prepared) {
-    if (prepared.length > 1 && !opts.quiet) {
-        console.log();
-        console.log(styleText(['bold', 'underline'], ref.name) + (ref.sha ? styleText('gray', '  ' + ref.sha.slice(0, 8)) : ''));
+        reportRun(suite, ref.cases);
     }
-    const cases = await runRef(ref, repo);
-    reportRun(suite, cases);
-    refs.push({ ref: ref.name, cases });
 }
-if (refs.length > 1) {
+if (comparing) {
     console.log();
     console.log(styleText(['bold', 'underline'], `Change vs ${refs[0].ref}`));
-    reportComparison(compare(suite, refs, threshold), refs);
+    reportComparison(compare(suite, refs, threshold), refs, {
+        unchanged: opts.unchanged,
+        partial: opts.partial,
+    });
 }
 if (opts.json) {
     await writeFile(opts.json, JSON.stringify({
@@ -270,9 +279,9 @@ if (failed.length) {
     for (const [message, where] of byMessage) {
         io.error(`${message}  (${where.length} configuration${where.length == 1 ? '' : 's'})`);
         for (const at of where.slice(0, 4))
-            console.error(styleText('gray', '    ' + at));
+            console.error(dim('    ' + at));
         if (where.length > 4)
-            console.error(styleText('gray', `    ...and ${where.length - 4} more`));
+            console.error(dim(`    ...and ${where.length - 4} more`));
     }
     process.exit(1);
 }

package/dist/compare.d.ts

@@ -1,5 +1,5 @@
-import { type Column } from './measure.js';
-import type { CaseResult } from './runner.js';
+import { type Column, type Matrix } from './measure.js';
+import type { CaseResult } from './types.js';
 import type { Suite, Test } from './config.js';
 /** Results for one of the states being compared. */
 export interface RefResults {
@@ -26,18 +26,31 @@ export interface Delta {
 }
 export interface ComparisonRow {
     configuration: string;
-    /** Indexed the same as the states being compared, baseline first */
-    cells: Cell[];
+    /** Indexed by state first (baseline first), then by column */
+    cells: Cell[][];
+    /**
+     * One per state, from the primary column.
+     * The metrics of a matrix are proportional to each other, so every column would report the
+     * same factor; taking it from one keeps the table free of repeated numbers.
+     */
     deltas: Delta[];
 }
+/** One matrix, with every column it reports lined up across the states being compared. */
 export interface Comparison {
     test: Test;
     /** Flag combination, empty when the test has no flags */
     label: string;
-    column: Column;
+    /** Columns reported once per configuration. The first is the one `deltas` come from. */
+    columns: Column[];
     rows: ComparisonRow[];
-    /** Present for aggregate metrics, which have one value per state rather than one per row */
-    aggregate?: ComparisonRow;
+    /** Columns reported once for the whole matrix rather than once per row */
+    aggregateColumns: Column[];
+    /** The single row those columns fill, when there are any */
+    aggregate: ComparisonRow | null;
+    /** Whether any row's factor beat both the threshold and the noise */
+    changed: boolean;
+    /** How many states produced at least one number here. Below 2 there is nothing to compare. */
+    reported: number;
 }
 /**
  * Compare a cell against the baseline.
@@ -45,7 +58,14 @@ export interface Comparison {
  */
 export declare function delta(baseline: Cell, other: Cell, higherIsBetter: boolean, threshold: number): Delta;
 /**
- * Line up every state's results against the baseline, one comparison per matrix column.
- * `ops/s` is left out because it is just the reciprocal of `avg`.
+ * The columns a comparison reports for a matrix.
+ * A test that marks any measurement with `compare` gets only those; otherwise everything but
+ * `ops/s`, which is just the reciprocal of `avg`.
+ */
+export declare function comparedColumns(matrix: Matrix): Column[];
+/**
+ * Line up every state's results against the baseline, one comparison per matrix.
+ * Every column of a matrix lands in the same table, so a speedup factor is reported once rather
+ * than once per measurement.
  */
 export declare function compare(suite: Suite, refs: RefResults[], threshold: number): Comparison[];

package/dist/compare.js

@@ -9,13 +9,14 @@ function cell(column, result) {
         return { value: null, noise: 0, missing: true };
     return { value: column.value(result), noise: timing(result).rsd, missing: false };
 }
+const noDelta = { factor: null, change: null, significant: false, better: false };
 /**
  * Compare a cell against the baseline.
  * A change is only called out when it is bigger than both `threshold` and the two runs' combined noise.
  */
 export function delta(baseline, other, higherIsBetter, threshold) {
     if (baseline.value === null || other.value === null || !baseline.value)
-        return { factor: null, change: null, significant: false, better: false };
+        return noDelta;
     const factor = higherIsBetter ? other.value / baseline.value : baseline.value / other.value;
     const change = ((other.value - baseline.value) / baseline.value) * 100;
     const noise = combinedNoise(baseline.noise, other.noise);
@@ -27,8 +28,44 @@ export function delta(baseline, other, higherIsBetter, threshold) {
     };
 }
 /**
- * Line up every state's results against the baseline, one comparison per matrix column.
- * `ops/s` is left out because it is just the reciprocal of `avg`.
+ * The columns a comparison reports for a matrix.
+ * A test that marks any measurement with `compare` gets only those; otherwise everything but
+ * `ops/s`, which is just the reciprocal of `avg`.
+ */
+export function comparedColumns(matrix) {
+    const all = columns(matrix);
+    const selected = all.filter(column => column.compare);
+    return selected.length ? selected : all.filter(column => column.label != 'ops/s');
+}
+/** Line up one configuration's cells across every state, with the primary column's delta. */
+function row(configuration, cells, primary, threshold) {
+    return {
+        configuration,
+        cells,
+        deltas: primary
+            ? cells.map(c => delta(cells[0][0], c[0], primary.higherIsBetter, threshold))
+            : cells.map(() => noDelta),
+    };
+}
+/** The matrix-wide value for each state, compared the same way a row is. */
+function aggregateRow(matrix, cols, refs, threshold) {
+    const flags = JSON.stringify(matrix.flags);
+    const cells = refs.map((ref) => {
+        const cases = ref.cases.filter(r => r.test == matrix.test.id && JSON.stringify(r.flags) == flags);
+        const noise = cases.filter(c => c.samples.length).map(c => timing(c).rsd);
+        // An aggregate spans the whole matrix, so its noise is the matrix's, not any one row's
+        const spread = noise.length ? noise.reduce((sum, v) => sum + v, 0) / noise.length : 0;
+        return cols.map((column) => {
+            const total = column.total?.(cases) ?? null;
+            return { value: total, noise: spread, missing: total === null };
+        });
+    });
+    return row('aggregate', cells, cols[0], threshold);
+}
+/**
+ * Line up every state's results against the baseline, one comparison per matrix.
+ * Every column of a matrix lands in the same table, so a speedup factor is reported once rather
+ * than once per measurement.
  */
 export function compare(suite, refs, threshold) {
     const [baseline, ...others] = refs;
@@ -38,43 +75,26 @@ export function compare(suite, refs, threshold) {
     const comparisons = [];
     for (const test of suite.tests) {
         for (const matrix of matrices(test, baseline.cases)) {
-            for (const column of columns(matrix).filter(c => c.label != 'ops/s')) {
-                const rows = matrix.cases.map(result => {
-                    const cells = indexes.map(index => cell(column, index.get(key(result))));
-                    return {
-                        configuration: result.configuration,
-                        cells,
-                        deltas: cells.map(c => delta(cells[0], c, column.higherIsBetter, threshold)),
-                    };
-                });
-                comparisons.push({
-                    test,
-                    label: matrix.label,
-                    column,
-                    rows: column.aggregate ? [] : rows,
-                    aggregate: column.aggregate ? aggregateRow(matrix, column, refs, threshold) : undefined,
-                });
-            }
+            const selected = comparedColumns(matrix);
+            const perRow = selected.filter(column => !column.aggregate);
+            const aggregateColumns = selected.filter(column => column.aggregate);
+            const rows = perRow.length
+                ? matrix.cases.map(result => row(result.configuration, indexes.map(index => perRow.map(column => cell(column, index.get(key(result))))), perRow[0], threshold))
+                : [];
+            const aggregate = aggregateColumns.length ? aggregateRow(matrix, aggregateColumns, refs, threshold) : null;
+            const all = aggregate ? [...rows, aggregate] : rows;
+            comparisons.push({
+                test,
+                label: matrix.label,
+                columns: perRow,
+                rows,
+                aggregateColumns,
+                aggregate,
+                // The baseline's delta against itself is never significant, so this only counts real changes
+                changed: all.some(r => r.deltas.some(d => d.significant)),
+                reported: refs.filter((_, r) => all.some(row => row.cells[r].some(c => c.value !== null))).length,
+            });
         }
     }
     return comparisons;
 }
-/** The matrix-wide value for each state, compared the same way a row is. */
-function aggregateRow(matrix, column, refs, threshold) {
-    const flags = JSON.stringify(matrix.flags);
-    const cells = refs.map((ref) => {
-        const cases = ref.cases.filter(r => r.test == matrix.test.id && JSON.stringify(r.flags) == flags);
-        const total = column.total?.(cases) ?? null;
-        const noise = cases.filter(c => c.samples.length).map(c => timing(c).rsd);
-        return {
-            value: total,
-            noise: noise.length ? noise.reduce((sum, v) => sum + v, 0) / noise.length : 0,
-            missing: total === null,
-        };
-    });
-    return {
-        configuration: 'aggregate',
-        cells,
-        deltas: cells.map(c => delta(cells[0], c, column.higherIsBetter, threshold)),
-    };
-}