webperf-snippets 1.15.0 → 1.17.0

package/snippets/_view/report.js

@@ -0,0 +1,60 @@
+// What a set of results found, the same for the audit of the extension and the reports of the CLI: the
+// issues of each result by severity, the overview of all of them and every issue in one list. Only
+// counts and order: how each one shows them is its own.
+
+const SEVERITY = ['error', 'warning', 'info']
+const RATING = { good: 'good', 'needs-improvement': 'needs improvement', poor: 'poor' }
+const NOTHING = 'The snippet returned nothing'
+
+// The issues of a result by severity and the worst of them. A result that failed counts as an error,
+// and a rating that is not good as a warning when no issue says more
+export function summarizeIssues(result) {
+  if (!result || typeof result !== 'object' || result.status === 'error') {
+    return { status: 'error', errors: 1, warnings: 0, infos: 0, top: result?.error ?? NOTHING }
+  }
+  const issues = Array.isArray(result.issues) ? result.issues : []
+  const count = (severity) => issues.filter((i) => i.severity === severity).length
+  const worst = [...issues].sort((a, b) => SEVERITY.indexOf(a.severity) - SEVERITY.indexOf(b.severity))[0]
+  const summary = { status: result.status, errors: count('error'), warnings: count('warning'), infos: count('info'), top: worst?.message ?? '' }
+  if (issues.length === 0 && (result.rating === 'poor' || result.rating === 'needs-improvement')) {
+    return { ...summary, warnings: 1, top: `Rated ${RATING[result.rating]}` }
+  }
+  return summary
+}
+
+// How many results, from their summaries, have errors, have warnings and no error, or have nothing to fix
+export function overviewOf(summaries) {
+  const withErrors = summaries.filter((s) => s.errors > 0).length
+  const withWarnings = summaries.filter((s) => s.errors === 0 && s.warnings > 0).length
+  return { snippets: summaries.length, withErrors, withWarnings, withNothingToFix: summaries.length - withErrors - withWarnings }
+}
+
+export const describeOverview = ({ snippets, withErrors, withWarnings, withNothingToFix }) =>
+  `${snippets} snippet${snippets === 1 ? '' : 's'}: ${withErrors} with errors, ${withWarnings} with warnings, ${withNothingToFix} with nothing to fix`
+
+// Every issue of a list of results, in their order, each with the id of its result; a result that
+// failed gives an error issue with its message
+export function issuesOf(results) {
+  return results.flatMap((result) => {
+    const failed = !result || result.status === 'error'
+    const list = failed ? [{ severity: 'error', message: result?.error ?? NOTHING }] : Array.isArray(result.issues) ? result.issues : []
+    return list.map((i) => ({ ...(i.code ? { code: i.code } : {}), severity: i.severity, id: result?.id, message: i.message, ...(i.target ? { target: i.target } : {}) }))
+  })
+}
+
+// Every issue of the results in one list, by severity, each with the id of its result and the path of
+// its snippet: the one the result carries, or the one pathOf gives for it. Array.prototype.sort keeps the
+// order of the results within a severity
+export function issueList(results, pathOf = (result) => result?.path) {
+  return results
+    .flatMap((result) =>
+      issuesOf([result]).map(({ code, severity, id, message, target }) => ({ ...(code ? { code } : {}), severity, id, path: pathOf(result), message, ...(target ? { target } : {}) }))
+    )
+    .sort((a, b) => SEVERITY.indexOf(a.severity) - SEVERITY.indexOf(b.severity))
+}
+
+// The audit as JSON: the results, each with the id and the path of its step, plus the overview and every
+// issue of every result in one list, by severity, with the path of its snippet
+export function auditReport({ url, date, results }) {
+  return { workflow: 'audit', url: url ?? null, date, overview: overviewOf(results.map(summarizeIssues)), issues: issueList(results), results }
+}

package/src/baseline.js

@@ -99,12 +99,18 @@ function statsOf(base, current, delta) {
 }
 
 // A regression must beat the threshold of its metric and, with five runs a side, be unlikely to come
-// from the noise of the runs (p < 0.05). With fewer, when the baseline has several runs, it must beat
-// the worst of them: a median inside the range of the baseline is noise the baseline already had.
+// from the noise of the runs (p < 0.05). A change that beats its threshold without that evidence is
+// inconclusive: the runs can tell neither a change nor its absence, so more runs, or another
+// measurement, decide. With fewer runs, when the baseline has several, it must beat the worst of them:
+// a median inside the range of the baseline is noise the baseline already had.
 function verdict(base, current, threshold, stats) {
   const worse = current.value - base.value > threshold;
   const better = base.value - current.value > threshold;
-  if (stats) return stats.p < ALPHA ? (worse ? "regressed" : better ? "improved" : "unchanged") : "unchanged";
+  if (stats) {
+    if (!worse && !better) return "unchanged";
+    if (stats.p >= ALPHA) return "inconclusive";
+    return worse ? "regressed" : "improved";
+  }
   if (worse && (!base.range || current.value > base.range.max)) return "regressed";
   if (better && (!base.range || current.value < base.range.min)) return "improved";
   return "unchanged";
@@ -278,12 +284,21 @@ function warningsOf(base, current, metrics) {
   return warnings;
 }
 
+// The metrics whose change beat their threshold for the worse without the evidence of the runs. Every
+// metric is better lower, CLS included, so a positive change is worse.
+export function inconclusiveForTheWorse({ metrics }) {
+  return Object.entries(metrics)
+    .filter(([, m]) => m.verdict === "inconclusive" && m.delta > 0)
+    .map(([metric]) => metric);
+}
+
 // A page against its baseline, as the `baseline` field of its JSON. The runs of the baseline say how
-// much to trust its values, in the header of the reports.
-export function compareRun(base, current, { file, thresholds = DEFAULT_THRESHOLDS }) {
+// much to trust its values, in the header of the reports. A baseline read from a file names it; one
+// measured in the same run, a live baseline, has no file, and its URL names it.
+export function compareRun(base, current, { file, live = false, thresholds = DEFAULT_THRESHOLDS }) {
   const metrics = compareMetrics(base, current, thresholds);
   return {
-    file,
+    ...(live ? { live: true } : { file }),
     url: base.url,
     runs: base.runs ? { count: base.runs.count, medianBy: base.runs.medianBy } : { count: 1 },
     metrics,

package/src/bin.js

@@ -4,6 +4,7 @@ import { parseArgs } from "node:util";
 import { loadSnippet } from "./load-snippet.js";
 import { resolveSnippetName } from "./snippet-names.js";
 import { runSnippets, runMeasurement, VIEWPORT_PRESETS } from "./runner.js";
+import { readInteractionScript, InteractionError } from "./interactions.js";
 import { WORKFLOWS } from "./workflows/index.js";
 import { RULES } from "./decision-tree.js";
 import { reportHuman, reportMatrixHuman } from "./reporters/human.js";
@@ -17,13 +18,15 @@ import { NETWORK_PRESETS, parseCpuFactor } from "./throttling.js";
 import { parseOwnDomains } from "./own-domains.js";
 import { MEDIAN_METRICS, measureRuns } from "./runs.js";
 import { loadMatrix } from "./matrix.js";
-import { blockingErrors } from "./blocking.js";
+import { failingIssues, parseFailOn } from "./blocking.js";
 import { MEASURED_BY, checkBudgets, parseBudgets, unmeasuredMetrics, violationsOf } from "./budgets.js";
 import { measurementConditions } from "./conditions.js";
+import { pathsOfRun, reportOf } from "./overview.js";
 import { machineOf } from "./machine.js";
 import { isHttpUrl } from "./url.js";
 import { loadBaseline, parseThreshold, thresholdsWith, compareRun, compareMatrix } from "./baseline.js";
-import { describeRegressions, describeMatrixRegressions, matrixComparisonView } from "./reporters/baseline-view.js";
+import { describeRegressions, describeInconclusive, describeMatrixRegressions, describeMatrixInconclusive, matrixComparisonView } from "./reporters/baseline-view.js";
+import { parseLiveBaseline, measureLiveBaseline } from "./live-baseline.js";
 
 
 const USAGE = `webperf-snippets <url> [options]
@@ -73,13 +76,22 @@ Options:
   --median-by <metric>      Metric that picks the median run: LCP, FCP, TTFB, CLS, INP,
                             navMs (default: the first of LCP, FCP, TTFB, INP, navMs
                             measured)
-  --baseline <file>         Compare the page, or each combination of a matrix, with the --json
-                            output of an earlier run: which metrics got worse or better beyond
-                            the noise, and which errors are new or fixed. It only informs,
-                            unless --fail-on-regression
+  --baseline <file|url>     Compare the page, or each combination of a matrix, with the --json
+                            output of an earlier run, or the page with another URL measured in
+                            this run, their runs alternating (a live baseline): which metrics got
+                            worse or better beyond the noise, and which errors are new or fixed.
+                            It only informs, unless --fail-on-regression
   --threshold <metric>=<v>  The threshold of a metric for --baseline, such as LCP=15%,
                             LCP=200ms or CLS=0.1. Repeat it for more metrics
   --fail-on-regression      Exit 1 when a metric regressed or an error is new against --baseline
+  --fail-on-inconclusive    Exit 1 when a metric is inconclusive for the worse against --baseline:
+                            over its threshold, without the evidence of the runs
+  --max-runs <n>            With a live baseline and --runs 5 or more: when a metric is
+                            inconclusive for the worse, measure both pages up to n runs each (at
+                            most 9) and compare once more
+  --fail-on <level>         The issues of the snippets that exit 1: error (default), warning
+                            (errors and warnings) or never. Budgets, --fail-on-regression
+                            and runs that fail keep exiting 1
   --crux-key <key>          Add CrUX field data (p75) next to the measured values.
   --crux-history <weeks>    Add the trend of the field data over the last 1 to 40 weeks (needs a CrUX key).
                             Also read from the CRUX_API_KEY environment variable
@@ -107,12 +119,22 @@ Examples:
   npx webperf-snippets --matrix matrix.json --runs 3 --budget-lcp 2500
   npx webperf-snippets https://example.com --runs 5 --json > baseline.json
   npx webperf-snippets https://pr-123.example.com --runs 5 --baseline baseline.json --fail-on-regression
+  npx webperf-snippets https://pr-123.example.com --runs 5 --max-runs 7 --baseline https://example.com --fail-on-regression
   npx webperf-snippets --matrix matrix.json --baseline last-week.json
   npx webperf-snippets https://example.com --snippet INP --interact-script interactions.json
   npx webperf-snippets https://example.com/dashboard --storage-state auth.json
   npx webperf-snippets https://review.example.com --header "CF-Access-Client-Id: $CF_ID" --header "CF-Access-Client-Secret: $CF_SECRET"
 `;
 
+// A pipe takes 64 KB at a time and process.exit() does not wait for the rest, so a report larger than
+// that reached `| jq` or a script cut short. The report is written whole before the code after it exits
+const writeReport = (text) => new Promise((resolve) => process.stdout.write(text, () => resolve()));
+// A reader that closes the pipe early, as head does, ends the writing, not the run: no error, and the
+// exit code is the one of the report
+process.stdout.on("error", (err) => {
+  if (err.code !== "EPIPE") throw err;
+});
+
 function fail(message, code = 2) {
   process.stderr.write(`${message}\n`);
   process.exit(code);
@@ -177,6 +199,9 @@ async function main() {
         baseline: { type: "string" },
         threshold: { type: "string", multiple: true },
         "fail-on-regression": { type: "boolean" },
+        "fail-on-inconclusive": { type: "boolean" },
+        "max-runs": { type: "string" },
+        "fail-on": { type: "string" },
         "crux-key": { type: "string" },
         "crux-history": { type: "string" },
         "report-to": { type: "string" },
@@ -207,15 +232,19 @@ async function main() {
   const url = positionals[0];
   const matrixPath = values.matrix;
   if (url && matrixPath) fail("Use a URL or --matrix, not both");
-  if (!url && !matrixPath) {
-    process.stdout.write(USAGE);
-    process.exit(2);
-  }
+  // To stderr, as every usage error: a pipe or a redirect of stdout gets no usage text as a report
+  if (!url && !matrixPath) fail(USAGE.trimEnd());
   // Without the URL itself, which can carry a token in its query, as a header can
   if (url && !isHttpUrl(url)) fail("The URL must be an http or https URL, such as https://example.com or http://localhost:3000");
   if (matrixPath && values.viewport) fail("--viewport does not apply to --matrix: set the viewport of each device in the file");
 
-  const waitMs = values.wait ? Number(values.wait) : 3000;
+  if (values.json && values.markdown) fail("Use --json or --markdown, not both");
+
+  // Number("") is 0 and Number("abc") is NaN, which the wait takes as no wait: only digits are a wait
+  if (values.wait != null && !/^\d+$/.test(values.wait)) {
+    fail(`Invalid --wait "${values.wait}". Use a number of milliseconds, such as 3000, or 0 for no wait`);
+  }
+  const waitMs = values.wait != null ? Number(values.wait) : 3000;
   const viewportName = values.viewport ?? "mobile";
   const viewport = VIEWPORT_PRESETS[viewportName];
   if (!viewport) {
@@ -235,7 +264,15 @@ async function main() {
     if (problem) fail(problem);
   }
 
-  const interactScript = values["interact-script"];
+  // The steps are read and checked before the browser opens: a wrong file fails at once
+  let interactions;
+  if (values["interact-script"] != null) {
+    try {
+      interactions = readInteractionScript(values["interact-script"]);
+    } catch (err) {
+      fail(err.message);
+    }
+  }
   const storageState = values["storage-state"];
   if (storageState && !existsSync(storageState)) {
     fail(`Storage state file not found: ${storageState}`);
@@ -276,21 +313,37 @@ async function main() {
     return cruxWeeks ? { ...crux, history: await fetchCruxHistory({ url, apiKey: cruxKey, formFactor, weeks: cruxWeeks }) } : crux;
   };
 
-  // The baseline is read and checked before the browser opens: a wrong file fails at once
+  // The baseline is read and checked before the browser opens: a wrong file fails at once. A URL is a
+  // live baseline, measured in this run with the page
   let baseline;
   let thresholds;
+  let live;
+  let maxRuns;
+  try {
+    ({ live, maxRuns } = parseLiveBaseline({ baseline: values.baseline, matrix: matrixPath, runs: runCount, maxRuns: values["max-runs"] }));
+  } catch (err) {
+    fail(err.message);
+  }
   if (values.baseline) {
     try {
-      baseline = loadBaseline(values.baseline);
+      if (!live) baseline = loadBaseline(values.baseline);
       thresholds = thresholdsWith((values.threshold ?? []).map(parseThreshold));
     } catch (err) {
       fail(err.message);
     }
-    if (!matrixPath && baseline.kind === "matrix") fail(`${values.baseline} is the baseline of a matrix: compare it with --matrix`);
-    if (matrixPath && baseline.kind === "page") fail(`${values.baseline} is the baseline of a page: compare it with a URL, not --matrix`);
+    if (!matrixPath && baseline?.kind === "matrix") fail(`${values.baseline} is the baseline of a matrix: compare it with --matrix`);
+    if (matrixPath && baseline?.kind === "page") fail(`${values.baseline} is the baseline of a page: compare it with a URL, not --matrix`);
   } else {
     if (values.threshold) fail("--threshold needs --baseline");
     if (values["fail-on-regression"]) fail("--fail-on-regression needs --baseline");
+    if (values["fail-on-inconclusive"]) fail("--fail-on-inconclusive needs --baseline");
+  }
+
+  let failOn;
+  try {
+    failOn = parseFailOn(values["fail-on"]);
+  } catch (err) {
+    fail(err.message);
   }
 
   let budgets;
@@ -303,12 +356,12 @@ async function main() {
   // What each page load runs, a snippet or a workflow, under the conditions of the load
   const items = values.snippet ? buildSnippetItem(values) : null;
   const workflow = values.snippet ? null : WORKFLOWS[values.workflow ?? "core-web-vitals"];
-  if (!values.snippet && !workflow) fail(`Unknown workflow: ${values.workflow}`);
+  if (!values.snippet && !workflow) fail(`Unknown workflow: "${values.workflow}". Choose from: ${Object.keys(WORKFLOWS).join(", ")}`);
   // A budget of a metric that no snippet of the run measures could never pass: a usage error
   const [unmeasured] = unmeasuredMetrics(budgets, items ? [items[0].path] : workflow.steps.map((step) => step.path));
   if (unmeasured) fail(budgetUsage(unmeasured, items ? `--snippet ${values.snippet}` : `the ${workflow.id} workflow`));
   const measureWith = (conditions) => {
-    const options = { waitMs, headless: !values.headed, interactScript, ...conditions };
+    const options = { waitMs, headless: !values.headed, interactions, ...conditions };
     return items ? runSnippets({ ...options, items }) : runMeasurement({ ...options, workflow, rules: RULES });
   };
   // What the JSON records of how each page was measured, for a later run to compare with. The machine is
@@ -318,6 +371,9 @@ async function main() {
   const conditionsFor = ({ viewport, cpu, network, browser }) =>
     measurementConditions({ viewport, waitMs, cpu, network, browser, machine, ...(items ? { snippet: items[0].path } : { workflow: values.workflow ?? "core-web-vitals" }) });
 
+  // The path of the snippet of each result, for the list of issues of the report
+  const pathOf = pathsOfRun({ items: items ?? [], workflow, rules: RULES });
+
   if (matrixPath) {
     let matrix;
     try {
@@ -325,26 +381,60 @@ async function main() {
     } catch (err) {
       fail(err.message);
     }
-    await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds, budgets, cruxKey, fieldDataOf });
+    await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, pathOf, failOn, baseline, thresholds, budgets, cruxKey, fieldDataOf });
     return;
   }
 
   const conditions = { url, viewport, storageState, headers, cpu, network, ownDomains };
+  // When the measurement started, in UTC
+  const date = new Date().toISOString();
+  // A page as the report gives it: what the results found goes before them, so a reader, or an agent,
+  // sees it first, and how it was measured after them
+  const pageOf = ({ browser, ...measured }) => ({
+    url: measured.url,
+    date,
+    navMs: measured.navMs,
+    ...reportOf(measured.results, pathOf),
+    ...measured,
+    conditions: conditionsFor({ viewport: viewportName, cpu, network, browser }),
+  });
+  // The median run of several, with their runs field; one run without --runs has none, as always
+  const medianOf = (median) => (runCount == null ? median.payload : { ...median.payload, runs: median.runs });
+
   let payload;
-  if (runCount == null) {
-    payload = await measureWith(conditions);
+  let comparison;
+  if (live) {
+    // Both pages under the same conditions, their runs alternating, and the comparison of the CLI
+    const urls = { baseline: values.baseline, current: url };
+    const measuredLive = await measureLiveBaseline((side) => measureWith({ ...conditions, url: urls[side] }), {
+      runs: runCount ?? 1,
+      maxRuns,
+      medianBy,
+      compare: (base, current) => compareRun(pageOf(medianOf(base)), pageOf(medianOf(current)), { live: true, thresholds }),
+      isFatal: (err) => err instanceof MissingBrowserError,
+      onRun: ({ side, run, runs, step, steps }) => process.stderr.write(`[${step}/${steps}] ${side} ${urls[side]} · run ${run}/${runs}\n`),
+      onExtend: ({ after, metrics, maxRuns }) =>
+        process.stderr.write(`${metrics.join(", ")} inconclusive for the worse after ${after} runs a side: measuring both pages up to ${maxRuns} runs each\n`),
+    });
+    if (!measuredLive.current.payload) throw new Error(`Every run failed: ${measuredLive.current.errors.join("; ")}`);
+    if (!measuredLive.baseline.payload) throw new Error(`Every run of the baseline ${values.baseline} failed: ${measuredLive.baseline.errors.join("; ")}`);
+    payload = pageOf(medianOf(measuredLive.current));
+    // The runs it added, next to the runs of the baseline
+    const { metrics, errors, warnings, ...head } = measuredLive.comparison;
+    comparison = { ...head, ...(measuredLive.extended ? { extended: measuredLive.extended } : {}), metrics, errors, warnings };
+  } else if (runCount == null) {
+    payload = pageOf(await measureWith(conditions));
   } else {
     const median = await measureRunsWithProgress(() => measureWith(conditions), runCount, medianBy, (i) => `[${i}/${runCount}] ${url} · run ${i}/${runCount}`);
     if (!median.payload) throw new Error(`Every run failed: ${median.errors.join("; ")}`);
-    payload = { ...median.payload, runs: median.runs };
+    payload = pageOf(medianOf(median));
   }
-  const { browser, ...measured } = payload;
-  payload = { ...measured, conditions: conditionsFor({ viewport: viewportName, cpu, network, browser }) };
 
   if (cruxKey) payload.crux = await fieldDataOf(url, formFactorFor(viewportName));
 
-  if (baseline) {
-    payload.baseline = compareRun(baseline.data, payload, { file: baseline.file, thresholds });
+  if (baseline) comparison = compareRun(baseline.data, payload, { file: baseline.file, thresholds });
+  if (comparison) {
+    payload.baseline = comparison;
     for (const warning of payload.baseline.warnings) process.stderr.write(`Warning: ${warning}\n`);
   }
 
@@ -359,7 +449,7 @@ async function main() {
   } else {
     output = reportHuman({ ...payload, verbose: values.verbose });
   }
-  process.stdout.write(output + "\n");
+  await writeReport(output + "\n");
 
   if (reportUrl) {
     const outcome = await reportTo({
@@ -371,6 +461,8 @@ async function main() {
         ...(values.snippet ? { snippet: values.snippet } : {}),
         timestamp: new Date().toISOString(),
         navMs: payload.navMs,
+        overview: payload.overview,
+        issues: payload.issues,
         results: payload.results,
         ...(payload.budgets ? { budgets: payload.budgets } : {}),
         meta: { viewport: viewportName, waitMs, ...(payload.throttling ? { throttling: payload.throttling } : {}), ...(payload.runs ? { runs: payload.runs } : {}), cli_version: cliVersion },
@@ -389,13 +481,15 @@ async function main() {
     for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
     process.exit(1);
   }
-  // Without --fail-on-regression the comparison only informs, and the exit code is the one of today
+  // Without --fail-on-regression or --fail-on-inconclusive the comparison only informs, and the exit
+  // code is the one of today. A live baseline is named by its URL, a file by its path
+  const against = live ? values.baseline : baseline?.file;
   const regressions = values["fail-on-regression"] ? describeRegressions(payload.baseline) : [];
-  if (regressions.length > 0) {
-    for (const r of regressions) process.stderr.write(`Regression against ${baseline.file}: ${r}\n`);
-    process.exit(1);
-  }
-  process.exit(blockingErrors(payload.results).length > 0 ? 1 : 0);
+  const inconclusive = values["fail-on-inconclusive"] ? describeInconclusive(payload.baseline) : [];
+  for (const r of regressions) process.stderr.write(`Regression against ${against}: ${r}\n`);
+  for (const i of inconclusive) process.stderr.write(`Inconclusive against ${against}: ${i}\n`);
+  if (regressions.length > 0 || inconclusive.length > 0) process.exit(1);
+  process.exit(failingIssues(payload.results, failOn).length > 0 ? 1 : 0);
 }
 
 // Each run is reported on stderr as it starts, so stdout keeps only the report
@@ -411,7 +505,7 @@ const measureRunsWithProgress = (measure, count, medianBy, progress) =>
 
 // Every combination of the matrix, in order, each one `matrix.runs` times. Each combination reports
 // its median run, and the exit code follows the same rules as one page, for every combination.
-async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds, budgets, cruxKey, fieldDataOf }) {
+async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, pathOf, failOn, baseline, thresholds, budgets, cruxKey, fieldDataOf }) {
   for (const warning of matrix.warnings) process.stderr.write(`Warning: ${warning}\n`);
   const viewportOf = new Map(matrix.combinations.map((c) => [c.id, c.viewport]));
 
@@ -419,15 +513,17 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
   const width = String(total).length;
   let started = 0;
   const combinations = [];
+  const startedAt = new Date().toISOString();
   for (const c of matrix.combinations) {
+    const date = new Date().toISOString();
     const conditions = { url: c.url, viewport: VIEWPORT_PRESETS[c.viewport], storageState: c.storageState, headers: c.headers, cpu: c.cpu, network: c.network, ownDomains: c.ownDomains };
     const median = await measureRunsWithProgress(() => measureWith(conditions), matrix.runs, matrix.medianBy, (i) => `[${String(++started).padStart(width)}/${total}] ${c.id} · run ${i}/${matrix.runs}`);
     const identity = { id: c.id, name: c.name, device: c.device, ...(c.variant ? { variant: c.variant } : {}), url: c.url };
     const { browser, ...measured } = median.payload ?? {};
     combinations.push(
       median.payload
-        ? { ...identity, ...measured, runs: median.runs, conditions: conditionsFor({ ...c, browser }) }
-        : { ...identity, status: "error", error: `Every run failed: ${median.errors.join("; ")}`, runs: { count: matrix.runs, failed: matrix.runs }, conditions: conditionsFor(c) }
+        ? { ...identity, date, navMs: measured.navMs, ...reportOf(measured.results, pathOf), ...measured, runs: median.runs, conditions: conditionsFor({ ...c, browser }) }
+        : { ...identity, date, status: "error", error: `Every run failed: ${median.errors.join("; ")}`, runs: { count: matrix.runs, failed: matrix.runs }, conditions: conditionsFor(c) }
     );
   }
 
@@ -456,8 +552,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
 
   const withBaseline = comparison ? { baseline: comparison } : {};
   const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose, ...withBaseline };
-  const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report);
-  process.stdout.write(output + "\n");
+  const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, date: startedAt, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report);
+  await writeReport(output + "\n");
 
   // One report for each combination, with the body of one page and the device, variant and runs of
   // the combination. The headers are conditions of the load, never part of the body.
@@ -472,6 +568,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
           ...(values.snippet ? { snippet: values.snippet } : {}),
           timestamp: new Date().toISOString(),
           navMs: c.navMs,
+          overview: c.overview,
+          issues: c.issues,
           results: c.results,
           ...(c.budgets ? { budgets: c.budgets } : {}),
           meta: {
@@ -495,15 +593,19 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
 
   const violations = combinations.flatMap((c) => violationsOf(c.budgets ?? []).map((v) => `${v} (${c.id})`));
   for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
-  const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0);
+  const failed = combinations.some((c) => c.error || failingIssues(c.results, failOn).length > 0);
   // Without --fail-on-regression the comparison only informs
   const regressions = values["fail-on-regression"] ? describeMatrixRegressions(combinations) : [];
   for (const r of regressions) process.stderr.write(`Regression against ${baseline.file} in ${r}\n`);
-  process.exit(violations.length > 0 || failed || regressions.length > 0 ? 1 : 0);
+  const inconclusive = values["fail-on-inconclusive"] ? describeMatrixInconclusive(combinations) : [];
+  for (const i of inconclusive) process.stderr.write(`Inconclusive against ${baseline.file} in ${i}\n`);
+  process.exit(violations.length > 0 || failed || regressions.length > 0 || inconclusive.length > 0 ? 1 : 0);
 }
 
 main().catch((err) => {
   // A missing Chromium is a step of the setup, not a bug: the message says what to run, without a stack
-  process.stderr.write(err instanceof MissingBrowserError ? `${err.message}\n` : `Error: ${err.stack ?? err.message}\n`);
+  // Nor is a step of --interact-script that the page did not take: its message names the step
+  const known = err instanceof MissingBrowserError || err instanceof InteractionError;
+  process.stderr.write(known ? `${err.message}\n` : `Error: ${err.stack ?? err.message}\n`);
   process.exit(1);
 });

package/src/blocking.js

@@ -1,14 +1,35 @@
-// The results that make the command exit with 1: a snippet that failed, and an issue of severity
-// error in a step the workflow runs itself. A follow-up (a result with a reason) only informs: its
-// error issues show in the report and leave the exit code to the budgets and to the steps.
-export function blockingErrors(results) {
-  const errors = [];
+// The results that make the command exit with 1: a snippet that failed, and an issue of the severities
+// that count in a step the workflow runs itself. A follow-up (a result with a reason) only informs: its
+// issues show in the report and leave the exit code to the budgets and to the steps.
+function failing(results, severities) {
+  const issues = [];
   for (const r of results) {
-    if (r.status === "error") errors.push({ id: r.id, message: r.error });
+    if (r.status === "error") issues.push({ id: r.id, message: r.error });
     if (r.reason || !Array.isArray(r.issues)) continue;
     for (const issue of r.issues) {
-      if (issue.severity === "error") errors.push({ id: r.id, message: issue.message, ...(issue.code ? { code: issue.code } : {}), ...(issue.target ? { target: issue.target } : {}) });
+      if (severities.includes(issue.severity)) {
+        issues.push({ id: r.id, message: issue.message, ...(issue.code ? { code: issue.code } : {}), ...(issue.target ? { target: issue.target } : {}) });
+      }
     }
   }
-  return errors;
+  return issues;
+}
+
+// The errors that decide the exit code by default, and the ones --baseline compares
+export const blockingErrors = (results) => failing(results, ["error"]);
+
+// What --fail-on takes: the issues that make the command exit with 1. error, the default, is the errors
+// above; warning adds the warnings of the steps; never leaves the exit code to the budgets,
+// --fail-on-regression and the runs that fail.
+export const FAIL_ON = ["error", "warning", "never"];
+
+export function parseFailOn(value) {
+  if (value === undefined) return "error";
+  if (FAIL_ON.includes(value)) return value;
+  throw new Error(`--fail-on must be error, warning or never, not "${value}"`);
+}
+
+export function failingIssues(results, failOn = "error") {
+  if (failOn === "never") return [];
+  return failing(results, failOn === "warning" ? ["error", "warning"] : ["error"]);
 }

package/src/interactions.js

@@ -1,18 +1,23 @@
-import { readFileSync } from "node:fs";
+import { readFileSync, existsSync } from "node:fs";
+
+// How long a step waits for its element. Playwright waits 30 s, which is a long silence for a selector
+// with a typo; the pages the MCP server keeps open wait the same 5 s (mcp/sessions.js)
+export const STEP_TIMEOUT_MS = 5000;
 
 export async function executeStep(page, step) {
+  const timeout = { timeout: STEP_TIMEOUT_MS };
   switch (step.action) {
     case "scroll":
       await page.evaluate((y) => window.scrollBy(0, y), step.y ?? 300);
       break;
     case "click":
-      await page.click(step.selector);
+      await page.click(step.selector, timeout);
       break;
     case "hover":
-      await page.hover(step.selector);
+      await page.hover(step.selector, timeout);
       break;
     case "type":
-      await page.type(step.selector, step.text);
+      await page.type(step.selector, step.text, timeout);
       break;
     case "wait":
       await page.waitForTimeout(step.ms);
@@ -27,6 +32,52 @@ export function loadInteractionSteps(source) {
   return Array.isArray(source) ? source : JSON.parse(readFileSync(source, "utf8")).interactions;
 }
 
+const ACTIONS = "click, hover, type, scroll or wait";
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+// What each action needs, as the schema of the steps of the MCP server (mcp/steps.js), which a test
+// keeps in agreement with this
+const STEP_PROBLEM = {
+  click: (step) => typeof step.selector !== "string" && 'needs a "selector", the CSS selector of its element',
+  hover: (step) => STEP_PROBLEM.click(step),
+  type: (step) => STEP_PROBLEM.click(step) || (typeof step.text !== "string" && 'needs a "text", the text to type'),
+  scroll: (step) => step.y !== undefined && !Number.isInteger(step.y) && 'takes "y" as a whole number of pixels',
+  wait: (step) => !(Number.isInteger(step.ms) && step.ms >= 0) && 'needs "ms", a whole number of milliseconds of at least 0',
+};
+
+// What is wrong in a list of steps, or null: the first problem, naming its step from 1
+export function stepsProblem(steps) {
+  if (!Array.isArray(steps)) return 'it needs a list of steps, as { "interactions": [{ "action": "click", "selector": "#buy" }] }';
+  if (steps.length === 0) return "it has no steps";
+  for (const [i, step] of steps.entries()) {
+    const name = `step ${i + 1}`;
+    if (!isObject(step)) return `${name} is not an object, such as { "action": "click", "selector": "#buy" }`;
+    if (step.action === undefined) return `${name} has no action. Use ${ACTIONS}`;
+    if (!Object.hasOwn(STEP_PROBLEM, step.action)) return `${name} has the unknown action ${JSON.stringify(step.action)}. Use ${ACTIONS}`;
+    const problem = STEP_PROBLEM[step.action](step);
+    if (problem) return `${name} (${step.action}) ${problem}`;
+  }
+  return null;
+}
+
+// The steps of the file of --interact-script, checked: an error here is one of usage, known before
+// the browser opens, and its message names the file and the step
+export function readInteractionScript(path) {
+  if (!existsSync(path)) throw new Error(`Interaction script not found: ${path}`);
+  let content;
+  try {
+    content = JSON.parse(readFileSync(path, "utf8"));
+  } catch (err) {
+    throw new Error(`Invalid --interact-script ${path}: it is not valid JSON (${err.message})`);
+  }
+  const problem = stepsProblem(isObject(content) ? content.interactions : undefined);
+  if (problem) throw new Error(`Invalid --interact-script ${path}: ${problem}`);
+  return content.interactions;
+}
+
+// A step that the page did not take: an element that is not there, hidden or covered. The page is the
+// cause, not the CLI, so the report is its message, without a stack
+export class InteractionError extends Error {}
+
 // A click or a key press creates the events that Interactions and Input-Latency-Breakdown measure;
 // a scroll, a hover and a wait do not.
 export function isInputStep(step) {
@@ -34,8 +85,14 @@ export function isInputStep(step) {
 }
 
 export async function runInteractions(page, source) {
-  for (const step of loadInteractionSteps(source)) {
-    await executeStep(page, step);
+  for (const [i, step] of loadInteractionSteps(source).entries()) {
+    try {
+      await executeStep(page, step);
+    } catch (err) {
+      const target = step.selector === undefined ? step.action : `${step.action} ${JSON.stringify(step.selector)}`;
+      // The first line: Playwright adds its call log after it
+      throw new InteractionError(`Step ${i + 1} of the interactions (${target}) failed: ${err.message.split("\n")[0]}`, { cause: err });
+    }
   }
 }