webperf-snippets 1.3.0 → 1.5.0

package/src/baseline.js

@@ -0,0 +1,225 @@
+import { readFileSync } from "node:fs";
+import { blockingErrors } from "./blocking.js";
+import { describeThrottling } from "./throttling.js";
+
+// A measurement compared with an earlier one, its baseline: which metrics got worse or better beyond
+// the noise of the lab, which errors appeared or went away, and what differs in how both were
+// measured. Pure functions over the JSON the CLI prints with --json, so a baseline is any earlier
+// output saved with --json > baseline.json.
+
+const METRICS = ["LCP", "FCP", "TTFB", "CLS", "INP"];
+
+// A change counts when it beats the larger of a share of the baseline and a minimum: 10% of a 3 s LCP
+// is 300 ms, and 10% of a 200 ms TTFB would be noise. CLS is a score, with no relative part.
+export const DEFAULT_THRESHOLDS = {
+  LCP: { relative: 0.1, absolute: 100 },
+  FCP: { relative: 0.1, absolute: 100 },
+  TTFB: { relative: 0.1, absolute: 100 },
+  INP: { relative: 0.1, absolute: 50 },
+  CLS: { absolute: 0.05 },
+};
+
+// The baseline in a file, as { kind: "page" | "matrix", file, data }. Throws with what to do when
+// the file is not an output of the CLI.
+export function readBaseline(text, file) {
+  const howTo = `save one with webperf-snippets <url> --json > ${file}`;
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch {
+    throw new Error(`${file} is not JSON: ${howTo}`);
+  }
+  if (Array.isArray(data?.combinations)) return { kind: "matrix", file, data };
+  if (typeof data?.url === "string" && Array.isArray(data.results)) return { kind: "page", file, data };
+  throw new Error(`${file} is not an output of webperf-snippets --json: it has neither the results of a page nor the combinations of a matrix; ${howTo}`);
+}
+
+export function loadBaseline(path) {
+  let text;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch (err) {
+    throw new Error(`Cannot read the baseline ${path}: ${err.code === "ENOENT" ? "the file does not exist" : err.message}`);
+  }
+  return readBaseline(text, path);
+}
+
+// One part of the threshold of a metric, from --threshold: LCP=15% (relative), LCP=200ms
+// (absolute), CLS=0.1 (CLS has only an absolute part, with no unit)
+export function parseThreshold(raw) {
+  const match = /^([A-Za-z]+)=(\d+(?:\.\d+)?)(%|ms)?$/.exec(raw.trim());
+  if (!match) throw new Error(`Invalid --threshold "${raw}": give METRIC=VALUE with its unit, such as LCP=15%, LCP=200ms or CLS=0.1`);
+  const [, name, number, unit] = match;
+  const metric = name.toUpperCase();
+  if (!METRICS.includes(metric)) throw new Error(`Unknown metric "${name}" in --threshold "${raw}". Choose from: ${METRICS.join(", ")}`);
+  const value = Number(number);
+  if (metric === "CLS") {
+    if (unit) throw new Error(`Invalid --threshold "${raw}": CLS is a score, with no unit and no relative part, such as CLS=0.1`);
+    return { metric, absolute: value };
+  }
+  if (unit === "%") return { metric, relative: value / 100 };
+  if (unit === "ms") return { metric, absolute: value };
+  throw new Error(`Invalid --threshold "${raw}": give the unit, such as ${metric}=15% or ${metric}=200ms`);
+}
+
+// The default thresholds with the parts that --threshold changes
+export function thresholdsWith(overrides) {
+  const thresholds = Object.fromEntries(Object.entries(DEFAULT_THRESHOLDS).map(([metric, parts]) => [metric, { ...parts }]));
+  for (const { metric, ...part } of overrides) Object.assign(thresholds[metric], part);
+  return thresholds;
+}
+
+const round = (n) => Math.round(n * 10000) / 10000;
+
+// The value of a metric in a page: the median of its runs, with their range when there are two or
+// more, or its one result, as runs.js picks it
+function measured(payload, metric) {
+  const summary = payload.runs?.summary?.[metric];
+  if (summary) {
+    const runs = summary.values.filter((value) => value !== null).length;
+    return { value: summary.median, ...(runs >= 2 ? { range: { min: summary.min, max: summary.max } } : {}) };
+  }
+  const result = payload.results?.find((r) => r.metric === metric && r.status === "ok" && typeof r.value === "number");
+  return result ? { value: result.value } : null;
+}
+
+// A regression must beat the threshold of its metric and, when the baseline has several runs, the
+// worst of them: a median inside the range of the baseline is noise the baseline already had.
+function verdict(base, current, threshold) {
+  if (current.value - base.value > threshold && (!base.range || current.value > base.range.max)) return "regressed";
+  if (base.value - current.value > threshold && (!base.range || current.value < base.range.min)) return "improved";
+  return "unchanged";
+}
+
+function compareMetrics(base, current, thresholds) {
+  const metrics = {};
+  for (const metric of METRICS) {
+    const b = measured(base, metric);
+    const c = measured(current, metric);
+    if (!b && !c) continue;
+    if (!b || !c) {
+      metrics[metric] = { ...(b ? { baseline: b.value } : { current: c.value }), verdict: "not measured" };
+      continue;
+    }
+    const { relative = 0, absolute } = thresholds[metric];
+    const threshold = round(Math.max(relative * b.value, absolute));
+    const delta = c.value - b.value;
+    metrics[metric] = {
+      baseline: b.value,
+      current: c.value,
+      delta: round(delta),
+      deltaPercent: b.value === 0 ? null : Math.round((delta / b.value) * 1000) / 10,
+      threshold,
+      verdict: verdict(b, c, threshold),
+    };
+  }
+  return metrics;
+}
+
+// A resource of the page itself by its path and query, so the review app of a PR, on another host,
+// matches main; any other target as it is
+function targetOf(target, pageUrl) {
+  if (!target) return "";
+  try {
+    const url = new URL(target);
+    return url.origin === new URL(pageUrl).origin ? url.pathname + url.search : url.href;
+  } catch {
+    return target;
+  }
+}
+
+// Two errors are the same when they share a code and a target or, when a snippet has no codes on one
+// side (a baseline of 1.4.0), when their messages differ only in their figures: "Render-blocking CSS
+// (69.21 KB)…" and "Render-blocking CSS (71.80 KB)…"
+const byMessage = (error) => `${error.id}\u0000${error.message.replace(/\d+(?:[.,]\d+)*/g, "#")}`;
+
+function compareErrors(base, current) {
+  const baseErrors = blockingErrors(base.results ?? []);
+  const currentErrors = blockingErrors(current.results ?? []);
+  // The snippets whose errors on one side all lack a code pair by message on both
+  const uncoded = (errors) => {
+    const ids = new Set(errors.map((e) => e.id));
+    for (const e of errors) if (e.code) ids.delete(e.id);
+    return ids;
+  };
+  const byMessageOnly = new Set([...uncoded(baseErrors), ...uncoded(currentErrors)]);
+  const keyOf = (error, pageUrl) =>
+    error.code && !byMessageOnly.has(error.id) ? `${error.id}\u0000${error.code}\u0000${targetOf(error.target, pageUrl)}` : byMessage(error);
+  const byKey = (errors, pageUrl) => new Map(errors.map((error) => [keyOf(error, pageUrl), error]));
+  const b = byKey(baseErrors, base.url);
+  const c = byKey(currentErrors, current.url);
+  return {
+    new: [...c].filter(([key]) => !b.has(key)).map(([, error]) => error),
+    fixed: [...b].filter(([key]) => !c.has(key)).map(([, error]) => error),
+  };
+}
+
+const describeWork = (conditions) => (conditions.snippet ? `snippet ${conditions.snippet}` : `workflow ${conditions.workflow}`);
+const describeLoad = (throttling) => (throttling ? `with ${describeThrottling(throttling)}` : "without throttling");
+const runCount = (payload) => payload.runs?.count ?? 1;
+const pathOf = (url) => {
+  try {
+    const { pathname, search } = new URL(url);
+    return pathname + search;
+  } catch {
+    return url;
+  }
+};
+
+// What differs in how both were measured. The comparison goes on: the warnings say how far to trust it.
+function warningsOf(base, current) {
+  const warnings = [];
+  const b = base.conditions;
+  const c = current.conditions;
+  if (!b) {
+    warnings.push("the baseline comes from a CLI without conditions (1.3.0 or earlier); it cannot say how it was measured");
+  } else if (c) {
+    if (describeWork(b) !== describeWork(c)) warnings.push(`the baseline was measured with ${describeWork(b)}, and this run with ${describeWork(c)}`);
+    if (b.viewport !== c.viewport) warnings.push(`the baseline was measured on ${b.viewport}, and this run on ${c.viewport}`);
+    if (b.waitMs !== c.waitMs) warnings.push(`the baseline waited ${b.waitMs} ms after the load, and this run ${c.waitMs} ms`);
+  }
+  if (describeLoad(base.throttling) !== describeLoad(current.throttling)) {
+    warnings.push(`the baseline was measured ${describeLoad(base.throttling)}, and this run ${describeLoad(current.throttling)}`);
+  }
+  if (runCount(base) < 2) warnings.push("the baseline has 1 run, so only the thresholds apply; measure both with --runs");
+  else if (runCount(base) !== runCount(current)) warnings.push(`the baseline has ${runCount(base)} runs, and this run ${runCount(current)}`);
+  if (b && c && b.cliVersion !== c.cliVersion) {
+    warnings.push(`the baseline was measured with CLI ${b.cliVersion}, and this run with ${c.cliVersion}; the snippets may have changed`);
+  }
+  // Another host is the review app of a pull request against production: the same page
+  if (pathOf(base.url) !== pathOf(current.url)) warnings.push(`the baseline measured ${pathOf(base.url)} and this run ${pathOf(current.url)}`);
+  return warnings;
+}
+
+// A page against its baseline, as the `baseline` field of its JSON. The runs of the baseline say how
+// much to trust its values, in the header of the reports.
+export function compareRun(base, current, { file, thresholds = DEFAULT_THRESHOLDS }) {
+  return {
+    file,
+    url: base.url,
+    runs: base.runs ? { count: base.runs.count, medianBy: base.runs.medianBy } : { count: 1 },
+    metrics: compareMetrics(base, current, thresholds),
+    errors: compareErrors(base, current),
+    warnings: warningsOf(base, current),
+  };
+}
+
+// A matrix against a baseline matrix: each combination with the one of the same id, and the ids that
+// only one side has
+export function compareMatrix(base, current, { file, thresholds = DEFAULT_THRESHOLDS }) {
+  const baseById = new Map(base.combinations.map((c) => [c.id, c]));
+  const currentIds = new Set(current.combinations.map((c) => c.id));
+  const combinations = {};
+  for (const c of current.combinations) {
+    if (baseById.has(c.id)) combinations[c.id] = compareRun(baseById.get(c.id), c, { file, thresholds });
+  }
+  return {
+    root: {
+      file,
+      added: current.combinations.filter((c) => !baseById.has(c.id)).map((c) => c.id),
+      missing: base.combinations.filter((c) => !currentIds.has(c.id)).map((c) => c.id),
+      warnings: [],
+    },
+    combinations,
+  };
+}

package/src/bin.js

@@ -18,6 +18,9 @@ import { parseOwnDomains } from "./own-domains.js";
 import { MEDIAN_METRICS, summarizeRuns } from "./runs.js";
 import { loadMatrix } from "./matrix.js";
 import { blockingErrors } from "./blocking.js";
+import { measurementConditions } from "./conditions.js";
+import { loadBaseline, parseThreshold, thresholdsWith, compareRun, compareMatrix } from "./baseline.js";
+import { describeRegressions, describeMatrixRegressions, matrixComparisonView } from "./reporters/baseline-view.js";
 
 
 const USAGE = `webperf-snippets <url> [options]
@@ -63,6 +66,13 @@ Options:
                             variant it lists, and report a table of the medians
   --median-by <metric>      Metric that picks the median run: LCP, FCP, TTFB, CLS, INP,
                             navMs (default: the first of LCP, FCP, TTFB, navMs measured)
+  --baseline <file>         Compare the page, or each combination of a matrix, with the --json
+                            output of an earlier run: which metrics got worse or better beyond
+                            the noise, and which errors are new or fixed. It only informs,
+                            unless --fail-on-regression
+  --threshold <metric>=<v>  The threshold of a metric for --baseline, such as LCP=15%,
+                            LCP=200ms or CLS=0.1. Repeat it for more metrics
+  --fail-on-regression      Exit 1 when a metric regressed or an error is new against --baseline
   --crux-key <key>          Add CrUX field data (p75) next to the measured values.
                             Also read from the CRUX_API_KEY environment variable
   --report-to <url>         POST the results to this https URL after the run
@@ -86,6 +96,9 @@ Examples:
   npx webperf-snippets https://example.com --workflow loading --own-domains cdn.example.net
   npx webperf-snippets https://example.com --runs 5 --budget-lcp 2500
   npx webperf-snippets --matrix matrix.json --runs 3 --budget-lcp 2500
+  npx webperf-snippets https://example.com --runs 5 --json > baseline.json
+  npx webperf-snippets https://pr-123.example.com --runs 5 --baseline baseline.json --fail-on-regression
+  npx webperf-snippets --matrix matrix.json --baseline last-week.json
   npx webperf-snippets https://example.com --snippet INP --interact-script interactions.json
   npx webperf-snippets https://example.com/dashboard --storage-state auth.json
   npx webperf-snippets https://review.example.com --header "CF-Access-Client-Id: $CF_ID" --header "CF-Access-Client-Secret: $CF_SECRET"
@@ -166,6 +179,9 @@ async function main() {
         matrix: { type: "string" },
         runs: { type: "string" },
         "median-by": { type: "string" },
+        baseline: { type: "string" },
+        threshold: { type: "string", multiple: true },
+        "fail-on-regression": { type: "boolean" },
         "crux-key": { type: "string" },
         "report-to": { type: "string" },
         "api-key": { type: "string" },
@@ -242,6 +258,23 @@ async function main() {
   }
   if (medianBy != null && runCount == null && !matrixPath) fail("--median-by needs --runs");
 
+  // The baseline is read and checked before the browser opens: a wrong file fails at once
+  let baseline;
+  let thresholds;
+  if (values.baseline) {
+    try {
+      baseline = loadBaseline(values.baseline);
+      thresholds = thresholdsWith((values.threshold ?? []).map(parseThreshold));
+    } catch (err) {
+      fail(err.message);
+    }
+    if (!matrixPath && baseline.kind === "matrix") fail(`${values.baseline} is the baseline of a matrix: compare it with --matrix`);
+    if (matrixPath && baseline.kind === "page") fail(`${values.baseline} is the baseline of a page: compare it with a URL, not --matrix`);
+  } else {
+    if (values.threshold) fail("--threshold needs --baseline");
+    if (values["fail-on-regression"]) fail("--fail-on-regression needs --baseline");
+  }
+
   // What each page load runs, a snippet or a workflow, under the conditions of the load
   const items = values.snippet ? buildSnippetItem(values) : null;
   const workflow = values.snippet ? null : WORKFLOWS[values.workflow ?? "core-web-vitals"];
@@ -250,6 +283,9 @@ async function main() {
     const options = { waitMs, headless: !values.headed, interactScript, ...conditions };
     return items ? runSnippets({ ...options, items }) : runMeasurement({ ...options, workflow, rules: RULES });
   };
+  // What the JSON records of how each page was measured, for a later run to compare with
+  const conditionsFor = (viewport) =>
+    measurementConditions({ viewport, waitMs, ...(items ? { snippet: items[0].path } : { workflow: values.workflow ?? "core-web-vitals" }) });
 
   if (matrixPath) {
     let matrix;
@@ -258,7 +294,7 @@ async function main() {
     } catch (err) {
       fail(err.message);
     }
-    await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl });
+    await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds });
     return;
   }
 
@@ -271,11 +307,17 @@ async function main() {
     if (!median.payload) throw new Error(`Every run failed: ${median.errors.join("; ")}`);
     payload = { ...median.payload, runs: median.runs };
   }
+  payload.conditions = conditionsFor(viewportName);
 
   const cruxKey = values["crux-key"] || process.env.CRUX_API_KEY;
   const crux = cruxKey ? await fetchCrux({ url, apiKey: cruxKey, formFactor: formFactorFor(viewportName) }) : undefined;
   if (crux) payload.crux = crux;
 
+  if (baseline) {
+    payload.baseline = compareRun(baseline.data, payload, { file: baseline.file, thresholds });
+    for (const warning of payload.baseline.warnings) process.stderr.write(`Warning: ${warning}\n`);
+  }
+
   let output;
   if (values.markdown) {
     output = reportMarkdown(payload);
@@ -315,6 +357,12 @@ async function main() {
     }
     process.exit(1);
   }
+  // Without --fail-on-regression the comparison only informs, and the exit code is the one of today
+  const regressions = values["fail-on-regression"] ? describeRegressions(payload.baseline) : [];
+  if (regressions.length > 0) {
+    for (const r of regressions) process.stderr.write(`Regression against ${baseline.file}: ${r}\n`);
+    process.exit(1);
+  }
   process.exit(blockingErrors(payload.results).length > 0 ? 1 : 0);
 }
 
@@ -337,7 +385,7 @@ async function measureRuns(measure, count, medianBy, progress) {
 
 // Every combination of the matrix, in order, each one `matrix.runs` times. Each combination reports
 // its median run, and the exit code follows the same rules as one page, for every combination.
-async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl }) {
+async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds }) {
   for (const warning of matrix.warnings) process.stderr.write(`Warning: ${warning}\n`);
   const viewportOf = new Map(matrix.combinations.map((c) => [c.id, c.viewport]));
 
@@ -351,8 +399,8 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
     const identity = { id: c.id, name: c.name, device: c.device, ...(c.variant ? { variant: c.variant } : {}), url: c.url };
     combinations.push(
       median.payload
-        ? { ...identity, ...median.payload, runs: median.runs }
-        : { ...identity, status: "error", error: `Every run failed: ${median.errors.join("; ")}`, runs: { count: matrix.runs, failed: matrix.runs } }
+        ? { ...identity, ...median.payload, runs: median.runs, conditions: conditionsFor(c.viewport) }
+        : { ...identity, status: "error", error: `Every run failed: ${median.errors.join("; ")}`, runs: { count: matrix.runs, failed: matrix.runs }, conditions: conditionsFor(c.viewport) }
     );
   }
 
@@ -369,8 +417,18 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
     }
   }
 
-  const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose };
-  const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, combinations }) : reportMatrixHuman(report);
+  // With --baseline, each combination is compared with the one of the same id in the baseline
+  let comparison;
+  if (baseline) {
+    const { root, combinations: byId } = compareMatrix(baseline.data, { combinations }, { file: baseline.file, thresholds });
+    for (const c of combinations) if (byId[c.id]) c.baseline = byId[c.id];
+    comparison = root;
+    for (const warning of matrixComparisonView(root, combinations).warnings) process.stderr.write(`Warning: ${warning}\n`);
+  }
+
+  const withBaseline = comparison ? { baseline: comparison } : {};
+  const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose, ...withBaseline };
+  const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report);
   process.stdout.write(output + "\n");
 
   // One report for each combination, with the body of one page and the device, variant and runs of
@@ -411,7 +469,10 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
     for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
   }
   const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0);
-  process.exit(violations.length > 0 || failed ? 1 : 0);
+  // Without --fail-on-regression the comparison only informs
+  const regressions = values["fail-on-regression"] ? describeMatrixRegressions(combinations) : [];
+  for (const r of regressions) process.stderr.write(`Regression against ${baseline.file} in ${r}\n`);
+  process.exit(violations.length > 0 || failed || regressions.length > 0 ? 1 : 0);
 }
 
 main().catch((err) => {

package/src/blocking.js

@@ -6,7 +6,9 @@ export function blockingErrors(results) {
   for (const r of results) {
     if (r.status === "error") errors.push({ id: r.id, message: r.error });
     if (r.reason || !Array.isArray(r.issues)) continue;
-    for (const issue of r.issues) if (issue.severity === "error") errors.push({ id: r.id, message: issue.message });
+    for (const issue of r.issues) {
+      if (issue.severity === "error") errors.push({ id: r.id, message: issue.message, ...(issue.code ? { code: issue.code } : {}), ...(issue.target ? { target: issue.target } : {}) });
+    }
   }
   return errors;
 }

package/src/conditions.js

@@ -0,0 +1,7 @@
+import { cliVersion } from "./report-to.js";
+
+// How a measurement was taken, kept in its JSON so that a later run compared with it can tell whether
+// both were measured the same way. A snippet is named by its path, so an alias and its path match.
+export function measurementConditions({ viewport, workflow, snippet, waitMs }) {
+  return { viewport, ...(snippet ? { snippet } : { workflow }), waitMs, cliVersion };
+}

package/src/mcp/documents.js

@@ -19,30 +19,50 @@ function readDocument(relativePath) {
 }
 
 // Splits SCHEMA.md into the general contract and the section of each snippet ("#### LCP" up to the next
-// heading of that level or above). The general contract is everything outside "## Script-Specific Schemas".
+// heading of that level or above). The general contract is everything outside "## Script-Specific Schemas"
+// and outside the tables of "## Issue codes", whose rules it keeps. The table of a snippet ("### LCP"
+// under Issue codes) goes with its section, or makes one when the snippet has none.
 export function splitSchema(markdown) {
   const lines = markdown.split("\n");
-  const start = lines.findIndex((l) => l.startsWith("## Script-Specific Schemas"));
-  const end = lines.findIndex((l, i) => i > start && l.startsWith("## "));
-  const general = [...lines.slice(0, start), ...lines.slice(end)].join("\n");
+  const sectionOf = (heading) => {
+    const start = lines.findIndex((l) => l.startsWith(heading));
+    if (start === -1) return [lines.length, lines.length];
+    const next = lines.findIndex((l, i) => i > start && l.startsWith("## "));
+    return [start, next === -1 ? lines.length : next];
+  };
+  const [schemasStart, schemasEnd] = sectionOf("## Script-Specific Schemas");
+  const [codesStart, codesEnd] = sectionOf("## Issue codes");
+  const firstTable = lines.findIndex((l, i) => i > codesStart && i < codesEnd && l.startsWith("### "));
+  const tablesStart = firstTable === -1 ? codesEnd : firstTable;
+  const general = lines
+    .filter((_, i) => !(i >= schemasStart && i < schemasEnd) && !(i >= tablesStart && i < codesEnd))
+    .join("\n");
 
   const sections = new Map();
-  let name = null;
-  let current = [];
-  const flush = () => name && sections.set(name, current.join("\n").trim());
-  for (const line of lines.slice(start, end)) {
-    if (line.startsWith("#### ")) {
-      flush();
-      name = line.slice(5).split(" ")[0];
-      current = [line];
-    } else if (/^#{1,3} /.test(line)) {
-      flush();
-      name = null;
-    } else if (name) {
-      current.push(line);
+  const collect = (from, to, level, add) => {
+    let name = null;
+    let current = [];
+    const flush = () => name && add(name, current.join("\n").trim());
+    for (const line of lines.slice(from, to)) {
+      if (line.startsWith(`${level} `)) {
+        flush();
+        name = line.slice(level.length + 1).split(" ")[0];
+        current = [line];
+      } else if (/^#{1,4} /.test(line)) {
+        flush();
+        name = null;
+      } else if (name) {
+        current.push(line);
+      }
     }
-  }
-  flush();
+    flush();
+  };
+  collect(schemasStart, schemasEnd, "####", (name, text) => sections.set(name, text));
+  collect(tablesStart, codesEnd, "###", (name, text) => {
+    // The last table runs into the rule that closes the section
+    const table = `Issue codes:\n${text.split("\n").slice(1).join("\n").replace(/\n+---$/, "")}`;
+    sections.set(name, sections.has(name) ? `${sections.get(name)}\n\n${table}` : `#### ${name}\n\n${table}`);
+  });
   return { general, sections };
 }
 
@@ -65,7 +85,7 @@ export function registerResources(server) {
     "webperf://schema",
     {
       title: "Snippet result contract",
-      description: "What every snippet returns: the base shape, the statuses, the execution patterns and how to read a result. The fields of each snippet are in webperf://schema/{snippet}.",
+      description: "What every snippet returns: the base shape, the statuses, the execution patterns, how to read a result and the rules of the issue codes. The fields and the issue codes of each snippet are in webperf://schema/{snippet}.",
       mimeType: MARKDOWN,
     },
     async (uri) => markdown(uri, schemaSection())
@@ -80,7 +100,7 @@ export function registerResources(server) {
     }),
     {
       title: "Result of one snippet",
-      description: "The fields one snippet returns, with an example, from snippets/SCHEMA.md.",
+      description: "The fields one snippet returns, with an example, and the codes of its issues, from snippets/SCHEMA.md.",
       mimeType: MARKDOWN,
     },
     async (uri, { snippet }) => {

package/src/reporters/baseline-view.js

@@ -0,0 +1,83 @@
+// The comparison with a baseline as the human and markdown reports draw it: one row per metric with
+// its two values, the change and the verdict, and why an unchanged one did not count. The values stay
+// numbers with their unit, for each report to format.
+
+const MINUS = "−";
+const signed = (n, text) => (n > 0 ? `+${text}` : n < 0 ? `${MINUS}${text}` : text);
+const arrow = (delta) => (delta > 0 ? "▲" : delta < 0 ? "▼" : "=");
+const unitOf = (metric) => (metric === "CLS" ? "score" : "ms");
+
+// "▲ +380ms (+18%)", "▼ −90ms (−15%)", "▼ −0.0020"
+function describeChange(metric, { delta, deltaPercent }) {
+  if (metric === "CLS") return `${arrow(delta)} ${signed(delta, Math.abs(delta).toFixed(4))}`;
+  const ms = signed(delta, `${Math.abs(Math.round(delta))}ms`);
+  const percent = deltaPercent == null ? "" : ` (${signed(deltaPercent, `${Math.abs(Math.round(deltaPercent))}%`)})`;
+  return `${arrow(delta)} ${ms}${percent}`;
+}
+
+// An unchanged metric that moved stayed under its threshold, or beat it inside the range of the runs
+// of the baseline: noise the baseline already had
+function noteOf(metric, m) {
+  if (m.verdict !== "unchanged" || m.delta === 0) return null;
+  if (Math.abs(m.delta) > m.threshold) return "inside the range of the baseline";
+  return `under ${metric === "CLS" ? m.threshold : `${Math.round(m.threshold)}ms`}`;
+}
+
+export function baselineView({ file, url, runs, metrics, errors, warnings }) {
+  const summary = `${url}, ${runs.count === 1 ? "1 run" : `${runs.count} runs, median by ${runs.medianBy}`}`;
+  const rows = Object.entries(metrics).map(([metric, m]) => ({
+    metric,
+    unit: unitOf(metric),
+    baseline: m.baseline,
+    current: m.current,
+    change: m.verdict === "not measured" ? null : describeChange(metric, m),
+    verdict: m.verdict,
+    note: m.verdict === "not measured" ? null : noteOf(metric, m),
+  }));
+  return { file, summary, rows, newErrors: errors.new, fixedErrors: errors.fixed, warnings };
+}
+
+// What got worse, one line each, for --fail-on-regression: the metrics that regressed and the errors
+// that only this run has
+export function describeRegressions({ metrics, errors }) {
+  return [
+    ...Object.entries(metrics)
+      .filter(([, m]) => m.verdict === "regressed")
+      .map(([metric, m]) => `${metric} ${describeChange(metric, m)}`),
+    ...errors.new.map((e) => `new error ${e.id}: ${e.message}`),
+  ];
+}
+
+// The change of a metric in a cell of the matrix table: the figure only when it counts, so the table
+// reads at a glance. "▲ +380ms", "▼ −400ms", "=" for unchanged, "·" when one side did not measure it.
+export function changeCell(metric, m) {
+  if (m.verdict === "not measured") return { text: "·", verdict: m.verdict };
+  if (m.verdict === "unchanged") return { text: "=", verdict: m.verdict };
+  const amount = metric === "CLS" ? Math.abs(m.delta).toFixed(4) : `${Math.abs(Math.round(m.delta))}ms`;
+  return { text: `${arrow(m.delta)} ${signed(m.delta, amount)}`, verdict: m.verdict };
+}
+
+// The comparison of a matrix below its table. A warning that every compared combination has, such as
+// a baseline of one run, shows once; the others name their combinations.
+export function matrixComparisonView(root, combinations) {
+  const compared = combinations.filter((c) => c.baseline);
+  const idsByWarning = new Map();
+  for (const c of compared) {
+    for (const warning of c.baseline.warnings) idsByWarning.set(warning, [...(idsByWarning.get(warning) ?? []), c.id]);
+  }
+  const warnings = [...idsByWarning].map(([warning, ids]) => (ids.length === compared.length ? warning : `${ids.join(", ")}: ${warning}`));
+  const errorsOf = (kind) => compared.flatMap((c) => c.baseline.errors[kind].map((e) => `${c.id}: ${e.id}: ${e.message}`));
+  return {
+    file: root.file,
+    warnings: [...root.warnings, ...warnings],
+    newErrors: errorsOf("new"),
+    fixedErrors: errorsOf("fixed"),
+    added: root.added,
+    missing: root.missing,
+  };
+}
+
+// The regressions of every combination, each named by its combination
+export function describeMatrixRegressions(combinations) {
+  return combinations.filter((c) => c.baseline).flatMap((c) => describeRegressions(c.baseline).map((r) => `${c.id}: ${r}`));
+}