webperf-snippets 1.3.0 → 1.5.0

package/src/reporters/human.js

@@ -3,6 +3,7 @@ import { fieldDataView } from "../crux.js";
 import { describeThrottling } from "../throttling.js";
 import { describeRuns } from "../runs.js";
 import { matrixView } from "./matrix-view.js";
+import { baselineView } from "./baseline-view.js";
 
 const RATING_ICON = {
   good: "🟢",
@@ -215,7 +216,26 @@ function renderRuns(runs) {
   return lines;
 }
 
-export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, title = `WebPerf Snippets — ${url}` }) {
+const VERDICT_STYLE = { regressed: "red", improved: "green", unchanged: "dim", "not measured": "dim", new: "cyan" };
+
+// The comparison with --baseline, after the report: its warnings first, since they say how far to
+// trust the table, then a row per metric and the errors that appeared or went away
+function renderBaseline(comparison) {
+  const view = baselineView(comparison);
+  const lines = ["", styleText("bold", `Compared with ${view.file} (${view.summary})`)];
+  for (const warning of view.warnings) lines.push(`  ${styleText("yellow", `Warning: ${warning}`)}`);
+  lines.push(styleText("dim", `  ${pad("Metric", 9)}${pad("Baseline", 11)}${pad("Now", 10)}Change`));
+  for (const row of view.rows) {
+    const value = (v) => (v == null ? "·" : formatValue(v, row.unit));
+    const verdict = row.note ? `${row.verdict} (${row.note})` : row.verdict;
+    lines.push(`  ${pad(row.metric, 9)}${pad(value(row.baseline), 11)}${pad(value(row.current), 10)}${pad(row.change ?? "", 20)}${styleText(VERDICT_STYLE[row.verdict], verdict)}`);
+  }
+  for (const e of view.newErrors) lines.push(`  ${styleText("red", "+ new error")}  ${e.id}: ${e.message}`);
+  for (const e of view.fixedErrors) lines.push(`  ${styleText("green", "− fixed")}      ${e.id}: ${e.message}`);
+  return lines;
+}
+
+export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, throttling, runs, baseline, title = `WebPerf Snippets — ${url}` }) {
   const lines = [];
   lines.push(styleText(["bold"], title));
   lines.push(styleText("dim", `Navigated in ${navMs}ms`));
@@ -232,13 +252,14 @@ export function reportHuman({ url, navMs, results, pageErrors, verbose, crux, th
     lines.push(styleText("yellow", "Page errors during run:"));
     for (const err of pageErrors) lines.push(`  ${styleText("dim", err)}`);
   }
+  if (baseline) lines.push(...renderBaseline(baseline));
   return lines.join("\n");
 }
 
 // One row per combination with the rated median of each metric, and below it the errors that decide
 // the exit code. With verbose, the report of the median run of each combination follows.
 export function reportMatrixHuman(report) {
-  const { file, summary, hasVariants, metrics, rows, errors } = matrixView(report);
+  const { file, summary, hasVariants, metrics, rows, errors, comparison } = matrixView(report);
   const columns = [["name", "Page"], ["device", "Device"], ...(hasVariants ? [["variant", "Variant"]] : [])];
   const widths = columns.map(([key, label]) => Math.max(...report.combinations.map((c) => c[key].length), label.length) + 3);
   const identity = (values) => values.map((value, i) => pad(value, widths[i])).join("");
@@ -246,14 +267,19 @@ export function reportMatrixHuman(report) {
   const lines = [];
   lines.push(styleText(["bold"], `WebPerf Snippets — ${file} (${summary})`));
   lines.push("");
-  lines.push(styleText("dim", `  ${identity(columns.map(([, label]) => label))}${metrics.map((m) => pad(m, 14)).join("").trimEnd()}`));
-  for (const { combination: c, cells } of rows) {
+  // With --baseline, a change column follows each metric
+  const headings = metrics.map((m) => pad(m, 14) + (comparison ? pad("Δ", 12) : "")).join("");
+  lines.push(styleText("dim", `  ${identity(columns.map(([, label]) => label))}${headings.trimEnd()}`));
+  for (const { combination: c, cells, changes } of rows) {
     const row = `  ${identity(columns.map(([key]) => c[key]))}`;
     if (!cells) {
       lines.push(row + styleText("red", "✗ every run failed"));
       continue;
     }
-    const drawn = cells.map((cell) => (cell ? pad(paint(cell.rating, `${RATING_ICON[cell.rating] ?? "·"} ${formatValue(cell.median, cell.unit)}`), 14) : pad("·", 14)));
+    const drawn = cells.map((cell, i) => {
+      const value = cell ? pad(paint(cell.rating, `${RATING_ICON[cell.rating] ?? "·"} ${formatValue(cell.median, cell.unit)}`), 14) : pad("·", 14);
+      return value + (changes ? pad(styleText(VERDICT_STYLE[changes[i].verdict], changes[i].text), 12) : "");
+    });
     lines.push(row + drawn.join("").trimEnd());
   }
 
@@ -263,6 +289,18 @@ export function reportMatrixHuman(report) {
     for (const e of errors) lines.push(`  ${styleText("red", "✗")} ${e}`);
   }
 
+  if (comparison) {
+    lines.push("", styleText("bold", `Compared with ${comparison.file}`));
+    for (const warning of comparison.warnings) lines.push(`  ${styleText("yellow", `Warning: ${warning}`)}`);
+    for (const e of comparison.newErrors) lines.push(`  ${styleText("red", "+ new error")}  ${e}`);
+    for (const e of comparison.fixedErrors) lines.push(`  ${styleText("green", "− fixed")}      ${e}`);
+    if (comparison.added.length) lines.push(`  Added: ${comparison.added.join(", ")}`);
+    if (comparison.missing.length) lines.push(`  Missing: ${comparison.missing.join(", ")}`);
+    if (!comparison.warnings.length && !comparison.newErrors.length && !comparison.fixedErrors.length && !comparison.added.length && !comparison.missing.length) {
+      lines.push(styleText("dim", "  The same combinations, no new or fixed errors, and no warnings"));
+    }
+  }
+
   if (report.verbose) {
     for (const c of report.combinations.filter((c) => !c.error)) {
       lines.push("", reportHuman({ ...c, verbose: true, title: `${c.id} — ${c.url}` }));

package/src/reporters/markdown.js

@@ -2,6 +2,7 @@ import { fieldDataView } from "../crux.js";
 import { describeThrottling } from "../throttling.js";
 import { describeRuns } from "../runs.js";
 import { matrixView } from "./matrix-view.js";
+import { baselineView } from "./baseline-view.js";
 
 const RATING_ICON = {
   good: "✅",
@@ -83,8 +84,27 @@ function renderRuns(runs) {
   return lines;
 }
 
+const VERDICT_ICON = { regressed: "❌", improved: "✅", unchanged: "➖", "not measured": "·" };
+
+// The comparison with --baseline. The warnings go in the comment itself: whoever reads the pull
+// request does not see the stderr of the CI.
+function renderBaseline(comparison) {
+  const view = baselineView(comparison);
+  const lines = ["", `### Compared with \`${view.file}\``, `> ${view.summary}`];
+  for (const warning of view.warnings) lines.push(">", `> ⚠️ ${warning}`);
+  lines.push("", "| Metric | Baseline | Now | Change | Verdict |", "|--------|----------|-----|--------|---------|");
+  for (const row of view.rows) {
+    const value = (v) => (v == null ? "·" : formatValue(v, row.unit));
+    const verdict = `${VERDICT_ICON[row.verdict]} ${row.verdict}${row.note ? ` (${row.note})` : ""}`;
+    lines.push(`| ${row.metric} | ${value(row.baseline)} | ${value(row.current)} | ${row.change ?? "·"} | ${verdict} |`);
+  }
+  if (view.newErrors.length) lines.push("", "#### New errors", ...view.newErrors.map((e) => `- \`${e.id}\`: ${e.message}`));
+  if (view.fixedErrors.length) lines.push("", "#### Fixed errors", ...view.fixedErrors.map((e) => `- \`${e.id}\`: ${e.message}`));
+  return lines;
+}
+
 // With a null title, the report has no heading, as inside the details block of a matrix
-export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, title = `WebPerf Results — ${url}` }) {
+export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttling, runs, baseline, title = `WebPerf Results — ${url}` }) {
   const lines = [];
   if (title) lines.push(`## ${title}`);
   lines.push(`> Navigated in ${navMs}ms`);
@@ -120,6 +140,8 @@ export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttli
     for (const err of pageErrors) lines.push(`- ${err}`);
   }
 
+  if (baseline) lines.push(...renderBaseline(baseline));
+
   return lines.join("\n");
 }
 
@@ -127,17 +149,22 @@ export function reportMarkdown({ url, navMs, results, pageErrors, crux, throttli
 // report of the median run of each combination folded into a details block, to keep a pull request
 // comment readable.
 export function reportMatrixMarkdown(report) {
-  const { file, summary, hasVariants, metrics, rows, errors } = matrixView(report);
-  const headings = ["Page", "Device", ...(hasVariants ? ["Variant"] : []), ...metrics];
+  const { file, summary, hasVariants, metrics, rows, errors, comparison } = matrixView(report);
+  // With --baseline, a change column follows each metric
+  const metricHeadings = metrics.flatMap((m) => (comparison ? [m, "Δ"] : [m]));
+  const headings = ["Page", "Device", ...(hasVariants ? ["Variant"] : []), ...metricHeadings];
   const row = (parts) => `|${parts.map((part) => (part ? ` ${part} ` : " ")).join("|")}|`;
 
   const lines = [`## WebPerf Results — ${file}`, `> ${summary}`, ""];
   lines.push(row(headings));
   lines.push(`|${headings.map((h) => "-".repeat(h.length + 2)).join("|")}|`);
-  for (const { combination: c, cells } of rows) {
+  for (const { combination: c, cells, changes } of rows) {
     const drawn = cells
-      ? cells.map((cell) => (cell ? `${RATING_ICON[cell.rating] ?? "·"} ${formatValue(cell.median, cell.unit)}` : "·"))
-      : ["❌ every run failed", ...metrics.slice(1).map(() => "")];
+      ? cells.flatMap((cell, i) => {
+          const value = cell ? `${RATING_ICON[cell.rating] ?? "·"} ${formatValue(cell.median, cell.unit)}` : "·";
+          return changes ? [value, changes[i].text] : [value];
+        })
+      : ["❌ every run failed", ...metricHeadings.slice(1).map(() => "")];
     lines.push(row([c.name, c.device, ...(hasVariants ? [c.variant] : []), ...drawn]));
   }
 
@@ -145,6 +172,19 @@ export function reportMatrixMarkdown(report) {
     lines.push("", "### Errors", ...errors.map((e) => `- ${e}`));
   }
 
+  if (comparison) {
+    lines.push("", `### Compared with \`${comparison.file}\``);
+    if (comparison.warnings.length) lines.push(...comparison.warnings.flatMap((w, i) => [...(i ? [">"] : []), `> ⚠️ ${w}`]), "");
+    const ids = (list) => list.map((id) => `\`${id}\``).join(", ");
+    const items = [
+      ...comparison.newErrors.map((e) => `- New error: ${e}`),
+      ...comparison.fixedErrors.map((e) => `- Fixed: ${e}`),
+      ...(comparison.added.length ? [`- Added: ${ids(comparison.added)}`] : []),
+      ...(comparison.missing.length ? [`- Missing: ${ids(comparison.missing)}`] : []),
+    ];
+    lines.push(...(items.length ? items : ["The same combinations, no new or fixed errors."]));
+  }
+
   for (const c of report.combinations.filter((c) => !c.error)) {
     lines.push("", `<details><summary>${c.id}</summary>`, "", reportMarkdown({ ...c, title: null }), "", "</details>");
   }

package/src/reporters/matrix-view.js

@@ -1,14 +1,16 @@
 import { basename } from "node:path";
 import { rate } from "../ratings.js";
 import { blockingErrors } from "../blocking.js";
+import { changeCell, matrixComparisonView } from "./baseline-view.js";
 
 // The order of the metric columns of the matrix table
 const METRICS = ["LCP", "FCP", "TTFB", "CLS", "INP"];
 
 // What the human and markdown reports of a matrix show, so they only differ in how they draw it:
 // the title, the metric columns, one row per combination with the rated median of each metric, and
-// the errors that decide the exit code.
-export function matrixView({ matrix, runs, combinations }) {
+// the errors that decide the exit code. With --baseline, each row also has the change of each metric
+// ("new" for a combination the baseline did not have), and the comparison goes below the table.
+export function matrixView({ matrix, runs, combinations, baseline }) {
   const measured = combinations.filter((c) => !c.error);
   const medianBys = [...new Set(measured.map((c) => c.runs.medianBy))];
   const metrics = METRICS.filter((metric) => measured.some((c) => c.runs.summary[metric]));
@@ -22,6 +24,9 @@ export function matrixView({ matrix, runs, combinations }) {
     metrics,
     rows: combinations.map((c) => ({
       combination: c,
+      ...(baseline
+        ? { changes: metrics.map((metric) => (!c.baseline ? { text: "new", verdict: "new" } : c.baseline.metrics[metric] ? changeCell(metric, c.baseline.metrics[metric]) : { text: "·", verdict: "not measured" })) }
+        : {}),
       cells: c.error
         ? null
         : metrics.map((metric) => {
@@ -29,6 +34,7 @@ export function matrixView({ matrix, runs, combinations }) {
             return value ? { median: value.median, unit: value.unit, rating: rate(metric, value.median) } : null;
           }),
     })),
+    ...(baseline ? { comparison: matrixComparisonView(baseline, combinations) } : {}),
     errors: combinations.flatMap((c) =>
       c.error ? [`${c.id}: ${c.error}`] : blockingErrors(c.results).map((e) => `${c.id}: ${e.id}: ${e.message}`)
     ),