webperf-snippets 1.16.0 → 1.17.0

package/src/reporters/baseline-view.js

@@ -1,6 +1,8 @@
+import { inconclusiveForTheWorse } from "../baseline.js";
+
 // The comparison with a baseline as the human and markdown reports draw it: one row per metric with
-// its two values, the change and the verdict, and why an unchanged one did not count. The values stay
-// numbers with their unit, for each report to format.
+// its two values, the change and the verdict, why an unchanged one did not count and what to do with an
+// inconclusive one. The values stay numbers with their unit, for each report to format.
 
 const MINUS = "−";
 const signed = (n, text) => (n > 0 ? `+${text}` : n < 0 ? `${MINUS}${text}` : text);
@@ -16,14 +18,20 @@ function describeChange(metric, { delta, deltaPercent }) {
 }
 
 // Three decimals where the verdict is decided, two above it: "0.028", "0.35"
-const pValue = (p) => (p < 0.001 ? "< 0.001" : p < 0.05 ? p.toFixed(3) : p.toFixed(2));
+// Three decimals up to 0.1, so a p-value next to the 0.05 it is compared with is not rounded away from it
+const pValue = (p) => (p < 0.001 ? "< 0.001" : p < 0.1 ? p.toFixed(3) : p.toFixed(2));
 const amountOf = (metric, n) => (metric === "CLS" ? signed(n, Math.abs(n).toFixed(4)) : signed(n, `${Math.abs(Math.round(n))}ms`));
 
+// What a person does with an inconclusive metric: the runs can tell neither a change nor its absence
+const MEASURE_AGAIN = "measure again, or with more runs";
+
 // The evidence of a change measured with five runs a side: its p-value, its shift and the 95 % interval
-// of the shift; for a change over the threshold that stayed unchanged, that the runs gave no evidence
+// of the shift; for a change over the threshold that is inconclusive, that the runs gave no evidence and
+// what to do about it
 function evidenceOf(metric, m) {
   const { p, shift, ci } = m.stats;
-  if (m.verdict === "unchanged") return Math.abs(m.delta) > m.threshold ? `no evidence: p = ${pValue(p)}` : null;
+  if (m.verdict === "inconclusive") return `no evidence: p = ${pValue(p)}; ${MEASURE_AGAIN}`;
+  if (m.verdict === "unchanged") return null;
   return `p = ${pValue(p)}, ${amountOf(metric, shift)}, 95% CI ${amountOf(metric, ci[0])} to ${amountOf(metric, ci[1])}`;
 }
 
@@ -36,8 +44,15 @@ function noteOf(metric, m) {
   return `under ${metric === "CLS" ? m.threshold : `${Math.round(m.threshold)}ms`}`;
 }
 
-export function baselineView({ file, url, runs, metrics, errors, warnings }) {
-  const summary = `${url}, ${runs.count === 1 ? "1 run" : `${runs.count} runs, median by ${runs.medianBy}`}`;
+// "LCP, FCP inconclusive for the worse after 5 runs a side: both pages measured 2 more, up to 7"
+const describeExtended = ({ after, metrics }, runs) =>
+  `${metrics.join(", ")} inconclusive for the worse after ${after} runs a side: both pages measured ${runs.count - after} more, up to ${runs.count}`;
+
+// A baseline from a file is named by its file, with the URL it measured; a live one, measured in this
+// run, by its URL, with the runs it took and, when it measured more, why
+export function baselineView({ file, live, url, runs, metrics, errors, warnings, extended }) {
+  const runsText = runs.count === 1 ? "1 run" : `${runs.count} runs, median by ${runs.medianBy}`;
+  const [source, summary] = live ? [url, `measured in this run, ${runsText}`] : [file, `${url}, ${runsText}`];
   const rows = Object.entries(metrics).map(([metric, m]) => ({
     metric,
     unit: unitOf(metric),
@@ -47,7 +62,7 @@ export function baselineView({ file, url, runs, metrics, errors, warnings }) {
     verdict: m.verdict,
     note: m.verdict === "not measured" ? null : noteOf(metric, m),
   }));
-  return { file, summary, rows, newErrors: errors.new, fixedErrors: errors.fixed, warnings };
+  return { source, summary, ...(extended ? { extended: describeExtended(extended, runs) } : {}), rows, newErrors: errors.new, fixedErrors: errors.fixed, warnings };
 }
 
 // What got worse, one line each, for --fail-on-regression: the metrics that regressed and the errors
@@ -61,13 +76,23 @@ export function describeRegressions({ metrics, errors }) {
   ];
 }
 
+// The metrics inconclusive for the worse, one line each, for --fail-on-inconclusive: their change, their
+// p-value and what to do. An inconclusive improvement never fails.
+export function describeInconclusive(comparison) {
+  return inconclusiveForTheWorse(comparison).map((metric) => {
+    const m = comparison.metrics[metric];
+    return `${metric} ${describeChange(metric, m)}, p = ${pValue(m.stats.p)}; ${MEASURE_AGAIN}`;
+  });
+}
+
 // The change of a metric in a cell of the matrix table: the figure only when it counts, so the table
 // reads at a glance. "▲ +380ms", "▼ −400ms", "=" for unchanged, "·" when one side did not measure it.
 export function changeCell(metric, m) {
   if (m.verdict === "not measured") return { text: "·", verdict: m.verdict };
   if (m.verdict === "unchanged") return { text: "=", verdict: m.verdict };
   const amount = metric === "CLS" ? Math.abs(m.delta).toFixed(4) : `${Math.abs(Math.round(m.delta))}ms`;
-  return { text: `${arrow(m.delta)} ${signed(m.delta, amount)}`, verdict: m.verdict };
+  // An inconclusive change counts for nothing yet: a question mark keeps it apart from a regression
+  return { text: `${arrow(m.delta)} ${signed(m.delta, amount)}${m.verdict === "inconclusive" ? "?" : ""}`, verdict: m.verdict };
 }
 
 // The comparison of a matrix below its table. A warning that every compared combination has, such as
@@ -94,3 +119,8 @@ export function matrixComparisonView(root, combinations) {
 export function describeMatrixRegressions(combinations) {
   return combinations.filter((c) => c.baseline).flatMap((c) => describeRegressions(c.baseline).map((r) => `${c.id}: ${r}`));
 }
+
+// The inconclusive metrics for the worse of every combination, each named by its combination
+export function describeMatrixInconclusive(combinations) {
+  return combinations.filter((c) => c.baseline).flatMap((c) => describeInconclusive(c.baseline).map((line) => `${c.id}: ${line}`));
+}

package/src/reporters/human.js

@@ -148,6 +148,11 @@ function renderResult(r, verbose) {
     return `  ${styleText("dim", "·")} ${pad(r.id, 16)} ${styleText("dim", `not measured: ${r.message ?? "nothing recorded"}`)}`;
   }
 
+  // The browser lacks the API the snippet reads, so there is no value to show
+  if (r.status === "unsupported") {
+    return `  ${styleText("dim", "·")} ${pad(r.id, 16)} ${styleText("dim", `not supported: ${r.error ?? "the browser lacks the API"}`)}`;
+  }
+
   if (r.script === "Fonts-Preloaded-Loaded-and-used-above-the-fold") {
     return renderFontsResult(r);
   }
@@ -257,7 +262,7 @@ function renderRuns(runs) {
   return lines;
 }
 
-const VERDICT_STYLE = { regressed: "red", improved: "green", unchanged: "dim", "not measured": "dim", new: "cyan" };
+const VERDICT_STYLE = { regressed: "red", improved: "green", unchanged: "dim", inconclusive: "yellow", "not measured": "dim", new: "cyan" };
 
 // Every budget the run was given, passed or not, so the report says which one decided the exit code
 function renderBudgets(budgets) {
@@ -273,7 +278,8 @@ function renderBudgets(budgets) {
 // trust the table, then a row per metric and the errors that appeared or went away
 function renderBaseline(comparison) {
   const view = baselineView(comparison);
-  const lines = ["", styleText("bold", `Compared with ${view.file} (${view.summary})`)];
+  const lines = ["", styleText("bold", `Compared with ${view.source} (${view.summary})`)];
+  if (view.extended) lines.push(styleText("dim", `  ${view.extended}`));
   for (const warning of view.warnings) lines.push(`  ${styleText("yellow", `Warning: ${warning}`)}`);
   lines.push(styleText("dim", `  ${pad("Metric", 9)}${pad("Baseline", 11)}${pad("Now", 10)}Change`));
   for (const row of view.rows) {

package/src/reporters/markdown.js

@@ -48,6 +48,11 @@ function renderNotMeasuredRow(r) {
   return `| ${r.id} | · not measured | ${cell(r.message ?? "nothing recorded")} |`;
 }
 
+// The browser lacks the API the snippet reads, so there is no value to show
+function renderUnsupportedRow(r) {
+  return `| ${r.id} | · not supported | ${cell(r.error ?? "the browser lacks the API")} |`;
+}
+
 // A result with a count and no value: its count, and its first numbers as the human report gives them
 function renderSummaryRow(r) {
   return `| ${r.id} | · ${r.count ?? ""} | ${summaryNumbers(r).join(" · ")} |`;
@@ -57,6 +62,7 @@ function renderRow(r) {
   if (r.status === "skipped") return renderSkippedRow(r);
   if (r.status === "error") return renderErrorRow(r);
   if (r.status === "tracking") return renderNotMeasuredRow(r);
+  if (r.status === "unsupported") return renderUnsupportedRow(r);
   if (Array.isArray(r.issues)) return renderAuditRow(r);
   if (isSummaryResult(r)) return renderSummaryRow(r);
   return renderMetricRow(r);
@@ -162,13 +168,14 @@ function renderRuns(runs) {
   return lines;
 }
 
-const VERDICT_ICON = { regressed: "❌", improved: "✅", unchanged: "➖", "not measured": "·" };
+const VERDICT_ICON = { regressed: "❌", improved: "✅", unchanged: "➖", inconclusive: "❔", "not measured": "·" };
 
 // The comparison with --baseline. The warnings go in the comment itself: whoever reads the pull
 // request does not see the stderr of the CI.
 function renderBaseline(comparison) {
   const view = baselineView(comparison);
-  const lines = ["", `### Compared with \`${view.file}\``, `> ${view.summary}`];
+  const lines = ["", `### Compared with \`${view.source}\``, `> ${view.summary}`];
+  if (view.extended) lines.push(">", `> ${view.extended}`);
   for (const warning of view.warnings) lines.push(">", `> ⚠️ ${warning}`);
   lines.push("", "| Metric | Baseline | Now | Change | Verdict |", "|--------|----------|-----|--------|---------|");
   for (const row of view.rows) {
@@ -176,8 +183,8 @@ function renderBaseline(comparison) {
     const verdict = `${VERDICT_ICON[row.verdict]} ${row.verdict}${row.note ? ` (${row.note})` : ""}`;
     lines.push(`| ${row.metric} | ${value(row.baseline)} | ${value(row.current)} | ${row.change ?? "·"} | ${verdict} |`);
   }
-  if (view.newErrors.length) lines.push("", "#### New errors", ...view.newErrors.map((e) => `- \`${e.id}\`: ${e.message}`));
-  if (view.fixedErrors.length) lines.push("", "#### Fixed errors", ...view.fixedErrors.map((e) => `- \`${e.id}\`: ${e.message}`));
+  if (view.newErrors.length) lines.push("", "#### New errors", ...view.newErrors.map((e) => `- \`${e.id}\`: ${md(oneLine(e.message))}`));
+  if (view.fixedErrors.length) lines.push("", "#### Fixed errors", ...view.fixedErrors.map((e) => `- \`${e.id}\`: ${md(oneLine(e.message))}`));
   return lines;
 }