webperf-snippets 1.2.0 → 1.4.0

package/src/bin.js

@@ -6,18 +6,25 @@ import { resolveSnippetName } from "./snippet-names.js";
 import { runSnippets, runMeasurement, VIEWPORT_PRESETS } from "./runner.js";
 import { WORKFLOWS } from "./workflows/index.js";
 import { RULES } from "./decision-tree.js";
-import { reportHuman } from "./reporters/human.js";
+import { reportHuman, reportMatrixHuman } from "./reporters/human.js";
 import { reportJson } from "./reporters/json.js";
-import { reportMarkdown } from "./reporters/markdown.js";
+import { reportMarkdown, reportMatrixMarkdown } from "./reporters/markdown.js";
 import { cliVersion, reportTo, validateReportUrl } from "./report-to.js";
 import { fetchCrux, formFactorFor } from "./crux.js";
 import { MissingBrowserError } from "./browser.js";
 import { parseHeader } from "./headers.js";
 import { NETWORK_PRESETS, parseCpuFactor } from "./throttling.js";
 import { parseOwnDomains } from "./own-domains.js";
+import { MEDIAN_METRICS, summarizeRuns } from "./runs.js";
+import { loadMatrix } from "./matrix.js";
+import { blockingErrors } from "./blocking.js";
+import { measurementConditions } from "./conditions.js";
+import { loadBaseline, parseThreshold, thresholdsWith, compareRun, compareMatrix } from "./baseline.js";
+import { describeRegressions, describeMatrixRegressions, matrixComparisonView } from "./reporters/baseline-view.js";
 
 
 const USAGE = `webperf-snippets <url> [options]
+webperf-snippets --matrix <file> [options]
 webperf-snippets mcp
 
 Run curated WebPerf Snippets headlessly via Playwright.
@@ -53,6 +60,19 @@ Options:
                             the token of an access proxy. Repeat it for more headers
   --own-domains <list>      Domains of the site to count as first party, such as its
                             own CDN, comma-separated or repeated
+  --runs <n>                Measure the page n times (1 to 9), one after the other, and
+                            report the median run
+  --matrix <file>           Measure every URL of a JSON file on every device and in every
+                            variant it lists, and report a table of the medians
+  --median-by <metric>      Metric that picks the median run: LCP, FCP, TTFB, CLS, INP,
+                            navMs (default: the first of LCP, FCP, TTFB, navMs measured)
+  --baseline <file>         Compare the page, or each combination of a matrix, with the --json
+                            output of an earlier run: which metrics got worse or better beyond
+                            the noise, and which errors are new or fixed. It only informs,
+                            unless --fail-on-regression
+  --threshold <metric>=<v>  The threshold of a metric for --baseline, such as LCP=15%,
+                            LCP=200ms or CLS=0.1. Repeat it for more metrics
+  --fail-on-regression      Exit 1 when a metric regressed or an error is new against --baseline
   --crux-key <key>          Add CrUX field data (p75) next to the measured values.
                             Also read from the CRUX_API_KEY environment variable
   --report-to <url>         POST the results to this https URL after the run
@@ -74,6 +94,11 @@ Examples:
   npx webperf-snippets https://example.com --budget-lcp 2500
   npx webperf-snippets https://example.com --cpu 4 --network slow-4g
   npx webperf-snippets https://example.com --workflow loading --own-domains cdn.example.net
+  npx webperf-snippets https://example.com --runs 5 --budget-lcp 2500
+  npx webperf-snippets --matrix matrix.json --runs 3 --budget-lcp 2500
+  npx webperf-snippets https://example.com --runs 5 --json > baseline.json
+  npx webperf-snippets https://pr-123.example.com --runs 5 --baseline baseline.json --fail-on-regression
+  npx webperf-snippets --matrix matrix.json --baseline last-week.json
   npx webperf-snippets https://example.com --snippet INP --interact-script interactions.json
   npx webperf-snippets https://example.com/dashboard --storage-state auth.json
   npx webperf-snippets https://review.example.com --header "CF-Access-Client-Id: $CF_ID" --header "CF-Access-Client-Secret: $CF_SECRET"
@@ -94,11 +119,25 @@ function buildSnippetItem(values) {
   return [{ id: values.snippet, path, source: loadSnippet(path) }];
 }
 
-function checkBudgets(results, values) {
+// The count given to --runs, or null when it is not a whole number from 1 to 9
+function parseRunCount(raw) {
+  return /^[1-9]$/.test(raw.trim()) ? Number(raw) : null;
+}
+
+function checkBudgets({ results, runs }, values) {
   const violations = [];
   const lcpBudget = values["budget-lcp"] ? Number(values["budget-lcp"]) : null;
   const clsBudget = values["budget-cls"] ? Number(values["budget-cls"]) : null;
 
+  // With several runs the budgets compare the median of each metric, not one sample
+  if (runs) {
+    const lcp = runs.summary.LCP?.median;
+    const cls = runs.summary.CLS?.median;
+    if (lcp != null && lcpBudget != null && lcp > lcpBudget) violations.push(`LCP ${lcp}ms exceeds budget ${lcpBudget}ms`);
+    if (cls != null && clsBudget != null && cls > clsBudget) violations.push(`CLS ${cls} exceeds budget ${clsBudget}`);
+    return violations;
+  }
+
   for (const r of results) {
     if (r.status !== "ok") continue;
     if (r.metric === "LCP" && lcpBudget != null && r.value > lcpBudget) {
@@ -137,6 +176,12 @@ async function main() {
         "storage-state": { type: "string" },
         header: { type: "string", multiple: true },
         "own-domains": { type: "string", multiple: true },
+        matrix: { type: "string" },
+        runs: { type: "string" },
+        "median-by": { type: "string" },
+        baseline: { type: "string" },
+        threshold: { type: "string", multiple: true },
+        "fail-on-regression": { type: "boolean" },
         "crux-key": { type: "string" },
         "report-to": { type: "string" },
         "api-key": { type: "string" },
@@ -158,10 +203,13 @@ async function main() {
   }
 
   const url = positionals[0];
-  if (!url) {
+  const matrixPath = values.matrix;
+  if (url && matrixPath) fail("Use a URL or --matrix, not both");
+  if (!url && !matrixPath) {
     process.stdout.write(USAGE);
     process.exit(2);
   }
+  if (matrixPath && values.viewport) fail("--viewport does not apply to --matrix: set the viewport of each device in the file");
 
   const waitMs = values.wait ? Number(values.wait) : 3000;
   const viewportName = values.viewport ?? "mobile";
@@ -202,21 +250,74 @@ async function main() {
   }
   const ownDomains = values["own-domains"] && parseOwnDomains(values["own-domains"]);
 
+  const runCount = values.runs != null ? parseRunCount(values.runs) : undefined;
+  if (runCount === null) fail(`Invalid --runs "${values.runs}". Use a whole number from 1 to 9`);
+  const medianBy = values["median-by"];
+  if (medianBy != null && !MEDIAN_METRICS.includes(medianBy)) {
+    fail(`Unknown --median-by "${medianBy}". Choose from: ${MEDIAN_METRICS.join(", ")}`);
+  }
+  if (medianBy != null && runCount == null && !matrixPath) fail("--median-by needs --runs");
+
+  // The baseline is read and checked before the browser opens: a wrong file fails at once
+  let baseline;
+  let thresholds;
+  if (values.baseline) {
+    try {
+      baseline = loadBaseline(values.baseline);
+      thresholds = thresholdsWith((values.threshold ?? []).map(parseThreshold));
+    } catch (err) {
+      fail(err.message);
+    }
+    if (!matrixPath && baseline.kind === "matrix") fail(`${values.baseline} is the baseline of a matrix: compare it with --matrix`);
+    if (matrixPath && baseline.kind === "page") fail(`${values.baseline} is the baseline of a page: compare it with a URL, not --matrix`);
+  } else {
+    if (values.threshold) fail("--threshold needs --baseline");
+    if (values["fail-on-regression"]) fail("--fail-on-regression needs --baseline");
+  }
+
+  // What each page load runs, a snippet or a workflow, under the conditions of the load
+  const items = values.snippet ? buildSnippetItem(values) : null;
+  const workflow = values.snippet ? null : WORKFLOWS[values.workflow ?? "core-web-vitals"];
+  if (!values.snippet && !workflow) fail(`Unknown workflow: ${values.workflow}`);
+  const measureWith = (conditions) => {
+    const options = { waitMs, headless: !values.headed, interactScript, ...conditions };
+    return items ? runSnippets({ ...options, items }) : runMeasurement({ ...options, workflow, rules: RULES });
+  };
+  // What the JSON records of how each page was measured, for a later run to compare with
+  const conditionsFor = (viewport) =>
+    measurementConditions({ viewport, waitMs, ...(items ? { snippet: items[0].path } : { workflow: values.workflow ?? "core-web-vitals" }) });
+
+  if (matrixPath) {
+    let matrix;
+    try {
+      matrix = loadMatrix(matrixPath, { cpu, network, storageState, headers, ownDomains, runs: runCount, medianBy });
+    } catch (err) {
+      fail(err.message);
+    }
+    await runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds });
+    return;
+  }
+
+  const conditions = { url, viewport, storageState, headers, cpu, network, ownDomains };
   let payload;
-  if (values.snippet) {
-    const items = buildSnippetItem(values);
-    payload = await runSnippets({ url, items, waitMs, headless: !values.headed, viewport, interactScript, storageState, headers, cpu, network, ownDomains });
+  if (runCount == null) {
+    payload = await measureWith(conditions);
   } else {
-    const workflowName = values.workflow ?? "core-web-vitals";
-    const workflow = WORKFLOWS[workflowName];
-    if (!workflow) fail(`Unknown workflow: ${workflowName}`);
-    payload = await runMeasurement({ url, workflow, rules: RULES, waitMs, headless: !values.headed, viewport, interactScript, storageState, headers, cpu, network, ownDomains });
+    const median = await measureRuns(() => measureWith(conditions), runCount, medianBy, (i) => `[${i}/${runCount}] ${url} · run ${i}/${runCount}`);
+    if (!median.payload) throw new Error(`Every run failed: ${median.errors.join("; ")}`);
+    payload = { ...median.payload, runs: median.runs };
   }
+  payload.conditions = conditionsFor(viewportName);
 
   const cruxKey = values["crux-key"] || process.env.CRUX_API_KEY;
   const crux = cruxKey ? await fetchCrux({ url, apiKey: cruxKey, formFactor: formFactorFor(viewportName) }) : undefined;
   if (crux) payload.crux = crux;
 
+  if (baseline) {
+    payload.baseline = compareRun(baseline.data, payload, { file: baseline.file, thresholds });
+    for (const warning of payload.baseline.warnings) process.stderr.write(`Warning: ${warning}\n`);
+  }
+
   let output;
   if (values.markdown) {
     output = reportMarkdown(payload);
@@ -238,7 +339,7 @@ async function main() {
         timestamp: new Date().toISOString(),
         navMs: payload.navMs,
         results: payload.results,
-        meta: { viewport: viewportName, waitMs, ...(payload.throttling ? { throttling: payload.throttling } : {}), cli_version: cliVersion },
+        meta: { viewport: viewportName, waitMs, ...(payload.throttling ? { throttling: payload.throttling } : {}), ...(payload.runs ? { runs: payload.runs } : {}), cli_version: cliVersion },
       },
     });
     if (!outcome.ok) {
@@ -249,21 +350,129 @@ async function main() {
   }
 
   // Exit codes.
-  const violations = checkBudgets(payload.results, values);
+  const violations = checkBudgets(payload, values);
   if (violations.length > 0) {
     if (!values.json) {
       for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
     }
     process.exit(1);
   }
+  // Without --fail-on-regression the comparison only informs, and the exit code is the one of today
+  const regressions = values["fail-on-regression"] ? describeRegressions(payload.baseline) : [];
+  if (regressions.length > 0) {
+    for (const r of regressions) process.stderr.write(`Regression against ${baseline.file}: ${r}\n`);
+    process.exit(1);
+  }
+  process.exit(blockingErrors(payload.results).length > 0 ? 1 : 0);
+}
+
+// Measures `count` runs one after the other, never in parallel: they would compete for the CPU and
+// skew the timings. Each run is reported on stderr as it starts, so stdout keeps only the report. A
+// run that fails is kept as { error } and left out of the median.
+async function measureRuns(measure, count, medianBy, progress) {
+  const runs = [];
+  for (let i = 1; i <= count; i++) {
+    process.stderr.write(`${progress(i)}\n`);
+    try {
+      runs.push(await measure());
+    } catch (err) {
+      if (err instanceof MissingBrowserError) throw err;
+      runs.push({ error: err.message });
+    }
+  }
+  return summarizeRuns(runs, { medianBy });
+}
+
+// Every combination of the matrix, in order, each one `matrix.runs` times. Each combination reports
+// its median run, and the exit code follows the same rules as one page, for every combination.
+async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, reportUrl, conditionsFor, baseline, thresholds }) {
+  for (const warning of matrix.warnings) process.stderr.write(`Warning: ${warning}\n`);
+  const viewportOf = new Map(matrix.combinations.map((c) => [c.id, c.viewport]));
+
+  const total = matrix.combinations.length * matrix.runs;
+  const width = String(total).length;
+  let started = 0;
+  const combinations = [];
+  for (const c of matrix.combinations) {
+    const conditions = { url: c.url, viewport: VIEWPORT_PRESETS[c.viewport], storageState: c.storageState, headers: c.headers, cpu: c.cpu, network: c.network, ownDomains: c.ownDomains };
+    const median = await measureRuns(() => measureWith(conditions), matrix.runs, matrix.medianBy, (i) => `[${String(++started).padStart(width)}/${total}] ${c.id} · run ${i}/${matrix.runs}`);
+    const identity = { id: c.id, name: c.name, device: c.device, ...(c.variant ? { variant: c.variant } : {}), url: c.url };
+    combinations.push(
+      median.payload
+        ? { ...identity, ...median.payload, runs: median.runs, conditions: conditionsFor(c.viewport) }
+        : { ...identity, status: "error", error: `Every run failed: ${median.errors.join("; ")}`, runs: { count: matrix.runs, failed: matrix.runs }, conditions: conditionsFor(c.viewport) }
+    );
+  }
+
+  // The field data of a URL depends on the form factor, not on the variant: one query for each pair
+  const cruxKey = values["crux-key"] || process.env.CRUX_API_KEY;
+  if (cruxKey) {
+    const queries = new Map();
+    for (const c of combinations.filter((c) => !c.error)) {
+      const formFactor = formFactorFor(viewportOf.get(c.id));
+      const key = `${c.url} ${formFactor}`;
+      if (!queries.has(key)) queries.set(key, fetchCrux({ url: c.url, apiKey: cruxKey, formFactor }));
+      const crux = await queries.get(key);
+      if (crux) c.crux = crux;
+    }
+  }
+
+  // With --baseline, each combination is compared with the one of the same id in the baseline
+  let comparison;
+  if (baseline) {
+    const { root, combinations: byId } = compareMatrix(baseline.data, { combinations }, { file: baseline.file, thresholds });
+    for (const c of combinations) if (byId[c.id]) c.baseline = byId[c.id];
+    comparison = root;
+    for (const warning of matrixComparisonView(root, combinations).warnings) process.stderr.write(`Warning: ${warning}\n`);
+  }
+
+  const withBaseline = comparison ? { baseline: comparison } : {};
+  const report = { matrix: matrixPath, runs: matrix.runs, combinations, verbose: values.verbose, ...withBaseline };
+  const output = values.markdown ? reportMatrixMarkdown(report) : values.json ? reportJson({ matrix: matrixPath, runs: matrix.runs, ...withBaseline, combinations }) : reportMatrixHuman(report);
+  process.stdout.write(output + "\n");
+
+  // One report for each combination, with the body of one page and the device, variant and runs of
+  // the combination. The headers are conditions of the load, never part of the body.
+  if (reportUrl) {
+    for (const c of combinations.filter((c) => !c.error)) {
+      const outcome = await reportTo({
+        url: reportUrl,
+        apiKey: values["api-key"] || process.env.PERF_REVIEWS_API_KEY,
+        body: {
+          url: c.url,
+          workflow: values.snippet ? null : (values.workflow ?? "core-web-vitals"),
+          ...(values.snippet ? { snippet: values.snippet } : {}),
+          timestamp: new Date().toISOString(),
+          navMs: c.navMs,
+          results: c.results,
+          meta: {
+            viewport: viewportOf.get(c.id),
+            device: c.device,
+            ...(c.variant ? { variant: c.variant } : {}),
+            waitMs,
+            ...(c.throttling ? { throttling: c.throttling } : {}),
+            runs: c.runs,
+            cli_version: cliVersion,
+          },
+        },
+      });
+      if (!outcome.ok) {
+        process.stderr.write(`Warning: could not report ${c.id} to ${new URL(reportUrl).origin} (${outcome.warning})\n`);
+      } else if (outcome.regressions.length > 0) {
+        process.stderr.write(`Regressions detected in ${c.id}:\n${outcome.regressions.map((r) => `  - ${typeof r === "string" ? r : JSON.stringify(r)}`).join("\n")}\n`);
+      }
+    }
+  }
 
-  const anyError = payload.results.some((r) => r.status === "error");
-  // A follow-up (a result with a reason) only informs: its error issues show in the report and
-  // leave the exit code to the budgets and to the steps the workflow runs itself.
-  const anyAuditViolation = payload.results.some(
-    (r) => !r.reason && Array.isArray(r.issues) && r.issues.some((i) => i.severity === "error"),
-  );
-  process.exit(anyError || anyAuditViolation ? 1 : 0);
+  const violations = combinations.filter((c) => !c.error).flatMap((c) => checkBudgets(c, values).map((v) => `${v} (${c.id})`));
+  if (!values.json) {
+    for (const v of violations) process.stderr.write(`Budget violation: ${v}\n`);
+  }
+  const failed = combinations.some((c) => c.error || blockingErrors(c.results).length > 0);
+  // Without --fail-on-regression the comparison only informs
+  const regressions = values["fail-on-regression"] ? describeMatrixRegressions(combinations) : [];
+  for (const r of regressions) process.stderr.write(`Regression against ${baseline.file} in ${r}\n`);
+  process.exit(violations.length > 0 || failed || regressions.length > 0 ? 1 : 0);
 }
 
 main().catch((err) => {

package/src/blocking.js

@@ -0,0 +1,14 @@
+// The results that make the command exit with 1: a snippet that failed, and an issue of severity
+// error in a step the workflow runs itself. A follow-up (a result with a reason) only informs: its
+// error issues show in the report and leave the exit code to the budgets and to the steps.
+export function blockingErrors(results) {
+  const errors = [];
+  for (const r of results) {
+    if (r.status === "error") errors.push({ id: r.id, message: r.error });
+    if (r.reason || !Array.isArray(r.issues)) continue;
+    for (const issue of r.issues) {
+      if (issue.severity === "error") errors.push({ id: r.id, message: issue.message, ...(issue.code ? { code: issue.code } : {}) });
+    }
+  }
+  return errors;
+}

package/src/conditions.js

@@ -0,0 +1,7 @@
+import { cliVersion } from "./report-to.js";
+
+// How a measurement was taken, kept in its JSON so that a later run compared with it can tell whether
+// both were measured the same way. A snippet is named by its path, so an alias and its path match.
+export function measurementConditions({ viewport, workflow, snippet, waitMs }) {
+  return { viewport, ...(snippet ? { snippet } : { workflow }), waitMs, cliVersion };
+}

package/src/crux.js

@@ -1,16 +1,16 @@
+import { rate } from "./ratings.js";
+
 const DEFAULT_ENDPOINT = "https://chromeuxreport.googleapis.com/v1/records:queryRecord";
 const CRUX_TIMEOUT_MS = 10000;
 
-// Good / needs-improvement upper bounds from web.dev/articles/vitals.
 const METRICS = [
-  { name: "LCP", key: "largest_contentful_paint", unit: "ms", good: 2500, poor: 4000 },
-  { name: "CLS", key: "cumulative_layout_shift", unit: "score", good: 0.1, poor: 0.25 },
-  { name: "INP", key: "interaction_to_next_paint", unit: "ms", good: 200, poor: 500 },
+  { name: "LCP", key: "largest_contentful_paint", unit: "ms" },
+  { name: "CLS", key: "cumulative_layout_shift", unit: "score" },
+  { name: "INP", key: "interaction_to_next_paint", unit: "ms" },
 ];
 
 const FORM_FACTOR_LABEL = { PHONE: "mobile", DESKTOP: "desktop", TABLET: "tablet" };
 
-const rate = (value, { good, poor }) => (value <= good ? "good" : value <= poor ? "needs-improvement" : "poor");
 
 // The viewport preset decides which population the field data describes.
 export function formFactorFor(viewportName) {
@@ -22,7 +22,7 @@ function parseMetrics(record) {
   for (const m of METRICS) {
     // CLS p75 comes as a string ("0.08"); LCP and INP as numbers.
     const p75 = Number(record?.metrics?.[m.key]?.percentiles?.p75);
-    if (Number.isFinite(p75)) metrics[m.name] = { p75, rating: rate(p75, m), unit: m.unit };
+    if (Number.isFinite(p75)) metrics[m.name] = { p75, rating: rate(m.name, p75), unit: m.unit };
   }
   return metrics;
 }

package/src/matrix.js

@@ -0,0 +1,227 @@
+// A matrix file: several pages, measured on several devices and in several variants, each
+// combination one or more times. This module reads, validates and expands the file into the
+// combinations to measure, with the conditions of each one already combined with the flags of the
+// command line. It opens no browser, so an invalid file fails before any page load.
+//
+// {
+//   "urls": { "home": "https://example.com/" },              or an array of URLs
+//   "devices": { "mobile": { "viewport": "mobile", "cpu": 4, "network": "slow-4g" } },
+//   "variants": { "consent": { "storageState": "consent.json", "headers": {}, "ownDomains": [] } },
+//   "runs": 3,
+//   "medianBy": "LCP"
+// }
+//
+// The conditions of the file replace cpu, network and storageState of the command line where they
+// set them; headers and own domains add to the ones of the command line, which carries the secrets.
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { VIEWPORT_PRESETS } from "./runner.js";
+import { resolveNetwork } from "./throttling.js";
+import { parseHeader } from "./headers.js";
+import { parseOwnDomains } from "./own-domains.js";
+import { MEDIAN_METRICS } from "./runs.js";
+
+const KEYS = {
+  matrix: ["urls", "devices", "variants", "runs", "medianBy"],
+  device: ["viewport", "cpu", "network"],
+  variant: ["storageState", "headers", "ownDomains"],
+};
+
+// Without devices, a mid-range phone, as Lighthouse measures by default
+const DEFAULT_DEVICE = { viewport: "mobile", cpu: 4, network: "slow-4g" };
+
+// A header whose name says it carries a secret, which a versioned file should not hold
+const LOOKS_SECRET = /secret|token|authorization|key|cookie/i;
+
+class MatrixError extends Error {
+  constructor(problem) {
+    super(`Invalid matrix: ${problem}`);
+  }
+}
+
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+
+function checkKeys(object, known, at) {
+  for (const key of Object.keys(object)) {
+    if (!known.includes(key)) {
+      throw new MatrixError(`unknown key "${at ? `${at}.` : ""}${key}". Known keys: ${known.join(", ")}`);
+    }
+  }
+}
+
+// The URLs by name. An array names each URL by its path and query, with the host when there are
+// several, which is what a reader recognises in a table of 5 rows or of 50.
+function readUrls(urls) {
+  if (urls === undefined) throw new MatrixError("urls is required");
+  const entries = Array.isArray(urls) ? urls.map((url, i) => [`[${i}]`, url]) : isObject(urls) ? Object.entries(urls).map(([name, url]) => [`.${name}`, url, name]) : null;
+  if (!entries) throw new MatrixError("urls must be an object of names and URLs, or an array of URLs");
+  if (entries.length === 0) throw new MatrixError("urls has no URL");
+
+  const parsed = entries.map(([at, url, name]) => {
+    let parsedUrl;
+    try {
+      parsedUrl = new URL(url);
+    } catch {
+      parsedUrl = null;
+    }
+    if (typeof url !== "string" || !parsedUrl || !["http:", "https:"].includes(parsedUrl.protocol)) {
+      throw new MatrixError(`urls${at} "${url}" is not an http or https URL`);
+    }
+    return { url, parsedUrl, name };
+  });
+  if (!Array.isArray(urls)) return parsed.map(({ name, url }) => ({ name, url }));
+
+  const severalHosts = new Set(parsed.map(({ parsedUrl }) => parsedUrl.host)).size > 1;
+  const named = parsed.map(({ url, parsedUrl }) => ({
+    name: `${severalHosts ? parsedUrl.host : ""}${parsedUrl.pathname}${parsedUrl.search}`,
+    url,
+  }));
+  const seen = new Set();
+  for (const { name, url } of named) {
+    if (seen.has(name)) {
+      const suggestion = name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "") || "home";
+      throw new MatrixError(`two URLs of urls are named "${name}". Name them with an object, such as { "${suggestion}": "${url}" }`);
+    }
+    seen.add(name);
+  }
+  return named;
+}
+
+function checkViewport(viewport, at) {
+  if (!VIEWPORT_PRESETS[viewport]) {
+    throw new MatrixError(`${at} "${viewport}" is not a preset. Choose from: ${Object.keys(VIEWPORT_PRESETS).join(", ")}`);
+  }
+}
+
+// The devices by name. An array of viewport presets is a shortcut for devices without throttling.
+function readDevices(devices, flags) {
+  if (devices === undefined) {
+    return [{ name: "mobile", viewport: DEFAULT_DEVICE.viewport, cpu: flags.cpu ?? DEFAULT_DEVICE.cpu, network: flags.network ?? DEFAULT_DEVICE.network }];
+  }
+  if (Array.isArray(devices) ? devices.length === 0 : isObject(devices) && Object.keys(devices).length === 0) {
+    throw new MatrixError("devices has no device");
+  }
+  if (Array.isArray(devices)) {
+    devices.forEach((viewport, i) => checkViewport(viewport, `devices[${i}]`));
+    return devices.map((viewport) => ({ name: viewport, viewport, cpu: flags.cpu, network: flags.network }));
+  }
+  if (!isObject(devices)) throw new MatrixError("devices must be an object of names and devices, or an array of viewport presets");
+
+  return Object.entries(devices).map(([name, device]) => {
+    const at = `devices.${name}`;
+    if (!isObject(device)) throw new MatrixError(`${at} must be an object, such as { "viewport": "mobile", "cpu": 4, "network": "slow-4g" }`);
+    checkKeys(device, KEYS.device, at);
+    const viewport = device.viewport ?? "mobile";
+    checkViewport(viewport, `${at}.viewport`);
+    if (device.cpu !== undefined && !(typeof device.cpu === "number" && device.cpu >= 1)) {
+      throw new MatrixError(`${at}.cpu must be a number of at least 1, such as 4`);
+    }
+    if (device.network !== undefined) {
+      try {
+        resolveNetwork(device.network);
+      } catch (err) {
+        throw new MatrixError(`${at}.network: ${err.message}`);
+      }
+    }
+    return { name, viewport, cpu: device.cpu ?? flags.cpu, network: device.network ?? flags.network };
+  });
+}
+
+// The variants by name. Without variants, one variant with no name and no conditions of its own,
+// so the combinations are the pages on the devices and the reports leave the variant out.
+function readVariants(variants, base, warnings) {
+  if (variants === undefined) return [{ headers: {}, ownDomains: [] }];
+  if (!isObject(variants)) throw new MatrixError("variants must be an object of names and variants");
+  if (Object.keys(variants).length === 0) throw new MatrixError("variants has no variant");
+
+  return Object.entries(variants).map(([name, variant]) => {
+    const at = `variants.${name}`;
+    if (!isObject(variant)) throw new MatrixError(`${at} must be an object, such as { "storageState": "consent.json" }`);
+    checkKeys(variant, KEYS.variant, at);
+
+    let storageState;
+    if (variant.storageState !== undefined) {
+      storageState = resolve(base, String(variant.storageState));
+      if (!existsSync(storageState)) throw new MatrixError(`${at}.storageState "${variant.storageState}" not found at ${storageState}`);
+    }
+
+    const headers = {};
+    if (variant.headers !== undefined) {
+      if (!isObject(variant.headers)) throw new MatrixError(`${at}.headers must be an object of header names and values`);
+      for (const [header, value] of Object.entries(variant.headers)) {
+        if (!parseHeader(`${header}: `)) throw new MatrixError(`${at}.headers "${header}" is not a valid header`);
+        if (typeof value !== "string") throw new MatrixError(`${at}.headers "${header}" must be a string`);
+        if (LOOKS_SECRET.test(header)) {
+          warnings.push(`${at}.headers.${header} looks like a secret, and a matrix file is usually versioned. Pass it with --header "${header}: $VARIABLE" instead.`);
+        }
+        headers[header] = value;
+      }
+    }
+
+    let ownDomains = [];
+    if (variant.ownDomains !== undefined) {
+      const parsed = Array.isArray(variant.ownDomains) && variant.ownDomains.every((d) => typeof d === "string") ? parseOwnDomains(variant.ownDomains) : null;
+      if (!parsed) throw new MatrixError(`${at}.ownDomains must be a list of domains without spaces`);
+      ownDomains = parsed;
+    }
+
+    return { name, storageState, headers, ownDomains };
+  });
+}
+
+// The headers of the command line and of the variant; the variant wins on the same header, whatever its case
+function mergeHeaders(fromFlags = {}, fromVariant) {
+  const names = new Set(Object.keys(fromVariant).map((name) => name.toLowerCase()));
+  const kept = Object.entries(fromFlags).filter(([name]) => !names.has(name.toLowerCase()));
+  return Object.fromEntries([...kept, ...Object.entries(fromVariant)]);
+}
+
+// The file at `path`, with the flags of the command line: cpu, network, storageState, headers,
+// ownDomains, runs and medianBy. Returns the runs, the metric of the median, the combinations in the
+// order they run (URL, then device, then variant) and the warnings to print.
+export function loadMatrix(path, flags = {}) {
+  if (!existsSync(path)) throw new Error(`Matrix file not found: ${path}`);
+  let matrix;
+  try {
+    matrix = JSON.parse(readFileSync(path, "utf8"));
+  } catch (err) {
+    throw new MatrixError(`the file is not valid JSON (${err.message})`);
+  }
+  if (!isObject(matrix)) throw new MatrixError("the file must hold an object, such as { \"urls\": [\"https://example.com/\"] }");
+  checkKeys(matrix, KEYS.matrix);
+
+  if (matrix.runs !== undefined && !(Number.isInteger(matrix.runs) && matrix.runs >= 1 && matrix.runs <= 9)) {
+    throw new MatrixError("runs must be a whole number from 1 to 9");
+  }
+  if (matrix.medianBy !== undefined && !MEDIAN_METRICS.includes(matrix.medianBy)) {
+    throw new MatrixError(`medianBy "${matrix.medianBy}" is not a metric. Choose from: ${MEDIAN_METRICS.join(", ")}`);
+  }
+
+  const warnings = [];
+  const urls = readUrls(matrix.urls);
+  const devices = readDevices(matrix.devices, flags);
+  const variants = readVariants(matrix.variants, dirname(resolve(path)), warnings);
+
+  const combinations = [];
+  for (const { name, url } of urls) {
+    for (const device of devices) {
+      for (const variant of variants) {
+        combinations.push({
+          id: [name, device.name, variant.name].filter(Boolean).join(" · "),
+          name,
+          url,
+          device: device.name,
+          ...(variant.name ? { variant: variant.name } : {}),
+          viewport: device.viewport,
+          cpu: device.cpu,
+          network: device.network,
+          storageState: variant.storageState ?? flags.storageState,
+          headers: mergeHeaders(flags.headers, variant.headers),
+          ownDomains: [...(flags.ownDomains ?? []), ...variant.ownDomains],
+        });
+      }
+    }
+  }
+
+  return { runs: flags.runs ?? matrix.runs ?? 1, medianBy: flags.medianBy ?? matrix.medianBy, combinations, warnings };
+}

package/src/ratings.js

@@ -0,0 +1,16 @@
+// Good / needs-improvement upper bounds from web.dev/articles/vitals, and the ones web.dev gives for
+// FCP and TTFB, the same the snippets rate with.
+export const THRESHOLDS = {
+  LCP: { good: 2500, poor: 4000 },
+  FCP: { good: 1800, poor: 3000 },
+  TTFB: { good: 800, poor: 1800 },
+  CLS: { good: 0.1, poor: 0.25 },
+  INP: { good: 200, poor: 500 },
+};
+
+// "good", "needs-improvement" or "poor", or undefined for a value without thresholds
+export function rate(metric, value) {
+  const thresholds = THRESHOLDS[metric];
+  if (!thresholds) return undefined;
+  return value <= thresholds.good ? "good" : value <= thresholds.poor ? "needs-improvement" : "poor";
+}