webperf-snippets 1.10.0 → 1.12.0

package/src/bin.js

@@ -15,7 +15,7 @@ import { MissingBrowserError } from "./browser.js";
 import { parseHeader } from "./headers.js";
 import { NETWORK_PRESETS, parseCpuFactor } from "./throttling.js";
 import { parseOwnDomains } from "./own-domains.js";
-import { MEDIAN_METRICS, summarizeRuns } from "./runs.js";
+import { MEDIAN_METRICS, measureRuns } from "./runs.js";
 import { loadMatrix } from "./matrix.js";
 import { blockingErrors } from "./blocking.js";
 import { MEASURED_BY, checkBudgets, parseBudgets, unmeasuredMetrics, violationsOf } from "./budgets.js";
@@ -89,6 +89,7 @@ Options:
   --verbose             Show all items, even for passing checks
   --headed              Show the browser window (debug)
   -h, --help            Show this help
+  -v, --version         Show the version
 
 Examples:
   npx webperf-snippets https://example.com
@@ -182,6 +183,7 @@ async function main() {
         verbose: { type: "boolean" },
         headed: { type: "boolean" },
         help: { type: "boolean", short: "h" },
+        version: { type: "boolean", short: "v" },
       },
       allowPositionals: true,
     });
@@ -196,6 +198,11 @@ async function main() {
     return;
   }
 
+  if (values.version) {
+    process.stdout.write(`${cliVersion}\n`);
+    return;
+  }
+
   const url = positionals[0];
   const matrixPath = values.matrix;
   if (url && matrixPath) fail("Use a URL or --matrix, not both");
@@ -325,7 +332,7 @@ async function main() {
   if (runCount == null) {
     payload = await measureWith(conditions);
   } else {
-    const median = await measureRuns(() => measureWith(conditions), runCount, medianBy, (i) => `[${i}/${runCount}] ${url} · run ${i}/${runCount}`);
+    const median = await measureRunsWithProgress(() => measureWith(conditions), runCount, medianBy, (i) => `[${i}/${runCount}] ${url} · run ${i}/${runCount}`);
     if (!median.payload) throw new Error(`Every run failed: ${median.errors.join("; ")}`);
     payload = { ...median.payload, runs: median.runs };
   }
@@ -388,22 +395,16 @@ async function main() {
   process.exit(blockingErrors(payload.results).length > 0 ? 1 : 0);
 }
 
-// Measures `count` runs one after the other, never in parallel: they would compete for the CPU and
-// skew the timings. Each run is reported on stderr as it starts, so stdout keeps only the report. A
-// run that fails is kept as { error } and left out of the median.
-async function measureRuns(measure, count, medianBy, progress) {
-  const runs = [];
-  for (let i = 1; i <= count; i++) {
-    process.stderr.write(`${progress(i)}\n`);
-    try {
-      runs.push(await measure());
-    } catch (err) {
-      if (err instanceof MissingBrowserError) throw err;
-      runs.push({ error: err.message });
-    }
-  }
-  return summarizeRuns(runs, { medianBy });
-}
+// Each run is reported on stderr as it starts, so stdout keeps only the report
+const measureRunsWithProgress = (measure, count, medianBy, progress) =>
+  measureRuns(
+    (i) => {
+      process.stderr.write(`${progress(i)}\n`);
+      return measure();
+    },
+    count,
+    { medianBy, isFatal: (err) => err instanceof MissingBrowserError }
+  );
 
 // Every combination of the matrix, in order, each one `matrix.runs` times. Each combination reports
 // its median run, and the exit code follows the same rules as one page, for every combination.
@@ -417,7 +418,7 @@ async function runMatrix(matrixPath, matrix, measureWith, { values, waitMs, repo
   const combinations = [];
   for (const c of matrix.combinations) {
     const conditions = { url: c.url, viewport: VIEWPORT_PRESETS[c.viewport], storageState: c.storageState, headers: c.headers, cpu: c.cpu, network: c.network, ownDomains: c.ownDomains };
-    const median = await measureRuns(() => measureWith(conditions), matrix.runs, matrix.medianBy, (i) => `[${String(++started).padStart(width)}/${total}] ${c.id} · run ${i}/${matrix.runs}`);
+    const median = await measureRunsWithProgress(() => measureWith(conditions), matrix.runs, matrix.medianBy, (i) => `[${String(++started).padStart(width)}/${total}] ${c.id} · run ${i}/${matrix.runs}`);
     const identity = { id: c.id, name: c.name, device: c.device, ...(c.variant ? { variant: c.variant } : {}), url: c.url };
     combinations.push(
       median.payload

package/src/crux.js

@@ -1,3 +1,4 @@
+import { DEFAULT_THRESHOLDS } from "./baseline.js";
 import { rate } from "./ratings.js";
 
 const DEFAULT_ENDPOINT = "https://chromeuxreport.googleapis.com/v1/records:queryRecord";
@@ -122,12 +123,17 @@ export function trendOf(p75s) {
 
 const BARS = "▁▂▃▄▅▆▇█";
 
-// The series as bars between its lowest and highest values, a space for a period without data
-export function sparkline(values) {
+// The series as bars, a space for a period without data. The bars span the range of the series, or
+// `minRange` when the series is narrower, centred in it: a CLS of 0.20 to 0.21 drawn from its lowest
+// to its highest value looked like a big swing. Eight equal steps, so a flat series sits in the middle.
+export function sparkline(values, minRange = 0) {
   const known = values.filter((v) => v !== null);
   const min = Math.min(...known);
   const max = Math.max(...known);
-  return values.map((v) => (v === null ? " " : max === min ? BARS[3] : BARS[Math.round(((v - min) / (max - min)) * 7)])).join("");
+  const range = Math.max(max - min, minRange);
+  if (range === 0) return values.map((v) => (v === null ? " " : BARS[4])).join("");
+  const low = max - min >= minRange ? min : (min + max) / 2 - range / 2;
+  return values.map((v) => (v === null ? " " : BARS[Math.min(7, Math.floor(((v - low) / range) * 8))])).join("");
 }
 
 const dateOf = ({ year, month, day }) => `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
@@ -202,7 +208,8 @@ export function historyView(history) {
   const rows = METRICS.filter((m) => history.metrics[m.name]).map((m) => {
     const { p75s, first, last, change, trend } = history.metrics[m.name];
     const changeText = change === null ? "·" : `${change > 0 ? "▲ +" : change < 0 ? "▼ −" : "= "}${Math.abs(change)}%`;
-    return { metric: m.name, unit: m.unit, first, last, change: changeText, trend: trend ?? "·", sparkline: sparkline(p75s) };
+    // The smallest change --baseline counts for the metric is the least the bars span
+    return { metric: m.name, unit: m.unit, first, last, change: changeText, trend: trend ?? "·", sparkline: sparkline(p75s, DEFAULT_THRESHOLDS[m.name].absolute) };
   });
   return { title, rows };
 }

package/src/mcp/baseline.js

@@ -0,0 +1,26 @@
+import { loadBaseline, compareRun, parseThreshold, thresholdsWith } from "../baseline.js";
+import { describeRegressions } from "../reporters/baseline-view.js";
+
+// The baseline of run_snippet and run_workflow, as --baseline of the CLI: the runId of an earlier run of
+// the connection, or the path of a page the CLI wrote with --json. The comparison is the one of the CLI
+// (baseline.js), so both give the same verdict for the same two pages.
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+// Read and checked before the page loads, as the CLI does before the browser opens: a wrong baseline
+// fails at once. Throws an error whose message says what to do.
+export function prepareBaseline({ baseline, thresholds = [] }, runs) {
+  const parsed = thresholdsWith(thresholds.map(parseThreshold));
+  // An id of a run the connection no longer keeps is not a file to look for: runs.page says so
+  if (runs.has(baseline) || UUID.test(baseline)) return { data: runs.page(baseline), source: { runId: baseline }, thresholds: parsed };
+  const { kind, data } = loadBaseline(baseline);
+  if (kind === "matrix") throw new Error(`${baseline} is the baseline of a matrix: this server measures one page; compare it with webperf-snippets --matrix <file> --baseline ${baseline}`);
+  return { data, source: { file: baseline }, thresholds: parsed };
+}
+
+// The page of this run against its baseline: the `baseline` of the JSON of the CLI, named by its runId or
+// its file, with the lines --fail-on-regression would print for it
+export function compareWithBaseline({ data, source, thresholds }, current) {
+  const { file, ...comparison } = compareRun(data, current, { file: source.file, thresholds });
+  return { ...source, ...comparison, regressions: describeRegressions(comparison) };
+}

package/src/mcp/queue.js

@@ -0,0 +1,72 @@
+// The reason of an abort is whatever the client sent, often a string, or nothing: a caller gets an error
+const gaveUp = () => new Error("The caller gave up while it waited for its turn");
+
+// One task at a time, in the order they arrive. A caller that gives up while it waits leaves the line
+// at once, so it never runs and holds up no one. A task that fails passes the turn like any other, and
+// its failure stays with its caller.
+//
+// A waiting caller hears how many tasks are ahead when it starts to wait, whenever the line moves, and
+// every `reportEveryMs` while it does not: a client can restart its timeout on each report, and a task
+// ahead can take minutes.
+export function createQueue({ reportEveryMs = 5000 } = {}) {
+  const waiting = [];
+  let taken = false;
+
+  // A report is a courtesy: one that fails, as one to a client that left, changes nothing
+  const report = async ({ onWait }, ahead) => {
+    try {
+      await onWait?.(ahead);
+    } catch {
+      // nothing to tell, and no one to tell it to
+    }
+  };
+  const reportAll = () => waiting.forEach((waiter, i) => report(waiter, i + 1));
+
+  function take({ signal, onWait }) {
+    if (signal?.aborted) return Promise.reject(gaveUp());
+    if (!taken) {
+      taken = true;
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      const waiter = { onWait };
+      const leave = () => {
+        waiting.splice(waiting.indexOf(waiter), 1);
+        clearInterval(timer);
+        signal?.removeEventListener("abort", cancel);
+      };
+      const cancel = () => {
+        leave();
+        reject(gaveUp());
+        reportAll();
+      };
+      waiter.enter = () => {
+        leave();
+        resolve();
+      };
+      const timer = setInterval(() => report(waiter, waiting.indexOf(waiter) + 1), reportEveryMs);
+      // The timer alone must not keep the process alive
+      timer.unref?.();
+      signal?.addEventListener("abort", cancel, { once: true });
+      waiting.push(waiter);
+      report(waiter, waiting.length);
+    });
+  }
+
+  function pass() {
+    taken = waiting.length > 0;
+    waiting[0]?.enter();
+    reportAll();
+  }
+
+  async function run(task, options = {}) {
+    await take(options);
+    try {
+      return await task();
+    } finally {
+      pass();
+    }
+  }
+
+  return { run };
+}