webperf-snippets 1.10.0 → 1.12.0

package/src/mcp/server.js

@@ -1,4 +1,4 @@
-import { McpServer } from "@modelcontextprotocol/server";
+import { McpServer, ResourceTemplate, ResourceNotFoundError } from "@modelcontextprotocol/server";
 import * as z from "zod/v4";
 import { cliVersion } from "../report-to.js";
 import { CATEGORIES, aliasesOf, listSnippets, resolveSnippetName } from "../snippet-names.js";
@@ -10,16 +10,25 @@ import { RULES } from "../decision-tree.js";
 import { fetchCrux, fetchCruxHistory } from "../crux.js";
 import { runInteractions, isInputStep, settle } from "../interactions.js";
 import { createSessions, SessionError } from "./sessions.js";
+import { createQueue } from "./queue.js";
 import { launchChromium, isMissingBrowser, MISSING_BROWSER } from "../browser.js";
 import { registerResources } from "./documents.js";
 import { registerPrompts } from "./prompts.js";
 import { shapeResults, createRuns, RunError } from "./summary.js";
 import { followUpCalls, trackingCall } from "./next-steps.js";
 import { isHttpUrl } from "../url.js";
+import { machineOf } from "../machine.js";
+import { measurementConditions } from "../conditions.js";
+import { prepareBaseline, compareWithBaseline } from "./baseline.js";
+import { MAX_RUNS, measureRuns } from "../runs.js";
 
 const VIEWPORTS = Object.keys(VIEWPORT_PRESETS);
 const STAGES = 3;
 const NAV_TIMEOUT_MS = 30000;
+// How long a page load can keep its turn: far beyond the load and the wait, which have their own limits
+// (30 s and 10 s), because a run that reaches this is not slow. A page whose main thread never yields
+// answers no evaluation, and Playwright has no timeout for one, so it would hold up the queue for ever.
+const CALL_LIMIT_MS = 120000;
 
 const HttpUrl = z
   .string()
@@ -36,8 +45,59 @@ const SnippetResult = z
 const SessionId = z.string().describe("The id open_page returned");
 
 const RunId = z.string().describe("The id of the run, for get_details");
-// Only a summary names its run: whole objects leave nothing for get_details to add
+// Only a summary of read_results names its run: whole objects leave nothing for get_details to add
 const SummaryRunId = RunId.optional().describe("The id of the run, for get_details; only with summaries");
+// A run of a page load is also the baseline of a later one, so it always has its id
+const PageRunId = RunId.describe("The id of the run: the baseline of a later run_snippet or run_workflow; with summaries, also for get_details");
+
+const Baseline = z
+  .string()
+  .min(1)
+  .optional()
+  .describe(
+    "Compare this run with a baseline, as --baseline of the CLI: the runId of an earlier run_snippet or run_workflow of this connection (listed as webperf://runs/{runId}), or the path of a page the CLI wrote with --json, absolute or relative to the working directory of the server"
+  );
+const Thresholds = z
+  .array(z.string())
+  .optional()
+  .describe("With baseline, the threshold of a metric, as --threshold of the CLI: LCP=15% (relative), LCP=200ms (absolute), CLS=0.1. The part not given keeps its default");
+
+const Runs = z
+  .number()
+  .int()
+  .min(1)
+  .max(MAX_RUNS)
+  .default(1)
+  .describe(
+    "Measure the page this many times, one after the other, as --runs of the CLI: the answer is the median run, plus runs with the median, min, max and agreement of each metric. With baseline, 5 or more on both sides give the statistical verdict: use 5 for a comparison, and more only when agreement says noisy. Each run is a full page load and wait, so it takes that much longer"
+  );
+
+const LooseObject = (shape) => z.looseObject(shape).meta({ additionalProperties: true });
+const RunsSummary = LooseObject({
+  count: z.number(),
+  failed: z.number().describe("Runs left out because they failed"),
+  medianRun: z.number().describe("The run the answer comes from, from 1"),
+  medianBy: z.string().describe("The metric that picked it"),
+  summary: z.record(z.string(), LooseObject({ median: z.number() })).describe("By metric: median, min, max, values of each run (null for a failed one), unit, cv and agreement (consistent, variable, noisy)"),
+})
+  .optional()
+  .describe("With several runs, the runs field of the JSON of the CLI");
+const BaselineError = LooseObject({ id: z.string(), message: z.string().optional() });
+const Comparison = z
+  .object({
+    runId: z.string().optional().describe("The run the baseline was, when it was a runId"),
+    file: z.string().optional().describe("The file the baseline was, when it was a path"),
+    url: z.string().describe("The URL the baseline measured"),
+    runs: LooseObject({ count: z.number() }).describe("The runs of the baseline"),
+    metrics: z
+      .record(z.string(), LooseObject({ verdict: z.string() }))
+      .describe("By metric: baseline, current, delta, deltaPercent, threshold, verdict (regressed, improved, unchanged, not measured) and, with five runs a side, stats (p, shift, ci, method)"),
+    errors: z.object({ new: z.array(BaselineError), fixed: z.array(BaselineError) }).describe("The errors only this run has, and the ones only the baseline had"),
+    warnings: z.array(z.string()).describe("What differs in how both were measured: how far to trust the comparison"),
+    regressions: z.array(z.string()).describe("What got worse, one line each: the metrics that regressed and the new errors"),
+  })
+  .optional()
+  .describe("With baseline, the comparison, as the baseline field of the JSON of the CLI");
 
 // auto, the default, returns whole objects when they are small and summaries when they are large: a
 // summary of a small result costs the agent a turn for the details, which is more than it saves
@@ -55,6 +115,8 @@ const NextSteps = z
 
 const DETAIL_NOTE =
   "Large results come as summaries with a runId for get_details; nextSteps, when present, is the call to make next.";
+const BASELINE_NOTE = "With baseline, it also compares the run with an earlier one, as --baseline of the CLI, and says what regressed.";
+const QUEUE_NOTE = "Measurements run one at a time, so several in one turn save no time.";
 
 // A page can put instructions for the agent in what a snippet returns: an alt text, a class in a
 // selector, the URL of a script. The tools that return what a page holds say how to read it, and so
@@ -92,13 +154,60 @@ const toolError = (text) => ({ isError: true, content: [{ type: "text", text }]
 
 const firstLine = (message) => message.split("\n")[0];
 
-// A client that wants progress sends a token with the call; the stages of a run go to it
-function progressFor(ctx) {
+// A client that wants progress sends a token with the call; the stages of each run go to it, numbered
+// across the runs. Returns the reporter of a run, by its number.
+function progressFor(ctx, runs) {
   const progressToken = ctx.mcpReq._meta?.progressToken;
-  return (progress, message) =>
+  const total = STAGES * runs;
+  return (run) => (stage, message) =>
     progressToken === undefined
       ? undefined
-      : ctx.mcpReq.notify({ method: "notifications/progress", params: { progressToken, progress, total: STAGES, message } });
+      : ctx.mcpReq.notify({
+          method: "notifications/progress",
+          params: { progressToken, progress: (run - 1) * STAGES + stage, total, message: runs > 1 ? `Run ${run} of ${runs}: ${message}` : message },
+        });
+}
+
+// The report of a call that waits for its turn, for a client that wants progress: how many calls are
+// ahead. Progress has to grow with each report, and the stages of a run start at 1, so it stays below 1.
+function waitProgressFor(ctx, total) {
+  const progressToken = ctx.mcpReq._meta?.progressToken;
+  if (progressToken === undefined) return undefined;
+  let reports = 0;
+  return (ahead) => {
+    reports++;
+    const message = `Waiting for its turn: ${ahead} ${ahead === 1 ? "call" : "calls"} ahead`;
+    return ctx.mcpReq.notify({ method: "notifications/progress", params: { progressToken, progress: reports / (reports + 1), ...(total ? { total } : {}), message } });
+  };
+}
+
+const stoppedAnswering = (limitMs) => `The page did not answer within ${limitMs / 1000} seconds`;
+
+// A page that stopped answering is closed, and the runs left of the call would only hold up the queue
+class PageStoppedError extends Error {}
+const endsTheCall = (err) => isMissingBrowser(err) || err instanceof PageStoppedError;
+
+// A page load measured once, or `count` times with the median run as the answer, as --runs of the CLI.
+// `measure` gets the signal of its run, which closes its Chromium. When the client gives up (a cancel,
+// its timeout, or leaving), the run in progress ends and the runs left never start.
+async function measurePage(measure, count, ctx, limitMs) {
+  const progress = progressFor(ctx, count);
+  const measureRun = async (run) => {
+    const limit = AbortSignal.timeout(limitMs);
+    let measured;
+    try {
+      measured = await measure(progress(run), AbortSignal.any([ctx.mcpReq.signal, limit]));
+    } catch (err) {
+      if (!limit.aborted) throw err;
+    }
+    // The runner keeps a failed evaluation as a result, and closing Chromium fails every one
+    if (limit.aborted) throw new PageStoppedError(stoppedAnswering(limitMs));
+    return measured;
+  };
+  if (count === 1) return measureRun(1);
+  const { payload, runs, errors } = await measureRuns(measureRun, count, { isFatal: endsTheCall, signal: ctx.mcpReq.signal });
+  if (!payload) throw new Error(`Every run failed: ${[...new Set(errors.map(firstLine))].join("; ")}`);
+  return { ...payload, runs };
 }
 
 const CRUX_FORM_FACTORS = { phone: "PHONE", desktop: "DESKTOP", tablet: "TABLET" };
@@ -116,16 +225,86 @@ function describeSnippet(name) {
   };
 }
 
+// The runs of the connection that measured a page, as resources: an agent picks the baseline of a run
+// among them. Each is the JSON the CLI prints with --json, so saved to a file it is a baseline of the
+// CLI too.
+function registerRuns(server, runs) {
+  const JSON_TYPE = "application/json";
+  server.registerResource(
+    "runs",
+    new ResourceTemplate("webperf://runs/{runId}", {
+      list: async () => ({
+        resources: runs.pages().map(({ runId, url, conditions, at, runs: count }) => ({
+          uri: `webperf://runs/${runId}`,
+          name: `${conditions.snippet ?? conditions.workflow} on ${url} (${conditions.viewport})${count ? `, ${count} runs` : ""}`,
+          description: `Measured at ${at}`,
+          mimeType: JSON_TYPE,
+        })),
+      }),
+    }),
+    {
+      title: "Runs of this connection",
+      description: `The page an earlier run_snippet or run_workflow of this connection measured, with its results and conditions, as the JSON of the CLI (--json). Its runId is the baseline argument of a later run. The server keeps the last 20 runs. ${PAGE_DATA}`,
+      mimeType: JSON_TYPE,
+    },
+    async (uri, { runId }) => {
+      let page;
+      try {
+        page = runs.page(runId);
+      } catch (err) {
+        if (err instanceof RunError) throw new ResourceNotFoundError(uri.href);
+        throw err;
+      }
+      return { contents: [{ uri: uri.href, mimeType: JSON_TYPE, text: JSON.stringify(page) }] };
+    }
+  );
+}
+
 // The MCP server of the CLI: the same snippets and runner, offered as tools an agent calls by name.
 // serveStdio creates one instance per connection, so the pages it keeps open belong to that client.
-export function createServer({ launchBrowser = () => launchChromium(), maxPages, idleMs, env = process.env } = {}) {
+export function createServer({ launchBrowser = () => launchChromium(), maxPages, idleMs, env = process.env, machine = machineOf, callLimitMs = CALL_LIMIT_MS, reportEveryMs } = {}) {
   const server = new McpServer({ name: "webperf-snippets", version: cliVersion }, { instructions: INSTRUCTIONS });
   const sessions = createSessions({ launchBrowser, maxPages, idleMs });
   const runs = createRuns();
+  // An agent sends several calls in one turn and the client runs them at once. The calls that make a page
+  // work take turns (run_snippet, run_workflow, open_page, interact): a load or a click competes for the
+  // CPU with any other, which inflates what each page reports. A call keeps its turn only while it runs,
+  // so it can wait only for something that ends on its own: never for a later call of the agent, and
+  // never for a page left open, which keeps no turn. read_results and close_page take no turn: they ask
+  // little of Chromium, and they are how an agent gets out of a page.
+  const queue = createQueue({ reportEveryMs });
+  const inTurn = (ctx, total, task) => queue.run(task, { signal: ctx.mcpReq.signal, onWait: waitProgressFor(ctx, total) });
   // The client left (end of stdin, or a closed transport): close the pages and Chromium
   server.server.onclose = () => sessions.closeAll();
   registerResources(server);
   registerPrompts(server);
+  registerRuns(server, runs);
+
+  // The machine goes in the conditions of every run, as in the JSON of the CLI, so a baseline measured
+  // on another one warns. It is measured once, in the turn of the first run, before its Chromium starts.
+  let machineOfServer;
+  const conditionsOf = (conditions) => measurementConditions({ ...conditions, machine: (machineOfServer ??= machine()) });
+
+  const measureInTurn = (conditions, measure, count, ctx) =>
+    inTurn(ctx, STAGES * count, async () => ({ conditions: conditionsOf(conditions), run: await measurePage(measure, count, ctx, callLimitMs) }));
+
+  // A run of a page load, kept with its page in the shape of the JSON of the CLI: its runId is the
+  // baseline of a later run, and the resources list it
+  const keepPage = (run, results, conditions) => {
+    const page = { url: run.url, navMs: run.navMs, pageErrors: run.pageErrors, ...(run.throttling ? { throttling: run.throttling } : {}), ...(run.runs ? { runs: run.runs } : {}), conditions };
+    const runId = runs.keep(results, page);
+    server.sendResourceListChanged();
+    return { runId, current: { ...page, results } };
+  };
+
+  // The baseline of a tool, read and checked before the page loads; a wrong one is a tool error
+  const baselineFor = ({ baseline, thresholds }) => {
+    if (baseline === undefined) {
+      if (thresholds?.length) throw new Error("thresholds need a baseline");
+      return undefined;
+    }
+    return prepareBaseline({ baseline, thresholds }, runs);
+  };
 
   // Runs a session tool, and turns the errors the model can act on into tool errors
   const withSession = (handler) => async (args, ctx) => {
@@ -162,28 +341,33 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
     {
       title: "Run a snippet",
       description:
-        `Opens the URL in a headless Chromium, runs one web performance snippet on the loaded page and returns the object the snippet returns, as defined in snippets/SCHEMA.md. Use list_snippets for the names. The snippets that record interactions (INP, Interactions, Input-Latency-Breakdown) run on a page opened with open_page instead. ${DETAIL_NOTE} ${PAGE_DATA}`,
+        `Opens the URL in a headless Chromium, runs one web performance snippet on the loaded page and returns the object the snippet returns, as defined in snippets/SCHEMA.md. Use list_snippets for the names. The snippets that record interactions (INP, Interactions, Input-Latency-Breakdown) run on a page opened with open_page instead. ${QUEUE_NOTE} ${BASELINE_NOTE} ${DETAIL_NOTE} ${PAGE_DATA}`,
       inputSchema: z.object({
         url: HttpUrl,
         snippet: z.string().describe("The snippet: its name (LCP-Subparts), an alias (render-blocking) or Category/Name"),
         viewport: z.enum(VIEWPORTS).default("mobile").describe("Device size of the page"),
         waitMs: z.number().int().min(0).max(10000).default(3000).describe("Wait after the load event before running the snippet, in ms"),
+        runs: Runs,
         detail: Detail,
+        baseline: Baseline,
+        thresholds: Thresholds,
       }),
       outputSchema: z.object({
         url: z.string(),
-        runId: SummaryRunId,
+        runId: PageRunId,
         detail: DetailOut,
         snippet: z.string().describe("Category/Name of the snippet that ran"),
         viewport: z.enum(VIEWPORTS),
         navMs: z.number().describe("From the start of the navigation to the end of the wait"),
         pageErrors: z.array(z.string()).describe("Uncaught errors of the page during the run"),
         result: SnippetResult,
+        runs: RunsSummary,
+        baseline: Comparison,
         nextSteps: NextSteps,
       }),
       annotations: { readOnlyHint: true, openWorldHint: true },
     },
-    async ({ url, snippet, viewport, waitMs, detail }, ctx) => {
+    async ({ url, snippet, viewport, waitMs, runs: count, detail, baseline, thresholds }, ctx) => {
       let path;
       try {
         path = resolveSnippetName(snippet);
@@ -194,22 +378,35 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
       if (isTrackingSnippet(path, source)) {
         return toolError(`${path} records interactions: open the page with open_page and this snippet in track, use it with interact, and read it with read_results.`);
       }
+      let base;
+      try {
+        base = baselineFor({ baseline, thresholds });
+      } catch (err) {
+        return toolError(err.message);
+      }
 
       try {
-        const run = await runSnippets({ url, items: [{ id: path, path, source }], waitMs, viewport: VIEWPORT_PRESETS[viewport], onProgress: progressFor(ctx) });
+        const { conditions, run } = await measureInTurn(
+          { viewport, snippet: path, waitMs },
+          (onProgress, signal) => runSnippets({ url, items: [{ id: path, path, source }], waitMs, viewport: VIEWPORT_PRESETS[viewport], onProgress, signal }),
+          count,
+          ctx
+        );
         const shaped = shapeResults(run.results.map(({ id, ...result }) => result), detail);
-        const runId = shaped.detail === "summary" ? runs.keep(run.results) : undefined;
+        const { runId, current } = keepPage(run, run.results, conditions);
         const id = run.results[0].id;
         const nextSteps = followUpCalls({ results: run.results, url, viewport, waitMs, ran: [path] });
         return toolResult({
           url,
-          ...(runId ? { runId } : {}),
+          runId,
           detail: shaped.detail,
           snippet: id,
           viewport,
           navMs: run.navMs,
           pageErrors: run.pageErrors,
           result: shaped.results[0],
+          ...(run.runs ? { runs: run.runs } : {}),
+          ...(base ? { baseline: compareWithBaseline(base, current) } : {}),
           ...(nextSteps.length ? { nextSteps } : {}),
         });
       } catch (err) {
@@ -225,17 +422,20 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
     {
       title: "Run a workflow",
       description:
-        `Opens the URL in a headless Chromium and runs a workflow: a sequence of snippets plus the follow-ups its decision tree adds from the results (for example LCP-Subparts when LCP is over 2.5 s, with the reason). core-web-vitals: LCP and CLS. audit: render-blocking resources, resource hints, critical CSS, TTFB, scripts, images and more. loading: TTFB, FCP, render-blocking, scripts and fonts. interaction: long tasks, animation frames, scroll and layout shifts while it scrolls the page. ${DETAIL_NOTE} ${PAGE_DATA}`,
+        `Opens the URL in a headless Chromium and runs a workflow: a sequence of snippets plus the follow-ups its decision tree adds from the results (for example LCP-Subparts when LCP is over 2.5 s, with the reason). core-web-vitals: LCP and CLS. audit: render-blocking resources, resource hints, critical CSS, TTFB, scripts, images and more. loading: TTFB, FCP, render-blocking, scripts and fonts. interaction: long tasks, animation frames, scroll and layout shifts while it scrolls the page. ${QUEUE_NOTE} ${BASELINE_NOTE} ${DETAIL_NOTE} ${PAGE_DATA}`,
       inputSchema: z.object({
         url: HttpUrl,
         workflow: z.enum(Object.keys(WORKFLOWS)).default("core-web-vitals"),
         viewport: z.enum(VIEWPORTS).default("mobile").describe("Device size of the page"),
         waitMs: z.number().int().min(0).max(10000).default(3000).describe("Wait after the load event before running the workflow, in ms"),
+        runs: Runs,
         detail: Detail,
+        baseline: Baseline,
+        thresholds: Thresholds,
       }),
       outputSchema: z.object({
         url: z.string(),
-        runId: SummaryRunId,
+        runId: PageRunId,
         detail: DetailOut,
         workflow: z.string(),
         viewport: z.enum(VIEWPORTS),
@@ -244,20 +444,26 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
         results: z
           .array(z.looseObject({ id: z.string(), status: z.string() }).meta({ additionalProperties: true }))
           .describe("One object per step, in order, then the follow-ups, each with the reason it was added"),
+        runs: RunsSummary,
+        baseline: Comparison,
         nextSteps: NextSteps,
       }),
       annotations: { readOnlyHint: true, openWorldHint: true },
     },
-    async ({ url, workflow, viewport, waitMs, detail }, ctx) => {
+    async ({ url, workflow, viewport, waitMs, runs: count, detail, baseline, thresholds }, ctx) => {
+      let base;
       try {
-        const run = await runMeasurement({
-          url,
-          workflow: WORKFLOWS[workflow],
-          rules: RULES,
-          waitMs,
-          viewport: VIEWPORT_PRESETS[viewport],
-          onProgress: progressFor(ctx),
-        });
+        base = baselineFor({ baseline, thresholds });
+      } catch (err) {
+        return toolError(err.message);
+      }
+      try {
+        const { conditions, run } = await measureInTurn(
+          { viewport, workflow, waitMs },
+          (onProgress, signal) => runMeasurement({ url, workflow: WORKFLOWS[workflow], rules: RULES, waitMs, viewport: VIEWPORT_PRESETS[viewport], onProgress, signal }),
+          count,
+          ctx
+        );
         // The runner explains a skipped step with the flag of the CLI; here the tools are what the model has
         const results = run.results.map((r) =>
           r.status === "skipped"
@@ -265,7 +471,7 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
             : r
         );
         const shaped = shapeResults(results, detail);
-        const runId = shaped.detail === "summary" ? runs.keep(results) : undefined;
+        const { runId, current } = keepPage(run, results, conditions);
         // The runner already ran the follow-ups of the decision tree; what is left is the steps it skipped
         const skipped = results
           .filter((r) => r.status === "skipped")
@@ -274,12 +480,14 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
         const nextSteps = skipped.length ? [trackingCall({ paths: skipped, url, viewport })] : [];
         return toolResult({
           url,
-          ...(runId ? { runId } : {}),
+          runId,
           ...shaped,
           workflow,
           viewport,
           navMs: run.navMs,
           pageErrors: run.pageErrors,
+          ...(run.runs ? { runs: run.runs } : {}),
+          ...(base ? { baseline: compareWithBaseline(base, current) } : {}),
           ...(nextSteps.length ? { nextSteps } : {}),
         });
       } catch (err) {
@@ -382,27 +590,33 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
         items.push({ id: path, path, source });
       }
 
-      let session;
-      try {
-        session = await sessions.create({ viewport: VIEWPORT_PRESETS[viewport], signal: ctx.mcpReq.signal });
-      } catch (err) {
-        if (isMissingBrowser(err)) return toolError(MISSING_BROWSER);
-        throw err;
-      }
-      try {
-        const navStart = Date.now();
-        await session.page.goto(url, { waitUntil: "load", timeout: NAV_TIMEOUT_MS });
-        if (waitMs > 0) await session.page.waitForTimeout(waitMs);
-        const navMs = Date.now() - navStart;
-        session.tracking = await evaluateItems(session.page, items);
-        session.stopListening();
-        const tracking = session.tracking.map(({ id, status }) => ({ snippet: id, status }));
-        return toolResult({ sessionId: session.id, url, viewport, navMs, tracking });
-      } catch (err) {
-        await sessions.close(session.id);
-        if (/page\.goto/.test(err.message)) return toolError(`Could not load ${url}: ${firstLine(err.message)}`);
-        return toolError(`Opening the page failed: ${firstLine(err.message)}`);
-      }
+      return inTurn(ctx, undefined, async () => {
+        const limit = AbortSignal.timeout(callLimitMs);
+        let session;
+        try {
+          session = await sessions.create({ viewport: VIEWPORT_PRESETS[viewport], signal: AbortSignal.any([ctx.mcpReq.signal, limit]) });
+        } catch (err) {
+          if (isMissingBrowser(err)) return toolError(MISSING_BROWSER);
+          throw err;
+        }
+        try {
+          const navStart = Date.now();
+          await session.page.goto(url, { waitUntil: "load", timeout: NAV_TIMEOUT_MS });
+          if (waitMs > 0) await session.page.waitForTimeout(waitMs);
+          const navMs = Date.now() - navStart;
+          session.tracking = await evaluateItems(session.page, items);
+          // evaluateItems keeps a failed evaluation as a result, and closing the page fails every one
+          if (limit.aborted) return toolError(`Opening the page failed: ${stoppedAnswering(callLimitMs)}`);
+          session.stopListening();
+          const tracking = session.tracking.map(({ id, status }) => ({ snippet: id, status }));
+          return toolResult({ sessionId: session.id, url, viewport, navMs, tracking });
+        } catch (err) {
+          await sessions.close(session.id);
+          if (limit.aborted) return toolError(`Opening the page failed: ${stoppedAnswering(callLimitMs)}`);
+          if (/page\.goto/.test(err.message)) return toolError(`Could not load ${url}: ${firstLine(err.message)}`);
+          return toolError(`Opening the page failed: ${firstLine(err.message)}`);
+        }
+      });
     })
   );
 
@@ -420,16 +634,28 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
       // A click can submit a form or change data on the site, so this tool is not read-only
       annotations: { readOnlyHint: false, openWorldHint: true },
     },
-    withSession(async ({ sessionId, steps }) => {
-      const session = sessions.get(sessionId);
-      try {
-        await runInteractions(session.page, steps);
-      } catch (err) {
-        return toolError(`A step failed: ${firstLine(err.message)}`);
-      }
-      session.hadInput ||= steps.some(isInputStep);
-      await settle(session.page);
-      return toolResult({ sessionId, steps: steps.length, url: session.page.url() });
+    withSession(async ({ sessionId, steps }, ctx) => {
+      // A page that is not open answers now, not after a turn it does not need
+      sessions.get(sessionId);
+      return inTurn(ctx, undefined, async () => {
+        const session = sessions.get(sessionId);
+        // A wait step is time the agent asked for. Closing the page fails whatever it stopped answering.
+        const limitMs = callLimitMs + steps.reduce((sum, step) => sum + (step.ms ?? 0), 0);
+        const limit = AbortSignal.timeout(limitMs);
+        const closePage = () => sessions.close(sessionId);
+        limit.addEventListener("abort", closePage, { once: true });
+        try {
+          await runInteractions(session.page, steps);
+          session.hadInput ||= steps.some(isInputStep);
+          await settle(session.page);
+          if (!limit.aborted) return toolResult({ sessionId, steps: steps.length, url: session.page.url() });
+        } catch (err) {
+          if (!limit.aborted) return toolError(`A step failed: ${firstLine(err.message)}`);
+        } finally {
+          limit.removeEventListener("abort", closePage);
+        }
+        return toolError(`${stoppedAnswering(limitMs)}, and it was closed. Open it again with open_page.`);
+      });
     })
   );
 
@@ -463,7 +689,12 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
         runId: RunId,
         snippet: z.string().describe("The id of the result in the summary, or its script"),
       }),
-      outputSchema: z.object({ runId: z.string(), snippet: z.string().describe("The id of the result in the run"), result: SnippetResult }),
+      outputSchema: z.object({
+        runId: z.string(),
+        snippet: z.string().describe("The id of the result in the run"),
+        medianRun: z.number().optional().describe("With several runs, the one this result comes from: the median run, the only one kept"),
+        result: SnippetResult,
+      }),
       annotations: { readOnlyHint: true, openWorldHint: false },
     },
     async ({ runId, snippet }) => {
@@ -482,7 +713,8 @@ export function createServer({ launchBrowser = () => launchChromium(), maxPages,
           if (!(err instanceof RunError) || !resolved || resolved === snippet) throw err;
           result = runs.get(runId, resolved);
         }
-        return toolResult({ runId, snippet: result.id, result });
+        const medianRun = runs.medianRun(runId);
+        return toolResult({ runId, snippet: result.id, ...(medianRun ? { medianRun } : {}), result });
       } catch (err) {
         if (err instanceof RunError) return toolError(err.message);
         throw err;

package/src/mcp/summary.js

@@ -29,27 +29,49 @@ export function shapeResults(results, detail = "auto", maxFullChars = FULL_UP_TO
 }
 
 // The full results of the last runs of an MCP connection, by run id, so get_details answers with the
-// values the summary came from: measuring again would give other ones
+// values the summary came from: measuring again would give other ones. A run of a page load also keeps
+// its page (url, navMs, pageErrors, conditions), so a later run can take it as its baseline.
 export function createRuns({ max = 20 } = {}) {
   const runs = new Map();
+  const unknown = (runId) =>
+    new RunError(`Unknown runId ${runId}: this server keeps the last ${max} runs of the connection. Run the tool again for a new one.`);
 
-  function keep(results) {
+  function keep(results, page) {
     const runId = randomUUID();
-    runs.set(runId, results);
+    runs.set(runId, { results, page, at: new Date().toISOString() });
     // A Map keeps the order of insertion, so the first key is the oldest run
     if (runs.size > max) runs.delete(runs.keys().next().value);
     return runId;
   }
 
   function get(runId, snippet) {
-    const results = runs.get(runId);
-    if (!results) {
-      throw new RunError(`Unknown runId ${runId}: this server keeps the last ${max} runs of the connection. Run the tool again for a new one.`);
-    }
+    const run = runs.get(runId);
+    if (!run) throw unknown(runId);
+    const { results } = run;
     const result = results.find((r) => r.id === snippet) ?? results.find((r) => r.script === snippet);
     if (!result) throw new RunError(`${snippet} is not in run ${runId}, which has: ${results.map((r) => r.id).join(", ")}.`);
     return result;
   }
 
-  return { keep, get };
+  // A run as the JSON of the CLI gives a page, which is what a baseline is
+  function page(runId) {
+    const run = runs.get(runId);
+    if (!run) throw unknown(runId);
+    if (!run.page) throw new RunError(`Run ${runId} read an open page; a baseline is a run of run_snippet or run_workflow.`);
+    return { ...run.page, results: run.results };
+  }
+
+  // The runs that can be a baseline, the oldest first, with the number of measurements of each when it
+  // has several: the statistics of a comparison need five a side
+  const pages = () =>
+    [...runs]
+      .filter(([, run]) => run.page)
+      .map(([runId, { page, at }]) => ({ runId, url: page.url, conditions: page.conditions, at, ...(page.runs ? { runs: page.runs.count } : {}) }));
+
+  // A measurement of several runs keeps the results of the median one, as the JSON of the CLI does
+  const medianRun = (runId) => runs.get(runId)?.page?.runs?.medianRun;
+
+  const has = (runId) => runs.has(runId);
+
+  return { keep, get, page, pages, medianRun, has };
 }