webmcp-gauge 0.1.0

package/judges/openai-compatible.mjs

@@ -0,0 +1,145 @@
+/**
+ * Judge adapter for any OpenAI-compatible chat-completions endpoint.
+ *
+ * Invocation rate is a property of (page, client, judge model), so the model id
+ * and endpoint are returned with every answer and belong in every report. The
+ * adapter deliberately does no retrying and no repair of malformed output: a
+ * judge that cannot follow the response contract is a measurement result, not
+ * an error to paper over.
+ */
+
+const DEFAULT_TIMEOUT_MS = 60000;
+
+/**
+ * Reasoning models spend completion tokens on thinking before they emit anything,
+ * and the budget covers both. At 1024, glm-5.3 spent 1021 tokens reasoning about
+ * one utterance and returned an empty string with finish_reason "length" - which
+ * looked exactly like a judge declining to pick a tool. A truncated judge is a
+ * harness misconfiguration, so the ceiling is set where truncation is unlikely and
+ * the caller is told when it happens anyway. Unused headroom costs nothing.
+ */
+const DEFAULT_MAX_TOKENS = 4096;
+
+/**
+ * agentrouter.org rejects requests that do not look like a CLI client, with
+ * `401 unauthorized client detected`, even when the key is valid.
+ */
+const HOST_HEADERS = {
+  'agentrouter.org': { 'User-Agent': 'claude-cli/1.0.80 (external, cli)', 'x-app': 'cli' },
+};
+
+const hostHeadersFor = (baseUrl) => {
+  const { hostname } = new URL(baseUrl);
+  return HOST_HEADERS[hostname] ?? {};
+};
+
+/**
+ * Key selection has to be deterministic and reportable. Qwen Code exports one
+ * `QWEN_CUSTOM_API_KEY_*` variable per configured provider, and picking whichever
+ * one enumerates first silently grabbed an Anthropic-scoped key for an OpenAI
+ * endpoint on the first run - it happened to work, which is worse than failing.
+ * Candidates are therefore ranked: explicit variable, then the project's own
+ * variable, then session variables whose name matches this endpoint's host,
+ * preferring the OpenAI-scoped one, in sorted order.
+ */
+const hostTokens = (baseUrl) => {
+  const { hostname } = new URL(baseUrl);
+  return hostname.toUpperCase().split('.').filter((part) => part.length > 2 && part !== 'WWW');
+};
+
+export const resolveApiKey = (env = process.env, explicitVar, baseUrl) => {
+  if (explicitVar) {
+    const value = env[explicitVar];
+    if (!value) throw new Error(`Judge API key variable ${explicitVar} is set to nothing`);
+    return { key: value, source: explicitVar };
+  }
+
+  if (env.WEBMCP_GAUGE_JUDGE_API_KEY) {
+    return { key: env.WEBMCP_GAUGE_JUDGE_API_KEY, source: 'WEBMCP_GAUGE_JUDGE_API_KEY' };
+  }
+
+  const sessionVars = Object.keys(env)
+    .filter((name) => name.startsWith('QWEN_CUSTOM_API_KEY_') && env[name])
+    .sort();
+  const tokens = baseUrl ? hostTokens(baseUrl) : [];
+  const hostMatches = sessionVars.filter((name) => tokens.every((token) => name.includes(token)));
+  const ranked = [
+    ...hostMatches.filter((name) => name.includes('OPENAI')),
+    ...hostMatches.filter((name) => !name.includes('OPENAI')),
+  ];
+
+  if (ranked.length > 0) return { key: env[ranked[0]], source: ranked[0] };
+
+  throw new Error(
+    `No judge API key found for ${baseUrl ?? 'the configured endpoint'}. Set WEBMCP_GAUGE_JUDGE_API_KEY (see .env.example).` +
+      (sessionVars.length > 0 ? ` Session variables present but not host-matched: ${sessionVars.join(', ')}` : '')
+  );
+};
+
+export const createJudge = ({
+  baseUrl,
+  model,
+  apiKeyVar,
+  env = process.env,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+  temperature = 0,
+}) => {
+  if (!baseUrl || !model) throw new Error('createJudge requires baseUrl and model');
+  const { key, source } = resolveApiKey(env, apiKeyVar, baseUrl);
+  const endpoint = `${baseUrl.replace(/\/$/, '')}/chat/completions`;
+
+  return {
+    id: model,
+    baseUrl,
+    keySource: source,
+
+    async complete({ system, user, maxTokens = DEFAULT_MAX_TOKENS }) {
+      const body = {
+        model,
+        temperature,
+        max_tokens: maxTokens,
+        messages: [
+          ...(system ? [{ role: 'system', content: system }] : []),
+          { role: 'user', content: user },
+        ],
+      };
+
+      const startedAt = Date.now();
+      const response = await fetch(endpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${key}`,
+          ...hostHeadersFor(baseUrl),
+        },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+
+      const text = await response.text();
+      const elapsedMs = Date.now() - startedAt;
+
+      if (!response.ok) {
+        // The body carries the useful part - a whitelist rejection reads very
+        // differently from an exhausted quota - so it is surfaced, not swallowed.
+        throw new Error(`judge HTTP ${response.status}: ${text.slice(0, 500)}`);
+      }
+
+      const payload = JSON.parse(text);
+      const message = payload.choices?.[0]?.message ?? {};
+      const finishReason = payload.choices?.[0]?.finish_reason ?? null;
+
+      return {
+        content: (message.content ?? '').trim(),
+        raw: text,
+        model: payload.model ?? model,
+        finishReason,
+        // Truncation is the caller's problem to classify, not something to hide:
+        // an empty answer cut off mid-thought is not a judge declining to answer.
+        truncated: finishReason === 'length',
+        usage: payload.usage ?? null,
+        elapsedMs,
+      };
+    },
+  };
+};

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "webmcp-gauge",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Measures whether an AI agent actually calls the tools a page exposes through WebMCP.",
+  "license": "MIT",
+  "bin": {
+    "webmcp-gauge": "bin/webmcp-gauge.mjs"
+  },
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "files": [
+    "bin",
+    "core",
+    "browser",
+    "judges",
+    "report",
+    "fixtures/airlock.utterances.json",
+    "fixtures/gallery",
+    "fixtures/broken",
+    "fixtures/README.md",
+    "docs",
+    "README.md",
+    "LICENSE",
+    "action.yml"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Svishwa2004/webmcp-gauge.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Svishwa2004/webmcp-gauge/issues"
+  },
+  "homepage": "https://github.com/Svishwa2004/webmcp-gauge#readme",
+  "keywords": [
+    "webmcp",
+    "mcp",
+    "model-context-protocol",
+    "ai-agents",
+    "linter",
+    "measurement",
+    "browser-automation",
+    "invocation-rate"
+  ],
+  "scripts": {
+    "gauge": "node bin/webmcp-gauge.mjs",
+    "test": "node --test"
+  },
+  "dependencies": {
+    "chrome-remote-interface": "0.33.3"
+  }
+}

package/report/badge.mjs

@@ -0,0 +1,110 @@
+/**
+ * The badge a CI run leaves behind.
+ *
+ * A badge is a bare number in a coloured pill, which is exactly the thing this
+ * project refuses to publish. That tension is not resolved by making the badge
+ * prettier; it is resolved by making the badge unable to overstate:
+ *
+ *  - a run with unmeasured trials shows **incomplete**, never a rate, because a
+ *    rate over a denominator the run did not choose is the wrong number however
+ *    it is coloured (this mirrors the exit-code contract, where incomplete
+ *    outranks a breach);
+ *  - the message carries `n`, so a 100% from twenty trials cannot be mistaken
+ *    for a 100% from a thousand;
+ *  - colour is a threshold, not a grade, and the thresholds are stated here
+ *    rather than tuned to flatter a particular subject.
+ *
+ * The output is the Shields "endpoint" schema, which any README can point at
+ * without this project hosting anything, plus a self-contained SVG for repos
+ * that would rather not call out to a third party at page load.
+ */
+
+/** Thresholds are deliberately coarse: a badge is a smoke alarm, not a gauge. */
+export const BADGE_THRESHOLDS = [
+  { atLeast: 0.95, color: 'brightgreen' },
+  { atLeast: 0.85, color: 'green' },
+  { atLeast: 0.7, color: 'yellowgreen' },
+  { atLeast: 0.5, color: 'yellow' },
+  { atLeast: 0.0, color: 'red' },
+];
+
+const colorFor = (rate) => BADGE_THRESHOLDS.find((band) => rate >= band.atLeast)?.color ?? 'lightgrey';
+
+/**
+ * Reads only what a finished report already carries, so a badge can never
+ * disagree with the report beside it.
+ */
+export const buildBadge = (report, { label = 'webmcp invocation' } = {}) => {
+  const overall = report?.invocation?.overall ?? null;
+
+  if (!overall || overall.trials === 0) {
+    return { schemaVersion: 1, label, message: 'no trials', color: 'lightgrey', isError: true };
+  }
+
+  // A schema-2 report has no `coverage` block, and `report/emit.mjs` is explicit
+  // that its absence means **unknown**, not complete — that is why the schema
+  // version moved. Reading a missing block as zero missing trials would publish a
+  // rate for a run whose completeness nobody established, which is the exact
+  // overstatement this file exists to prevent.
+  if (!report.coverage) {
+    return { schemaVersion: 1, label, message: 'coverage unknown', color: 'lightgrey', isError: true };
+  }
+
+  const missing = report.coverage.missingTrials ?? 0;
+  if (missing > 0) {
+    const expected = report.coverage.expectedTrials ?? overall.trials + missing;
+    return {
+      schemaVersion: 1,
+      label,
+      // Named for what it is. "incomplete" is not a bad score, it is the absence
+      // of a score, and a reader must not be able to read it as one.
+      message: `incomplete (${expected - missing}/${expected})`,
+      color: 'lightgrey',
+      isError: true,
+    };
+  }
+
+  const rate = overall.ok / overall.trials;
+  return {
+    schemaVersion: 1,
+    label,
+    message: `${(rate * 100).toFixed(0)}% (n=${overall.trials})`,
+    color: colorFor(rate),
+  };
+};
+
+const PALETTE = {
+  brightgreen: '#4c1',
+  green: '#97ca00',
+  yellowgreen: '#a4a61d',
+  yellow: '#dfb317',
+  red: '#e05d44',
+  lightgrey: '#9f9f9f',
+};
+
+/** Rough advance width for the 11px DejaVu-ish face Shields uses. */
+const textWidth = (text) => Math.round([...text].reduce((sum, ch) => sum + (/[iljI.,:'|]/.test(ch) ? 3 : /[mwMW%]/.test(ch) ? 9 : 6.4), 0)) + 10;
+
+/**
+ * A self-contained SVG, so a README can commit the badge instead of fetching it.
+ * Deliberately plain: no gradients, no external font reference, nothing that
+ * needs a network request to render.
+ */
+export const renderBadgeSvg = (badge) => {
+  const labelWidth = textWidth(badge.label);
+  const messageWidth = textWidth(badge.message);
+  const total = labelWidth + messageWidth;
+  const fill = PALETTE[badge.color] ?? PALETTE.lightgrey;
+  const escape = (text) => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+  return `<svg xmlns="http://www.w3.org/2000/svg" width="${total}" height="20" role="img" aria-label="${escape(badge.label)}: ${escape(badge.message)}">
+  <title>${escape(badge.label)}: ${escape(badge.message)}</title>
+  <rect width="${labelWidth}" height="20" fill="#555"/>
+  <rect x="${labelWidth}" width="${messageWidth}" height="20" fill="${fill}"/>
+  <g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
+    <text x="${labelWidth / 2}" y="14">${escape(badge.label)}</text>
+    <text x="${labelWidth + messageWidth / 2}" y="14">${escape(badge.message)}</text>
+  </g>
+</svg>
+`;
+};

package/report/badge.test.mjs

@@ -0,0 +1,97 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { buildBadge, renderBadgeSvg, BADGE_THRESHOLDS } from './badge.mjs';
+
+const report = ({ ok, trials, missing = 0, expected = trials }) => ({
+  invocation: { overall: { ok, trials } },
+  coverage: missing > 0 ? { missingTrials: missing, expectedTrials: expected } : { missingTrials: 0, expectedTrials: expected },
+});
+
+test('a complete run shows the rate and the trial count, so 20 cannot pass for 1000', () => {
+  const badge = buildBadge(report({ ok: 399, trials: 480 }));
+  assert.equal(badge.message, '83% (n=480)');
+  assert.equal(badge.schemaVersion, 1);
+  assert.ok(!badge.isError);
+});
+
+/**
+ * The rule this file exists for. A rate over a denominator the run did not choose
+ * is the wrong number however it is coloured, and the exit-code contract already
+ * says incomplete outranks everything else.
+ */
+test('a run with unmeasured trials shows incomplete, never a rate', () => {
+  const badge = buildBadge(report({ ok: 100, trials: 100, missing: 60, expected: 160 }));
+  assert.equal(badge.message, 'incomplete (100/160)');
+  assert.equal(badge.color, 'lightgrey');
+  assert.equal(badge.isError, true);
+  assert.ok(!/%/.test(badge.message), 'a percentage must not appear on an incomplete run');
+});
+
+test('a perfect but incomplete run still says incomplete, not 100%', () => {
+  const badge = buildBadge(report({ ok: 20, trials: 20, missing: 140, expected: 160 }));
+  assert.match(badge.message, /^incomplete/);
+});
+
+test('no trials at all is an error state rather than a zero', () => {
+  const badge = buildBadge(report({ ok: 0, trials: 0 }));
+  assert.equal(badge.message, 'no trials');
+  assert.equal(badge.isError, true);
+  const missingReport = buildBadge({});
+  assert.equal(missingReport.message, 'no trials');
+});
+
+/**
+ * Caught by generating a badge from a real published report: the 960-trial
+ * reference run predates schema 3 and carries no `coverage` block at all, and
+ * `report/emit.mjs` says plainly that its absence means unknown rather than
+ * complete. The first version of this file read it as zero missing trials and
+ * happily published "99% (n=840)" for a run whose completeness nobody had
+ * established.
+ */
+test('a schema-2 report with no coverage block says unknown, not a rate', () => {
+  const badge = buildBadge({ invocation: { overall: { ok: 831, trials: 840 } } });
+  assert.equal(badge.message, 'coverage unknown');
+  assert.equal(badge.color, 'lightgrey');
+  assert.equal(badge.isError, true);
+  assert.ok(!/%/.test(badge.message));
+});
+
+test('zero of many is red and says so, which is different from having no trials', () => {
+  const badge = buildBadge(report({ ok: 0, trials: 60 }));
+  assert.equal(badge.message, '0% (n=60)');
+  assert.equal(badge.color, 'red');
+});
+
+test('colour bands are ordered and cover the whole range', () => {
+  const rates = BADGE_THRESHOLDS.map((band) => band.atLeast);
+  assert.deepEqual(rates, [...rates].sort((a, b) => b - a), 'bands must be descending or the first match is wrong');
+  assert.equal(rates.at(-1), 0, 'the last band must catch every remaining rate');
+});
+
+test('the band boundaries are inclusive, so exactly 95% is the better colour', () => {
+  assert.equal(buildBadge(report({ ok: 95, trials: 100 })).color, 'brightgreen');
+  assert.equal(buildBadge(report({ ok: 94, trials: 100 })).color, 'green');
+  assert.equal(buildBadge(report({ ok: 85, trials: 100 })).color, 'green');
+  assert.equal(buildBadge(report({ ok: 60, trials: 100 })).color, 'yellow');
+});
+
+test('the label can be overridden, because a repo may measure more than one page', () => {
+  const badge = buildBadge(report({ ok: 1, trials: 1 }), { label: 'airlock' });
+  assert.equal(badge.label, 'airlock');
+});
+
+test('the SVG is self-contained: no external font, no network reference', () => {
+  const svg = renderBadgeSvg(buildBadge(report({ ok: 399, trials: 480 })));
+  assert.match(svg, /^<svg xmlns="http:\/\/www\.w3\.org\/2000\/svg"/);
+  assert.ok(!/<image|xlink:href|@import|https?:\/\/(?!www\.w3\.org)/.test(svg), 'nothing may be fetched to render this');
+  assert.match(svg, /83% \(n=480\)/);
+  assert.match(svg, /role="img"/);
+  assert.match(svg, /<title>/, 'a badge without a title is unreadable to a screen reader');
+});
+
+test('angle brackets and ampersands in a label cannot break the SVG', () => {
+  const svg = renderBadgeSvg(buildBadge(report({ ok: 1, trials: 1 }), { label: 'a & b <c>' }));
+  assert.match(svg, /a &amp; b &lt;c&gt;/);
+  assert.ok(!/<c>/.test(svg));
+});

package/report/emit.mjs

@@ -0,0 +1,282 @@
+/**
+ * Report emitters.
+ *
+ * A report that does not name its judge, its browser build and its utterance-set
+ * version is not a measurement, because invocation rate is a property of
+ * (page, client, judge, utterances) rather than of the page alone. Every stamp
+ * therefore travels with the numbers, and the Markdown is generated from the same
+ * object as the JSON so the two cannot drift.
+ *
+ * Two spreads are reported and never merged: between-session sigma, which compares
+ * whole sessions that shared nothing but the machine, and within-session sigma,
+ * which compares repeats that shared a browser and a warm cache. The first two
+ * sweeps of this project reported the second and described it as the first.
+ *
+ * Schema 3 adds `coverage` and `gate`: whether the run measured what it planned to,
+ * and the verdict a CI job exits on. In schema 2 reports the absence of `coverage`
+ * means unknown rather than complete, which is why the version moved.
+ */
+import { rollUpControls, rollUpTool } from '../core/stats.mjs';
+
+const percent = (value) =>
+  value === null || value === undefined ? '—' : `${(value * 100).toFixed(1)}%`;
+
+const sigma = (value) => (typeof value === 'number' ? value.toFixed(3) : '—');
+
+const interval = (wilsonResult) =>
+  !wilsonResult || wilsonResult.rate === null
+    ? '—'
+    : `${percent(wilsonResult.rate)} [${percent(wilsonResult.low)}, ${percent(wilsonResult.high)}]`;
+
+export const buildReport = ({
+  fixture,
+  records,
+  harnessFailures = [],
+  recoveredFailures = 0,
+  coverage = null,
+  judge,
+  settings,
+  sessionResults = [],
+  timing,
+}) => {
+  const toolRecords = records.filter((record) => record.kind === 'tool');
+  const controlRecords = records.filter((record) => record.kind === 'control');
+
+  const toolNames = [...new Set(toolRecords.map((record) => record.expectedTool))];
+  const tools = toolNames.map((tool) =>
+    rollUpTool({ tool, records: toolRecords.filter((record) => record.expectedTool === tool) })
+  );
+
+  const byTag = {};
+  for (const record of toolRecords) {
+    const tag = record.tag ?? 'untagged';
+    byTag[tag] ??= { trials: 0, ok: 0 };
+    byTag[tag].trials += 1;
+    if (record.outcome === 'ok') byTag[tag].ok += 1;
+  }
+  for (const tag of Object.keys(byTag)) {
+    byTag[tag].rate = byTag[tag].ok / byTag[tag].trials;
+  }
+
+  const sessionMetas = [];
+  for (const record of records) {
+    const session = record.session ?? 1;
+    if (sessionMetas.some((entry) => entry.session === session)) continue;
+    sessionMetas.push({ session, ...(record.sessionMeta ?? {}) });
+  }
+
+  return {
+    schema: 'webmcp-gauge/report/3',
+    generatedAt: new Date().toISOString(),
+    /**
+     * The subject label is explicit because the utterance set and the page under
+     * test are separable: the frozen Airlock set also drives a deliberately
+     * mis-described local twin, and a report that inherited the fixture's name
+     * would claim to have measured Airlock.
+     */
+    subject: {
+      url: settings.url,
+      name: settings.subjectName ?? fixture.subject?.name ?? null,
+      servedFrom: settings.servedFrom ?? null,
+    },
+    stamps: {
+      utteranceSet: {
+        version: fixture.version,
+        frozen: fixture.frozen === true,
+        authoringModel: fixture.authoring?.modelId ?? null,
+      },
+      judge,
+      harness: {
+        sessions: settings.sessions,
+        repeatsPerSession: settings.repeatsPerSession,
+        concurrency: settings.concurrency,
+        gapSeconds: settings.gapSeconds ?? 0,
+        isolatedSessions: settings.isolatedSessions !== false,
+      },
+      browsers: sessionMetas,
+    },
+    /**
+     * What a session does not isolate. Stated in the artifact rather than in a
+     * commit message, because someone reading the number a month from now is the
+     * person who needs it.
+     */
+    isolationCaveats: [
+      'Sessions share the machine, the OS network stack and the route to the provider.',
+      'Provider-side state (routing, caches, rate-limit counters, model version behind a slug) is not controlled.',
+      settings.gapSeconds > 0
+        ? `Sessions were spaced ${settings.gapSeconds}s apart, which is not the same as spanning days.`
+        : 'Sessions ran back to back, so this measures process and browser independence, not day-to-day drift.',
+      settings.isolatedSessions === false
+        ? 'This run attached to a pre-existing browser (--port), so sessions shared a browser process and page cache: between-session sigma here is not isolated.'
+        : 'Each session ran in its own OS process with its own browser and a cold profile, so no HTTP connection pool, renderer or page cache was shared.',
+    ],
+    invocation: {
+      overall: {
+        trials: toolRecords.length,
+        ok: toolRecords.filter((record) => record.outcome === 'ok').length,
+      },
+      perTool: tools,
+      perTag: byTag,
+    },
+    controls: controlRecords.length > 0 ? rollUpControls({ records: controlRecords }) : null,
+    /**
+     * Whether the run measured what it planned to measure, derived from the plan
+     * against the checkpoint rather than from the failure log. A log only knows
+     * about trials that failed loudly: a session killed mid-plan leaves no entry
+     * and would otherwise report as complete.
+     */
+    coverage,
+    harnessFailures,
+    recoveredFailures,
+    sessionResults,
+    timing,
+  };
+};
+
+export const toMarkdown = (report) => {
+  const lines = [];
+  const { harness } = report.stamps;
+
+  lines.push(`# webmcp-gauge — ${report.subject.name ?? report.subject.url}`);
+  lines.push('');
+  if (report.subject.servedFrom) {
+    lines.push(
+      `Subject: \`${report.subject.url}\`, served locally from \`${report.subject.servedFrom}\` — a fixture page in this repository, not a deployed site.`
+    );
+    lines.push('');
+  }
+  lines.push(
+    `Utterance set \`${report.stamps.utteranceSet.version}\`${report.stamps.utteranceSet.frozen ? ' (frozen)' : ' (DRAFT — numbers are not comparable)'} · judge \`${report.stamps.judge.model}\` · ${harness.sessions} session${harness.sessions === 1 ? '' : 's'} × ${harness.repeatsPerSession} repeat${harness.repeatsPerSession === 1 ? '' : 's'} · concurrency ${harness.concurrency}${harness.isolatedSessions ? '' : ' · **shared browser, sessions not isolated**'}`
+  );
+  lines.push('');
+  lines.push(
+    `Authored by \`${report.stamps.utteranceSet.authoringModel ?? 'unrecorded'}\`, which is disqualified as a judge for these numbers.`
+  );
+  lines.push('');
+
+  if (report.gate) {
+    lines.push(`**Gate:** ${report.gate.summary} _(exit ${report.gate.code})_`);
+    lines.push('');
+  }
+
+  lines.push('## Invocation rate');
+  lines.push('');
+  lines.push('| Tool | Rate (95% Wilson) | σ between sessions | σ within session | Trials | Outcomes |');
+  lines.push('|---|---|---|---|---|---|');
+  for (const tool of [...report.invocation.perTool].sort(
+    (a, b) => (b.invocation.rate ?? 0) - (a.invocation.rate ?? 0)
+  )) {
+    const outcomes = Object.entries(tool.outcomes)
+      .sort((a, b) => b[1] - a[1])
+      .map(([outcome, count]) => `${outcome} ${count}`)
+      .join(', ');
+    lines.push(
+      `| \`${tool.tool}\` | ${interval(tool.invocation)} | ${sigma(tool.betweenSession.sigma)} | ${sigma(tool.withinSession.sigma)} | ${tool.trials} | ${outcomes} |`
+    );
+  }
+  lines.push('');
+  lines.push(
+    '**σ between sessions** compares whole sessions, each with its own process, browser and cold profile — the only figure that speaks to reproducibility. **σ within session** compares repeats that shared a warm page and one provider connection, so it is a floor. Where the within-session column reads `—`, only one repeat per session was run.'
+  );
+  lines.push('');
+
+  lines.push('## By phrasing difficulty');
+  lines.push('');
+  lines.push('| Tag | Rate | Trials |');
+  lines.push('|---|---|---|');
+  for (const tag of ['plain', 'paraphrase', 'oblique']) {
+    const entry = report.invocation.perTag[tag];
+    if (!entry) continue;
+    lines.push(`| ${tag} | ${percent(entry.rate)} | ${entry.trials} |`);
+  }
+  lines.push('');
+
+  if (report.controls) {
+    lines.push('## Control false positives');
+    lines.push('');
+    lines.push(
+      'A control utterance is one no registered tool can serve, so **not selecting anything is the pass**. This rate is never pooled with invocation rate.'
+    );
+    lines.push('');
+    lines.push(
+      `False positive rate: **${interval(report.controls.falsePositiveRate)}** over ${report.controls.trials} trials · σ between sessions ${sigma(report.controls.betweenSession.sigma)}.`
+    );
+    lines.push('');
+    lines.push('| Control class | False positives | Rate |');
+    lines.push('|---|---|---|');
+    for (const [tag, entry] of Object.entries(report.controls.byClass)) {
+      lines.push(`| ${tag} | ${entry.falsePositives}/${entry.trials} | ${interval(entry.rate)} |`);
+    }
+    lines.push('');
+    if (report.controls.injectionFailures.length > 0) {
+      lines.push(
+        `⚠️ **Injection-class false positives — a safety finding, not a scoring miss:** ${report.controls.injectionFailures
+          .map((failure) => `${failure.id} (session ${failure.session}) → \`${failure.selected}\``)
+          .join(', ')}`
+      );
+      lines.push('');
+    }
+  }
+
+  lines.push('## What a session does not isolate');
+  lines.push('');
+  for (const caveat of report.isolationCaveats) lines.push(`- ${caveat}`);
+  lines.push('');
+
+  if (report.stamps.browsers.length > 0) {
+    lines.push('| Session | Browser | Headless | Profile |');
+    lines.push('|---|---|---|---|');
+    for (const entry of report.stamps.browsers) {
+      lines.push(
+        `| ${entry.session} | ${entry.build ?? 'attached'} | ${entry.headless === undefined ? '—' : String(entry.headless)} | ${entry.profileDir ?? entry.note ?? '—'} |`
+      );
+    }
+    lines.push('');
+  }
+
+  if (
+    report.harnessFailures.length > 0 ||
+    report.recoveredFailures > 0 ||
+    (report.coverage?.missingTrials ?? 0) > 0
+  ) {
+    lines.push('## Harness failures');
+    lines.push('');
+    lines.push(
+      'Trials that produced no measurement at all — an unreachable or truncated judge, or a page that never loaded, says nothing about the page under test. These are excluded from every rate above rather than counted as outcomes, and `--resume` retries them.'
+    );
+    lines.push('');
+    if (report.recoveredFailures > 0) {
+      lines.push(
+        `**${report.recoveredFailures} earlier failure${report.recoveredFailures === 1 ? '' : 's'} recovered by \`--resume\`** and are counted in the rates above.`
+      );
+      lines.push('');
+    }
+    if ((report.coverage?.missingTrials ?? 0) > 0) {
+      lines.push(
+        `**${report.coverage.missingTrials} of ${report.coverage.expectedTrials} planned trials have no measurement**, so every rate above is over a denominator this run did not choose. Re-run with \`--resume\`.`
+      );
+      if (report.coverage.missing?.length > 0) {
+        lines.push('');
+        lines.push(
+          `Missing (session:repeat:utterance, first ${report.coverage.missing.length}): ${report.coverage.missing.map((key) => `\`${key}\``).join(', ')}`
+        );
+      }
+      lines.push('');
+    } else if (report.harnessFailures.length === 0) {
+      lines.push('No outstanding gaps: every planned trial has a measurement.');
+      lines.push('');
+    }
+    for (const failure of report.harnessFailures) {
+      lines.push(
+        `- \`${failure.utteranceId}\` (session ${failure.session ?? '?'}, repeat ${failure.repeat}, ${failure.kind ?? 'unknown'}): ${failure.error}`
+      );
+    }
+    lines.push('');
+  }
+
+  lines.push(
+    `_${report.invocation.overall.ok}/${report.invocation.overall.trials} tool trials returned \`ok\`${report.coverage ? ` · ${report.coverage.expectedTrials - report.coverage.missingTrials}/${report.coverage.expectedTrials} planned trials measured` : ''}. Generated ${report.generatedAt} in ${(report.timing.elapsedMs / 1000).toFixed(0)}s._`
+  );
+
+  return `${lines.join('\n')}\n`;
+};