webmcp-gauge 0.1.0

package/bin/webmcp-gauge.test.mjs

@@ -0,0 +1,354 @@
+/**
+ * The exit-code contract, end to end through the CLI.
+ *
+ * These tests drive the real binary, with a pre-seeded checkpoint and `--port 1`
+ * so no browser is launched and no judge is called: every planned trial is already
+ * in the checkpoint, so `--resume` has nothing to run. That makes the contract a
+ * CI job depends on - 0 pass, 1 breach, 2 unmeasurable - testable in milliseconds
+ * and without a provider, which is the only reason it can be a test at all rather
+ * than a paragraph in a log entry.
+ */
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildPlan } from '../core/sweep.mjs';
+
+const binPath = fileURLToPath(new URL('./webmcp-gauge.mjs', import.meta.url));
+const fixture = JSON.parse(
+  await readFile(new URL('../fixtures/airlock.utterances.json', import.meta.url), 'utf8')
+);
+
+const GATED_TOOL = 'top_expenses';
+
+/**
+ * Synthesises the checkpoint a finished run would have left, from the same plan
+ * builder the sweep uses - so a fixture change moves the expectation with it
+ * instead of leaving a stale hand-written id list behind.
+ */
+const seedCheckpoint = async ({ dir, ok, drop = 0 }) => {
+  const plan = buildPlan({
+    fixture,
+    repeatsPerSession: 1,
+    tools: [GATED_TOOL],
+    includeControls: false,
+  });
+  const kept = plan.slice(0, plan.length - drop);
+  const lines = kept.map((item, index) =>
+    JSON.stringify({
+      session: 1,
+      repeat: item.repeat,
+      kind: item.kind,
+      utteranceId: item.utterance.id,
+      tag: item.utterance.tag ?? null,
+      expectedTool: item.toolName,
+      outcome: index < ok ? 'ok' : 'wrong_tool',
+      reason: 'synthetic record: exit-code contract test, no judge involved',
+      selection: { tool: index < ok ? item.toolName : 'describe_dataset', arguments: {} },
+    })
+  );
+  await writeFile(join(dir, 'sweep.jsonl'), `${lines.join('\n')}\n`, 'utf8');
+  return { planned: plan.length, seeded: kept.length };
+};
+
+const runCli = (dir, extra = []) =>
+  new Promise((resolve) => {
+    const child = spawn(
+      process.execPath,
+      [
+        binPath,
+        'run',
+        '--resume',
+        // Port 1 answers nothing, which is deliberate: nothing should need it.
+        '--port',
+        '1',
+        '--sessions',
+        '1',
+        '--repeats',
+        '1',
+        '--tools',
+        GATED_TOOL,
+        '--no-controls',
+        '--judge',
+        'exit-code-contract-judge',
+        '--base-url',
+        'https://judge.invalid/v1',
+        '--out',
+        dir,
+        ...extra,
+      ],
+      {
+        env: { ...process.env, WEBMCP_GAUGE_JUDGE_API_KEY: 'never-sent-anywhere' },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('close', (code) => resolve({ code, stdout, stderr }));
+  });
+
+/**
+ * The static lint path needs no browser, no judge and no key, which is the claim
+ * being tested as much as the exit code: L0 is the free on-ramp, so no judge
+ * variable is passed here at all.
+ */
+const runLint = (extra = []) =>
+  new Promise((resolve) => {
+    const child = spawn(
+      process.execPath,
+      [binPath, 'lint', '--manifest', fileURLToPath(new URL('../fixtures/broken/tools.json', import.meta.url)), ...extra],
+      {
+        env: {
+          ...process.env,
+          WEBMCP_GAUGE_JUDGE_MODEL: '',
+          WEBMCP_GAUGE_JUDGE_BASE_URL: '',
+          WEBMCP_GAUGE_JUDGE_API_KEY: '',
+        },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('close', (code) => resolve({ code, stdout, stderr }));
+  });
+
+const withTempRun = async (body) => {
+  const dir = await mkdtemp(join(tmpdir(), 'webmcp-gauge-exit-'));
+  try {
+    return await body(dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+};
+
+const readReport = async (dir) => JSON.parse(await readFile(join(dir, 'report.json'), 'utf8'));
+const readBadge = async (dir) => JSON.parse(await readFile(join(dir, 'badge.json'), 'utf8'));
+
+test('a complete run above the threshold exits 0', async () => {
+  await withTempRun(async (dir) => {
+    const { planned, seeded } = await seedCheckpoint({ dir, ok: 20 });
+    assert.equal(seeded, planned, 'this case is only meaningful with the full plan measured');
+
+    const result = await runCli(dir, ['--fail-under', '0.9']);
+
+    assert.equal(result.code, 0, result.stderr);
+    assert.match(result.stderr, /gate: PASS —/);
+
+    const report = await readReport(dir);
+    assert.equal(report.gate.code, 0);
+    assert.equal(report.coverage.missingTrials, 0);
+    assert.equal(report.coverage.expectedTrials, planned);
+  });
+});
+
+/**
+ * The badge is written by the same command that writes the report, so these two
+ * cases check the wiring rather than the formatting (report/badge.test.mjs owns
+ * that): a complete run gets a rate, and an incomplete one must not.
+ */
+test('a complete run leaves a badge carrying the rate and the trial count', async () => {
+  await withTempRun(async (dir) => {
+    const { planned } = await seedCheckpoint({ dir, ok: 20 });
+
+    const result = await runCli(dir, ['--badge-label', 'airlock']);
+
+    assert.equal(result.code, 0, result.stderr);
+    const badge = await readBadge(dir);
+    assert.equal(badge.label, 'airlock');
+    assert.equal(badge.message, `100% (n=${planned})`);
+    assert.equal(badge.color, 'brightgreen');
+
+    const svg = await readFile(join(dir, 'badge.svg'), 'utf8');
+    assert.match(svg, /<svg/);
+    assert.match(svg, new RegExp(`100% \\(n=${planned}\\)`));
+  });
+});
+
+test('an incomplete run leaves a badge that says incomplete, not a rate', async () => {
+  await withTempRun(async (dir) => {
+    const { planned } = await seedCheckpoint({ dir, ok: 12, drop: 8 });
+
+    const result = await runCli(dir);
+
+    assert.equal(result.code, 2, result.stderr);
+    const badge = await readBadge(dir);
+    assert.equal(badge.message, `incomplete (${planned - 8}/${planned})`);
+    assert.equal(badge.isError, true);
+    assert.ok(!/%/.test(badge.message), 'an unmeasured run must not publish a percentage');
+  });
+});
+
+/**
+ * Found by CI, not by reasoning: Chrome failed to start on a runner, the CLI's
+ * top-level await turned it into an unhandled rejection, Node exited 1, and the
+ * Action announced "the manifest has findings" for a lint that never ran. A broken
+ * environment must never be reportable as a bad page.
+ */
+test('a browser that cannot start is exit 2, not a threshold breach', async () => {
+  const root = fileURLToPath(new URL('..', import.meta.url));
+  const result = await new Promise((resolve) => {
+    const child = spawn(
+      process.execPath,
+      [binPath, 'lint', '--serve', 'fixtures/broken', '--url', 'twin.html?variant=clean'],
+      {
+        cwd: root,
+        env: {
+          ...process.env,
+          // A path no browser lives at. Everything else about the invocation is
+          // exactly what CI runs.
+          WEBMCP_GAUGE_CHROME: join(root, 'no-such-chrome-binary'),
+          WEBMCP_GAUGE_LAUNCH_TIMEOUT_MS: '2000',
+        },
+      }
+    );
+    let stderr = '';
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('close', (code) => resolve({ code, stderr }));
+  });
+
+  assert.equal(result.code, 2, `expected "could not measure", got ${result.code}: ${result.stderr}`);
+  assert.match(result.stderr, /could not measure/);
+});
+
+test('a complete run with a rate below --fail-under exits 1', async () => {
+  await withTempRun(async (dir) => {
+    await seedCheckpoint({ dir, ok: 16 });
+
+    const result = await runCli(dir, ['--fail-under', '0.9']);
+
+    assert.equal(result.code, 1, result.stderr);
+    assert.match(result.stderr, /gate: FAIL —/);
+    assert.match(result.stderr, new RegExp(`\`${GATED_TOOL}\` 80\\.0%`));
+
+    const report = await readReport(dir);
+    assert.equal(report.gate.status, 'breach');
+    assert.equal(report.gate.breaches[0].tool, GATED_TOOL);
+    assert.equal(report.coverage.missingTrials, 0);
+  });
+});
+
+test('trials that could not be measured exit 2, not 1', async () => {
+  await withTempRun(async (dir) => {
+    // Three trials are missing from the checkpoint, so the resumed session tries to
+    // run them and cannot: port 1 has no browser. That is the real unmeasurable
+    // path, not a simulated one.
+    await seedCheckpoint({ dir, ok: 17, drop: 3 });
+
+    const result = await runCli(dir, ['--fail-under', '0.9']);
+
+    assert.equal(result.code, 2, result.stderr);
+    assert.match(result.stderr, /gate: INCOMPLETE —/);
+    assert.match(result.stderr, /3 of 20 planned trials produced no measurement/);
+    assert.match(result.stderr, /trial_threw/);
+    assert.match(result.stderr, /not a threshold breach/);
+
+    const report = await readReport(dir);
+    assert.equal(report.gate.code, 2);
+    assert.equal(report.coverage.missingTrials, 3);
+    assert.equal(report.harnessFailures.length, 3);
+
+    // One line per failed trial, written during the session rather than at the end
+    // of it. A double-append would show six, and an end-of-session append would
+    // lose all three if the process were killed first.
+    const log = await readFile(join(dir, 'harness-failures.jsonl'), 'utf8');
+    assert.equal(log.trim().split('\n').length, 3);
+  });
+});
+
+test('an incomplete run exits 2 even with no threshold, where it used to exit 1', async () => {
+  await withTempRun(async (dir) => {
+    await seedCheckpoint({ dir, ok: 19, drop: 1 });
+
+    const result = await runCli(dir);
+
+    assert.equal(result.code, 2, result.stderr);
+    assert.equal((await readReport(dir)).gate.failUnder, null);
+  });
+});
+
+test('a complete run with no threshold exits 0 and stamps the verdict into the artifact', async () => {
+  await withTempRun(async (dir) => {
+    await seedCheckpoint({ dir, ok: 12 });
+
+    const result = await runCli(dir);
+
+    assert.equal(result.code, 0, result.stderr);
+    assert.match(result.stdout, /\*\*Gate:\*\* COMPLETE —/);
+
+    const report = await readReport(dir);
+    assert.equal(report.schema, 'webmcp-gauge/report/3');
+    assert.equal(report.gate.status, 'pass');
+    assert.equal(report.gate.gatedTools, 0);
+  });
+});
+
+test('--fail-under 90 is refused rather than gating every build against 9000%', async () => {
+  await withTempRun(async (dir) => {
+    await seedCheckpoint({ dir, ok: 20 });
+
+    const result = await runCli(dir, ['--fail-under', '90']);
+
+    assert.equal(result.code, 2);
+    assert.match(result.stderr, /use 0\.9, not 90/);
+  });
+});
+
+test('lint runs with no judge configured at all and exits 0 on a clean manifest', async () => {
+  const result = await runLint(['--variant', 'clean']);
+
+  assert.equal(result.code, 0, result.stderr);
+  assert.match(result.stdout, /No findings/);
+});
+
+test('lint exits 1 on a manifest with error-level findings, and says it is not a measured rate', async () => {
+  const result = await runLint(['--variant', 'degraded']);
+
+  assert.equal(result.code, 1, result.stderr);
+  assert.match(result.stdout, /ERROR\s+name\/invalid-characters/);
+  assert.match(result.stderr, /not a measured invocation rate/);
+});
+
+test('--fail-on warning promotes advice to a failure, and the default does not', async () => {
+  // The clean manifest carries no findings at all until the description floor is
+  // raised past what the reference page ships, which then makes every description
+  // thin: warnings only, so the two --fail-on levels are separable on one input.
+  const advisory = await runLint(['--variant', 'clean', '--min-description', '400']);
+  assert.equal(advisory.code, 0, 'warnings alone must not fail a build by default');
+  assert.match(advisory.stdout, /WARN\s+description\/thin/);
+
+  const strict = await runLint(['--variant', 'clean', '--min-description', '400', '--fail-on', 'warning']);
+  assert.equal(strict.code, 1, strict.stderr);
+});
+
+test('lint exits 2 when the manifest cannot be read as one', async () => {
+  const result = await runLint(['--variant', 'no-such-variant']);
+
+  assert.equal(result.code, 2);
+  assert.match(result.stderr, /has no variants\.no-such-variant array/);
+});
+
+test('lint refuses a --fail-on level it does not implement rather than guessing', async () => {
+  const result = await runLint(['--variant', 'clean', '--fail-on', 'nit']);
+
+  assert.equal(result.code, 2);
+  assert.match(result.stderr, /takes 'error' or 'warning'/);
+});

package/browser/launch.mjs

@@ -0,0 +1,188 @@
+/**
+ * Launches and tears down one flagged Chrome per measurement session.
+ *
+ * This exists because of what the two 1.2.0/1.3.0 sweeps showed: repeats that
+ * share a browser process, a warm page cache and one provider session are
+ * correlated by construction, so the sigma computed across them describes session
+ * stability rather than run-to-run stability. A session needs its own browser and
+ * its own cold cache before its variance means anything.
+ *
+ * Measured 2026-08-30: a brand-new user-data-dir containing nothing but
+ * {"browser":{"enabled_labs_experiments":["enable-webmcp-testing@1"]}} in
+ * `Local State` is enough for Chrome 152 to expose document.modelContext, and it
+ * works in --headless=new. No copied profile, no flag UI, and a genuinely cold
+ * cache per session.
+ */
+import { spawn } from 'node:child_process';
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { createServer } from 'node:net';
+import { resolve } from 'node:path';
+
+const DEFAULT_CHROME =
+  process.env.WEBMCP_GAUGE_CHROME ?? 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe';
+
+const WEBMCP_FLAG_PROFILE = JSON.stringify({
+  browser: { enabled_labs_experiments: ['enable-webmcp-testing@1'] },
+});
+
+export const findFreePort = () =>
+  new Promise((resolve, reject) => {
+    const server = createServer();
+    server.unref();
+    server.on('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const { port } = server.address();
+      server.close(() => resolve(port));
+    });
+  });
+
+const waitForDevTools = async (port, { timeoutMs = 30000, pollTimeoutMs = 2000 } = {}) => {
+  const deadline = Date.now() + timeoutMs;
+  let lastError = 'never answered';
+
+  while (Date.now() < deadline) {
+    try {
+      // Each poll is bounded on its own. A deadline around an unbounded fetch is
+      // not a deadline: one request that never answers holds the loop open past it
+      // forever, which is how a launch turns into a silent hang instead of an error.
+      const response = await fetch(`http://127.0.0.1:${port}/json/version`, {
+        signal: AbortSignal.timeout(Math.min(pollTimeoutMs, Math.max(250, deadline - Date.now()))),
+      });
+      if (response.ok) {
+        const payload = await response.json();
+        return { build: payload.Browser, protocol: payload['Protocol-Version'] };
+      }
+      lastError = `HTTP ${response.status}`;
+    } catch (error) {
+      lastError = String(error.message ?? error);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+
+  throw new Error(`Chrome did not expose DevTools on ${port} within ${timeoutMs}ms (${lastError})`);
+};
+
+/**
+ * Windows leaves Chrome's renderer and GPU children alive when only the parent is
+ * killed, and those children keep the profile directory locked, which then fails
+ * the cleanup and silently reuses a warm profile next session.
+ *
+ * Exported because the orchestrator needs the same thing for a session process:
+ * killing the node child alone would orphan the Chrome it launched.
+ */
+export const killTree = (pid, { timeoutMs = 10000 } = {}) =>
+  new Promise((resolve) => {
+    if (process.platform !== 'win32') {
+      try {
+        process.kill(-pid, 'SIGKILL');
+      } catch {
+        try {
+          process.kill(pid, 'SIGKILL');
+        } catch {
+          // Already gone.
+        }
+      }
+      resolve();
+      return;
+    }
+    const killer = spawn('taskkill', ['/PID', String(pid), '/T', '/F'], { stdio: 'ignore' });
+    // Even the killer gets a deadline: cleanup that can hang is one more way for a
+    // sweep to stop without saying anything.
+    const timer = setTimeout(() => resolve(), timeoutMs);
+    const done = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    killer.on('close', done);
+    killer.on('error', done);
+  });
+
+export const launchSession = async ({
+  profileDir,
+  port,
+  headless = true,
+  chromePath = DEFAULT_CHROME,
+  extraArgs = [],
+  keepProfile = false,
+} = {}) => {
+  const chosenPort = port ?? (await findFreePort());
+  // Chrome must be given an absolute --user-data-dir: with a relative one it can
+  // start against a different directory than the one seeded with the flag, or fail
+  // to start at all, and the only symptom is a DevTools port that never answers.
+  const dir = resolve(profileDir ?? `artifacts/sessions/session-${chosenPort}`);
+
+  await rm(dir, { recursive: true, force: true });
+  await mkdir(dir, { recursive: true });
+  await writeFile(`${dir}/Local State`, WEBMCP_FLAG_PROFILE, 'utf8');
+
+  const args = [
+    `--remote-debugging-port=${chosenPort}`,
+    `--user-data-dir=${dir}`,
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-background-networking',
+    '--disable-sync',
+    ...(headless ? ['--headless=new'] : []),
+    ...extraArgs,
+    'about:blank',
+  ];
+
+  const child = spawn(chromePath, args, {
+    // Chrome's own stderr is the only place a launch failure explains itself, and
+    // swallowing it turns any startup problem into an unhelpful timeout — which is
+    // exactly what happened in CI on 2026-09-02: "did not expose DevTools within
+    // 30000ms" with no reason attached. So it is captured always, and the tail is
+    // attached to the error; `WEBMCP_GAUGE_CHROME_LOG` still streams it live.
+    stdio: process.env.WEBMCP_GAUGE_CHROME_LOG ? 'inherit' : ['ignore', 'ignore', 'pipe'],
+    detached: process.platform !== 'win32',
+  });
+
+  let stderr = '';
+  child.stderr?.on('data', (chunk) => {
+    stderr = `${stderr}${chunk}`.slice(-4000);
+  });
+
+  let exited = null;
+  child.on('exit', (code, signal) => {
+    exited = { code, signal };
+  });
+
+  try {
+    const version = await waitForDevTools(chosenPort, {
+      // CI runners are slower and more variable than a developer's machine, and a
+      // fixed 30 s is a coin toss there rather than a diagnosis.
+      timeoutMs: Number(process.env.WEBMCP_GAUGE_LAUNCH_TIMEOUT_MS ?? 30000),
+    });
+    return {
+      port: String(chosenPort),
+      pid: child.pid,
+      profileDir: dir,
+      headless,
+      build: version.build,
+      protocol: version.protocol,
+      startedAt: new Date().toISOString(),
+      exitInfo: () => exited,
+      async close() {
+        if (child.pid) await killTree(child.pid);
+        if (!keepProfile) {
+          // Chrome releases the profile lock asynchronously; one retry is enough
+          // in practice and a failure here is not worth aborting a sweep over.
+          await new Promise((resolve) => setTimeout(resolve, 300));
+          await rm(dir, { recursive: true, force: true }).catch(async () => {
+            await new Promise((resolve) => setTimeout(resolve, 1500));
+            await rm(dir, { recursive: true, force: true }).catch(() => {});
+          });
+        }
+      },
+    };
+  } catch (error) {
+    if (child.pid) await killTree(child.pid);
+    // Say why, not just that. The exit status and Chrome's own last words are the
+    // difference between "this runner is slow" and "this build cannot start here".
+    const detail = [
+      exited ? `chrome exited code=${exited.code} signal=${exited.signal}` : 'chrome was still running',
+      stderr.trim() ? `chrome stderr (tail): ${stderr.trim().split(/\r?\n/).slice(-6).join(' | ')}` : 'chrome wrote nothing to stderr',
+    ].join('; ');
+    throw new Error(`${error.message} — ${detail}`);
+  }
+};

package/browser/serve.mjs

@@ -0,0 +1,78 @@
+/**
+ * A static file server for fixture pages, so a measurement can run against a page
+ * this repo controls.
+ *
+ * WebMCP needs a real origin: Chrome does not expose `document.modelContext` to
+ * `file://`, and the fixture fetches a CSV beside itself, which `file://` blocks
+ * anyway. This is deliberately the smallest thing that answers GET for a known set
+ * of extensions - it serves fixtures on 127.0.0.1 and nothing else, so it is not a
+ * general web server and must not grow into one.
+ *
+ * Port 0 lets the OS pick, which matters when sessions run concurrently: each
+ * session process starts its own server, exactly as it starts its own browser.
+ */
+import { createServer } from 'node:http';
+import { readFile, stat } from 'node:fs/promises';
+import { extname, join, normalize, resolve, sep } from 'node:path';
+
+const TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.mjs': 'text/javascript; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.csv': 'text/csv; charset=utf-8',
+  '.svg': 'image/svg+xml',
+};
+
+export const startFixtureServer = async ({ root, port = 0, host = '127.0.0.1' }) => {
+  const rootPath = resolve(root);
+  const rootStat = await stat(rootPath).catch(() => null);
+  if (!rootStat?.isDirectory()) throw new Error(`--serve needs a directory, got ${rootPath}`);
+
+  const server = createServer(async (request, response) => {
+    const requested = decodeURIComponent(new URL(request.url, 'http://127.0.0.1').pathname);
+    const relative = normalize(requested).replace(/^([/\\])+/, '');
+    const filePath = join(rootPath, relative === '' ? 'index.html' : relative);
+
+    // A fixture server that can be walked out of its own directory is a hole, even
+    // on localhost: the harness runs it with the repo one level up.
+    if (filePath !== rootPath && !filePath.startsWith(rootPath + sep)) {
+      response.writeHead(403).end('outside the fixture root');
+      return;
+    }
+
+    try {
+      const body = await readFile(filePath);
+      response.writeHead(200, {
+        'Content-Type': TYPES[extname(filePath).toLowerCase()] ?? 'application/octet-stream',
+        'Content-Length': body.length,
+        'Cache-Control': 'no-store',
+      });
+      response.end(body);
+    } catch (error) {
+      response.writeHead(error.code === 'ENOENT' ? 404 : 500).end(String(error.code ?? error));
+    }
+  });
+
+  await new Promise((ready, failed) => {
+    server.once('error', failed);
+    server.listen(port, host, ready);
+  });
+
+  const address = server.address();
+
+  return {
+    port: address.port,
+    origin: `http://${host}:${address.port}`,
+    /** Resolves a fixture-relative path (query string included) against this server. */
+    urlFor(pathAndQuery) {
+      return new URL(String(pathAndQuery).replace(/^\/+/, ''), `http://${host}:${address.port}/`).href;
+    },
+    close: () =>
+      new Promise((closed) => {
+        server.closeAllConnections?.();
+        server.close(closed);
+      }),
+  };
+};