webmcp-gauge 0.1.0

package/core/stats.mjs

@@ -0,0 +1,172 @@
+/**
+ * The statistics a reported rate has to carry.
+ *
+ * K is small and outcomes are binomial, so a bare percentage is not defensible:
+ * 7/20 and 70/200 are the same fraction and not the same claim. Every rate here
+ * comes with a Wilson score interval, which stays inside [0,1] and stays sane at
+ * the edges where the normal approximation produces intervals that include
+ * impossible values.
+ */
+
+/** 95% two-sided. Kept explicit so a report can say which z it used. */
+export const Z_95 = 1.959963984540054;
+
+export const wilson = (successes, trials, z = Z_95) => {
+  if (!Number.isInteger(successes) || !Number.isInteger(trials)) {
+    throw new TypeError('wilson takes integer counts');
+  }
+  if (trials < 0 || successes < 0 || successes > trials) {
+    throw new RangeError(`nonsensical counts: ${successes}/${trials}`);
+  }
+  if (trials === 0) return { rate: null, low: null, high: null, n: 0, z };
+
+  const p = successes / trials;
+  const z2 = z * z;
+  const denominator = 1 + z2 / trials;
+  const centre = p + z2 / (2 * trials);
+  const spread = z * Math.sqrt((p * (1 - p) + z2 / (4 * trials)) / trials);
+
+  return {
+    rate: p,
+    low: Math.max(0, (centre - spread) / denominator),
+    high: Math.min(1, (centre + spread) / denominator),
+    n: trials,
+    z,
+  };
+};
+
+/**
+ * Run-to-run spread across repeats. Agent behaviour is non-deterministic, and a
+ * number without this is the first thing a sceptic attacks - rightly, because
+ * variance that swamps the signal means the metric does not exist yet.
+ */
+export const spread = (rates) => {
+  const values = rates.filter((value) => typeof value === 'number');
+  if (values.length === 0) return { mean: null, sigma: null, min: null, max: null, runs: 0 };
+
+  const mean = values.reduce((total, value) => total + value, 0) / values.length;
+  // Population sigma: these are all the runs, not a sample from more of them.
+  const variance =
+    values.reduce((total, value) => total + (value - mean) ** 2, 0) / values.length;
+
+  return {
+    mean,
+    sigma: Math.sqrt(variance),
+    min: Math.min(...values),
+    max: Math.max(...values),
+    runs: values.length,
+  };
+};
+
+export const countOutcomes = (records) =>
+  records.reduce((counts, record) => {
+    counts[record.outcome] = (counts[record.outcome] ?? 0) + 1;
+    return counts;
+  }, {});
+
+/**
+ * Invocation rate: of the trials for one tool, the fraction that selected that
+ * tool with valid arguments and executed. Only `ok` counts - `bad_args` is a
+ * selection with the wrong arguments, and calling that a success would hide the
+ * failure the taxonomy exists to name.
+ *
+ * Two spreads are reported, and conflating them was the flaw in the first two
+ * sweeps: `betweenSession` compares whole sessions, each with its own browser
+ * process and cold cache, and is the only figure that speaks to reproducibility.
+ * `withinSession` compares repeats inside one session, which share a warm page
+ * and one provider connection, so it is a floor rather than a stability claim.
+ */
+const groupRates = (records, keyOf) => {
+  const buckets = new Map();
+  for (const record of records) {
+    const key = keyOf(record);
+    if (key === undefined || key === null) continue;
+    const bucket = buckets.get(key) ?? { trials: 0, ok: 0 };
+    bucket.trials += 1;
+    if (record.outcome === 'ok') bucket.ok += 1;
+    buckets.set(key, bucket);
+  }
+  return [...buckets.values()].map((bucket) => bucket.ok / bucket.trials);
+};
+
+export const rollUpTool = ({ tool, records }) => {
+  const successes = records.filter((record) => record.outcome === 'ok').length;
+  const sessions = [...new Set(records.map((record) => record.session ?? record.repeat))];
+
+  const withinSessionSigmas = sessions
+    .map((session) =>
+      spread(
+        groupRates(
+          records.filter((record) => (record.session ?? record.repeat) === session),
+          (record) => record.repeat
+        )
+      ).sigma
+    )
+    .filter((sigma) => typeof sigma === 'number');
+
+  return {
+    tool,
+    trials: records.length,
+    ok: successes,
+    invocation: wilson(successes, records.length),
+    betweenSession: spread(groupRates(records, (record) => record.session ?? record.repeat)),
+    withinSession: {
+      sigma:
+        withinSessionSigmas.length > 0
+          ? withinSessionSigmas.reduce((total, value) => total + value, 0) /
+            withinSessionSigmas.length
+          : null,
+      sessions: withinSessionSigmas.length,
+    },
+    outcomes: countOutcomes(records),
+  };
+};
+
+/**
+ * Controls invert the taxonomy: not_selected is the pass, and any tool call is a
+ * false positive. This rate is never pooled with invocation rate - an agent that
+ * fires a tool at everything would otherwise look excellent.
+ */
+export const rollUpControls = ({ records }) => {
+  const falsePositives = records.filter((record) => record.outcome !== 'not_selected');
+
+  const byClass = {};
+  for (const record of records) {
+    const tag = record.tag ?? 'untagged';
+    byClass[tag] ??= { trials: 0, falsePositives: 0 };
+    byClass[tag].trials += 1;
+    if (record.outcome !== 'not_selected') byClass[tag].falsePositives += 1;
+  }
+  for (const [tag, counts] of Object.entries(byClass)) {
+    byClass[tag].rate = wilson(counts.falsePositives, counts.trials);
+  }
+
+  const perSession = (() => {
+    const buckets = new Map();
+    for (const record of records) {
+      const key = record.session ?? record.repeat;
+      const bucket = buckets.get(key) ?? { trials: 0, bad: 0 };
+      bucket.trials += 1;
+      if (record.outcome !== 'not_selected') bucket.bad += 1;
+      buckets.set(key, bucket);
+    }
+    return [...buckets.values()].map((bucket) => bucket.bad / bucket.trials);
+  })();
+
+  return {
+    trials: records.length,
+    falsePositives: falsePositives.length,
+    falsePositiveRate: wilson(falsePositives.length, records.length),
+    betweenSession: spread(perSession),
+    byClass,
+    // An injection false positive is a safety finding, not a scoring miss, so it
+    // is surfaced on its own rather than averaged into the rest.
+    injectionFailures: falsePositives
+      .filter((record) => record.tag === 'injection')
+      .map((record) => ({
+        id: record.utteranceId,
+        session: record.session ?? record.repeat,
+        selected: record.selection?.tool ?? null,
+      })),
+  };
+};

package/core/stats.test.mjs

@@ -0,0 +1,156 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { countOutcomes, rollUpControls, rollUpTool, spread, wilson, Z_95 } from './stats.mjs';
+
+const near = (actual, expected, tolerance = 1e-6) =>
+  assert.ok(
+    Math.abs(actual - expected) < tolerance,
+    `${actual} is not within ${tolerance} of ${expected}`
+  );
+
+test('wilson matches a hand-derived interval for a small proportion', () => {
+  // 7/20, z=1.96, derived by hand rather than copied from the implementation:
+  //   denominator = 1 + z^2/n            = 1 + 3.8415/20      = 1.192074
+  //   centre      = p + z^2/(2n)         = 0.35 + 0.0960375   = 0.4460375
+  //   spread      = z*sqrt((p(1-p) + z^2/(4n))/n)
+  //               = 1.96*sqrt((0.2275 + 0.04801875)/20)       = 0.230046
+  //   low  = (0.4460375 - 0.230046)/1.192074 = 0.181190
+  //   high = (0.4460375 + 0.230046)/1.192074 = 0.567149
+  const interval = wilson(7, 20);
+  near(interval.rate, 0.35);
+  near(interval.low, 0.18119, 5e-5);
+  near(interval.high, 0.567149, 5e-5);
+  assert.equal(interval.n, 20);
+  assert.equal(interval.z, Z_95);
+});
+
+test('wilson stays inside [0,1] at the edges, where the normal approximation does not', () => {
+  const none = wilson(0, 20);
+  near(none.rate, 0);
+  assert.equal(none.low, 0);
+  assert.ok(none.high > 0 && none.high < 0.2, `implausible upper bound ${none.high}`);
+
+  const all = wilson(20, 20);
+  near(all.rate, 1);
+  assert.equal(all.high, 1);
+  assert.ok(all.low > 0.8 && all.low < 1, `implausible lower bound ${all.low}`);
+});
+
+test('wilson narrows as n grows for the same proportion', () => {
+  const small = wilson(7, 20);
+  const large = wilson(70, 200);
+  near(small.rate, large.rate);
+  assert.ok(
+    large.high - large.low < small.high - small.low,
+    'the same fraction from more trials must be a tighter claim'
+  );
+});
+
+test('wilson refuses nonsensical counts rather than returning a number', () => {
+  assert.throws(() => wilson(3, 2), RangeError);
+  assert.throws(() => wilson(-1, 10), RangeError);
+  assert.throws(() => wilson(1.5, 10), TypeError);
+});
+
+test('an empty set has no rate at all, rather than zero', () => {
+  const interval = wilson(0, 0);
+  assert.equal(interval.rate, null);
+  assert.equal(interval.low, null);
+  assert.equal(interval.n, 0);
+});
+
+test('spread reports population sigma across repeats', () => {
+  const result = spread([0.4, 0.5, 0.6]);
+  near(result.mean, 0.5);
+  near(result.sigma, Math.sqrt(((0.1 ** 2) * 2) / 3));
+  near(result.min, 0.4);
+  near(result.max, 0.6);
+  assert.equal(result.runs, 3);
+});
+
+test('identical repeats have zero spread, and one repeat has no spread to report', () => {
+  near(spread([0.5, 0.5, 0.5]).sigma, 0);
+  near(spread([0.5]).sigma, 0);
+  assert.equal(spread([]).sigma, null);
+});
+
+test('outcome counts are exact, not bucketed', () => {
+  assert.deepEqual(
+    countOutcomes([{ outcome: 'ok' }, { outcome: 'ok' }, { outcome: 'bad_args' }]),
+    { ok: 2, bad_args: 1 }
+  );
+});
+
+test('only ok counts towards invocation rate — bad_args is not a partial success', () => {
+  const records = [
+    { session: 1, repeat: 1, outcome: 'ok' },
+    { session: 1, repeat: 1, outcome: 'bad_args' },
+    { session: 2, repeat: 1, outcome: 'ok' },
+    { session: 2, repeat: 1, outcome: 'wrong_tool' },
+  ];
+  const rollUp = rollUpTool({ tool: 'sum_by_category', records });
+
+  assert.equal(rollUp.trials, 4);
+  assert.equal(rollUp.ok, 2);
+  near(rollUp.invocation.rate, 0.5);
+  assert.deepEqual(rollUp.outcomes, { ok: 2, bad_args: 1, wrong_tool: 1 });
+  near(rollUp.betweenSession.mean, 0.5);
+  near(rollUp.betweenSession.sigma, 0);
+  assert.equal(rollUp.betweenSession.runs, 2);
+});
+
+test('between-session and within-session sigma are computed from different groupings', () => {
+  // Session 1 is internally stable at 1.0; session 2 is internally split 1.0/0.0.
+  // Between-session sigma compares 1.0 against 0.5; within-session averages 0 and 0.5.
+  const records = [
+    { session: 1, repeat: 1, outcome: 'ok' },
+    { session: 1, repeat: 2, outcome: 'ok' },
+    { session: 2, repeat: 1, outcome: 'ok' },
+    { session: 2, repeat: 2, outcome: 'wrong_tool' },
+  ];
+  const rollUp = rollUpTool({ tool: 'filter_rows', records });
+
+  near(rollUp.betweenSession.mean, 0.75);
+  near(rollUp.betweenSession.sigma, 0.25);
+  assert.equal(rollUp.betweenSession.runs, 2);
+  near(rollUp.withinSession.sigma, 0.25);
+  assert.equal(rollUp.withinSession.sessions, 2);
+});
+
+test('identical sessions report zero between-session sigma, which is a floor not a proof', () => {
+  const records = [
+    { session: 1, repeat: 1, outcome: 'wrong_tool' },
+    { session: 2, repeat: 1, outcome: 'wrong_tool' },
+    { session: 3, repeat: 1, outcome: 'wrong_tool' },
+  ];
+  const rollUp = rollUpTool({ tool: 'sum_by_category', records });
+  near(rollUp.invocation.rate, 0);
+  near(rollUp.betweenSession.sigma, 0);
+});
+
+test('controls invert the taxonomy: not_selected is the pass', () => {
+  const records = [
+    { session: 1, repeat: 1, outcome: 'not_selected', tag: 'off_topic', utteranceId: 'control-01' },
+    { session: 1, repeat: 1, outcome: 'ok', tag: 'out_of_scope', utteranceId: 'control-07' },
+    {
+      session: 2,
+      repeat: 1,
+      outcome: 'ok',
+      tag: 'injection',
+      utteranceId: 'control-19',
+      selection: { tool: 'describe_dataset' },
+    },
+  ];
+  const rollUp = rollUpControls({ records });
+
+  assert.equal(rollUp.trials, 3);
+  assert.equal(rollUp.falsePositives, 2);
+  near(rollUp.falsePositiveRate.rate, 2 / 3);
+  assert.equal(rollUp.byClass.off_topic.falsePositives, 0);
+  assert.equal(rollUp.byClass.out_of_scope.falsePositives, 1);
+  assert.deepEqual(rollUp.injectionFailures, [
+    { id: 'control-19', session: 2, selected: 'describe_dataset' },
+  ]);
+  // Session 1 was half bad, session 2 fully bad: sigma over [0.5, 1.0].
+  near(rollUp.betweenSession.sigma, 0.25);
+});

package/core/sweep.mjs

@@ -0,0 +1,274 @@
+/**
+ * One measurement session: every utterance, P repeats deep, one fresh tab per trial.
+ *
+ * A session owns its browser process and its cold profile, and runs in its own
+ * OS process, so nothing is shared with the sessions before or after it - not the
+ * page cache, not the renderer, not the judge's HTTP connection pool. That
+ * separation is the whole point: repeats inside a session are correlated by
+ * construction, and the first two sweeps reported their sigma as if it described
+ * reproducibility.
+ *
+ * A fresh tab per trial is the expensive choice and the correct one - reusing a
+ * tab means trial N's highlighting is trial N+1's starting state.
+ *
+ * Every completed trial is appended to a JSONL checkpoint before the next one
+ * starts, so a run that dies mid-session loses nothing and resumes.
+ */
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { openSession } from '../browser/session.mjs';
+import { runTrial } from './trial.mjs';
+
+/**
+ * Bounds one trial. Every wait inside a trial is bounded on its own now, but a
+ * sweep is a long-running unattended thing and a stall is its worst failure mode:
+ * the run that produced this comment sat on trial 160 of 160 for ninety minutes,
+ * printing nothing, because a CDP command never answered and nothing was watching.
+ * A trial that overruns is recorded as a non-measurement and retried by --resume,
+ * which is what every other unmeasurable trial already does.
+ */
+const withDeadline = (promise, ms, label) => {
+  let timer;
+  const guard = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      const error = new Error(`${label} did not finish within ${ms}ms`);
+      error.code = 'DEADLINE';
+      reject(error);
+    }, ms);
+  });
+  return Promise.race([promise, guard]).finally(() => clearTimeout(timer));
+};
+
+/** Exported for tests: the deadline is the invariant, not an implementation detail. */
+export const __withDeadline = withDeadline;
+
+export const trialKey = (session, repeat, utteranceId) => `${session}:${repeat}:${utteranceId}`;
+
+/**
+ * Appends harness failures as they happen rather than at the end of a session.
+ *
+ * This was written at session end once, and a session killed mid-plan then took its
+ * failure kinds with it: the run that produced `reports/discrimination-2026-08-30.md`
+ * lost three of them to an external timeout. Coverage still caught the missing
+ * trials, because that is computed from the plan rather than from this log - but the
+ * diagnosis was gone, and a diagnosis that only survives a clean exit is not much of
+ * a diagnosis.
+ */
+export const appendFailures = async (path, failures) => {
+  if (!path || failures.length === 0) return;
+  await mkdir(dirname(path), { recursive: true });
+  await appendFile(path, `${failures.map((failure) => JSON.stringify(failure)).join('\n')}\n`, 'utf8');
+};
+
+/**
+ * Reads the failure log, keeping the newest entry per trial. The log accumulates
+ * across `--resume` attempts, so the same trial can appear more than once; a report
+ * wants "how many trials failed at least once", not "how many attempts failed".
+ */
+export const readFailures = async (path) => {
+  try {
+    const text = await readFile(path, 'utf8');
+    const byTrial = new Map();
+    for (const line of text.split('\n')) {
+      if (line.trim().length === 0) continue;
+      const failure = JSON.parse(line);
+      byTrial.set(trialKey(failure.session ?? 1, failure.repeat, failure.utteranceId), failure);
+    }
+    return [...byTrial.values()];
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+};
+
+export const readCheckpoint = async (path) => {
+  try {
+    const text = await readFile(path, 'utf8');
+    const records = text
+      .split('\n')
+      .filter((line) => line.trim().length > 0)
+      .map((line) => JSON.parse(line));
+    return {
+      records,
+      keys: new Set(records.map((r) => trialKey(r.session ?? 1, r.repeat, r.utteranceId))),
+    };
+  } catch (error) {
+    if (error.code === 'ENOENT') return { records: [], keys: new Set() };
+    throw error;
+  }
+};
+
+/** Flattens the fixture into one session's trial plan, controls included. */
+export const buildPlan = ({
+  fixture,
+  repeatsPerSession = 1,
+  tools = null,
+  includeControls = true,
+}) => {
+  const plan = [];
+
+  for (let repeat = 1; repeat <= repeatsPerSession; repeat += 1) {
+    for (const toolBlock of fixture.tools) {
+      if (tools && !tools.includes(toolBlock.name)) continue;
+      for (const utterance of toolBlock.utterances) {
+        plan.push({
+          repeat,
+          kind: 'tool',
+          toolName: toolBlock.name,
+          utterance,
+          setup: toolBlock.setup ?? null,
+        });
+      }
+    }
+
+    if (includeControls && fixture.controls) {
+      for (const utterance of fixture.controls.utterances) {
+        plan.push({ repeat, kind: 'control', toolName: null, utterance, setup: null });
+      }
+    }
+  }
+
+  return plan;
+};
+
+const toCheckpointRecord = ({ item, record, session, sessionMeta }) => ({
+  session,
+  repeat: item.repeat,
+  kind: item.kind,
+  utteranceId: item.utterance.id,
+  tag: item.utterance.tag ?? null,
+  expectedTool: item.toolName,
+  outcome: record.outcome,
+  reason: record.reason,
+  selection: record.selection,
+  violations: record.violations ?? null,
+  execution: record.execution ?? null,
+  observationChanged: record.observation?.changed ?? null,
+  client: record.client,
+  judge: record.judge,
+  seed: record.seed ?? null,
+  startedAt: record.trial.startedAt,
+  sessionMeta,
+});
+
+export const runSessionSweep = async ({
+  fixture,
+  judge,
+  url,
+  session = 1,
+  sessionMeta = null,
+  repeatsPerSession = 1,
+  tools = null,
+  includeControls = true,
+  concurrency = 1,
+  port,
+  checkpointPath,
+  failureLogPath = null,
+  /**
+   * Generous on purpose: the judge alone may take 60s, and a trial is a navigation,
+   * a manifest settle, a judge call and an execution. The observed median is under
+   * ten seconds, so this catches stalls rather than slow work.
+   */
+  trialTimeoutMs = 180000,
+  onProgress = () => {},
+}) => {
+  const plan = buildPlan({ fixture, repeatsPerSession, tools, includeControls });
+  const { keys: done } = checkpointPath
+    ? await readCheckpoint(checkpointPath)
+    : { keys: new Set() };
+  const pending = plan.filter(
+    (item) => !done.has(trialKey(session, item.repeat, item.utterance.id))
+  );
+
+  if (checkpointPath) await mkdir(dirname(checkpointPath), { recursive: true });
+
+  const written = [];
+  const failures = [];
+  let completed = 0;
+  let cursor = 0;
+
+  const worker = async () => {
+    while (cursor < pending.length) {
+      const item = pending[cursor];
+      cursor += 1;
+
+      // Recorded the moment it happens, not at the end of the session: a killed
+      // process must not take the reason with it.
+      const recordFailure = async (failure) => {
+        failures.push(failure);
+        await appendFailures(failureLogPath, [failure]).catch(() => {});
+      };
+
+      let tab;
+      try {
+        // The work is started as its own promise so the deadline can stop *waiting*
+        // for it. Abandoned work still settles later, and its rejection must not
+        // reach the process as an unhandled one, hence the bare catch.
+        const attempt = (async () => {
+          tab = await openSession({ port });
+          return runTrial({
+            session: tab,
+            judge,
+            url,
+            toolName: item.toolName,
+            utterance: item.utterance,
+            expectation: item.kind === 'tool' ? item.utterance : {},
+            setup: item.setup,
+            fixtureVersion: fixture.version,
+            controlMode: item.kind === 'control',
+          });
+        })();
+        attempt.catch(() => {});
+
+        const record = await withDeadline(
+          attempt,
+          trialTimeoutMs,
+          `trial ${item.utterance.id} (session ${session}, repeat ${item.repeat})`
+        );
+
+        if (record.outcome === null) {
+          // The trial ran but produced no measurement - an unreachable or truncated
+          // judge says nothing about the page. It stays out of the checkpoint so a
+          // later --resume retries it instead of baking a non-result into the rates.
+          await recordFailure({
+            session,
+            repeat: item.repeat,
+            utteranceId: item.utterance.id,
+            kind: record.harnessFailure?.kind ?? 'unknown',
+            error: record.harnessFailure?.detail ?? 'no outcome and no reason given',
+          });
+        } else {
+          const checkpointRecord = toCheckpointRecord({ item, record, session, sessionMeta });
+          written.push(checkpointRecord);
+          if (checkpointPath) {
+            await appendFile(checkpointPath, `${JSON.stringify(checkpointRecord)}\n`, 'utf8');
+          }
+        }
+      } catch (error) {
+        await recordFailure({
+          session,
+          repeat: item.repeat,
+          utteranceId: item.utterance.id,
+          kind: error.code === 'DEADLINE' ? 'trial_timeout' : 'trial_threw',
+          error: String(error.message ?? error),
+        });
+      } finally {
+        if (tab) await tab.close().catch(() => {});
+      }
+
+      completed += 1;
+      onProgress({ completed, total: pending.length, item, session, failures: failures.length });
+    }
+  };
+
+  const startedMs = Date.now();
+  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+
+  return {
+    session,
+    written,
+    failures,
+    plan: { total: plan.length, attempted: pending.length },
+    elapsedMs: Date.now() - startedMs,
+  };
+};