webmcp-gauge 0.1.0

package/core/gate.mjs

@@ -0,0 +1,164 @@
+/**
+ * What an exit code is allowed to mean.
+ *
+ * A gate is only useful if `1` means one thing. Until now any harness failure
+ * exited 1, so "two trials need a --resume" and "the invocation rate fell off a
+ * cliff" produced the same signal, and a CI job could not tell a regression from
+ * a flaky provider. The codes are therefore split by what the run *knows*:
+ *
+ *   0  every planned trial was measured, and nothing gated fell below the threshold
+ *   1  every planned trial was measured, and a rate is below --fail-under
+ *   2  the run cannot answer: planned trials have no measurement, or bad usage
+ *
+ * Incomplete outranks breach deliberately. Gaps are not random - a judge outage or
+ * a page that never loaded can take out one tool's utterances and nothing else, so
+ * a rate computed over a run with holes is a rate over a denominator the run did
+ * not choose. Reporting that as a regression would be lying with a plausible
+ * number, and the whole point of the taxonomy is that a non-measurement is not an
+ * outcome.
+ *
+ * The threshold is compared against the **point rate**, not the Wilson lower
+ * bound. Gating on the bound was considered and rejected: 20 of 20 has a lower
+ * bound of 83.9%, so a page that never failed once would breach `--fail-under 0.9`
+ * on sample size alone. The interval is printed beside the rate instead, so a
+ * breach that sits inside the noise is visible to the person reading it.
+ */
+
+export const EXIT = { pass: 0, breach: 1, incomplete: 2 };
+
+const percent = (value) => `${(value * 100).toFixed(1)}%`;
+
+/**
+ * `--fail-under 90` is the mistake this exists to catch: it parses as a number,
+ * compares against every rate, and fails the build forever. Rates are fractions
+ * here and nowhere else in the CLI, so the boundary check belongs at the boundary.
+ */
+export const parseFailUnder = (raw) => {
+  if (raw === undefined || raw === null) return null;
+  if (raw === true) throw new RangeError('--fail-under needs a rate, e.g. --fail-under 0.9');
+
+  const value = Number(raw);
+  if (!Number.isFinite(value) || value < 0 || value > 1) {
+    const hint = Number.isFinite(value) && value > 1 && value <= 100 ? ` — use ${value / 100}, not ${value}` : '';
+    throw new RangeError(`--fail-under takes a rate between 0 and 1, not '${raw}'${hint}`);
+  }
+  return value;
+};
+
+const countKinds = (failures) => {
+  const kinds = {};
+  for (const failure of failures) {
+    const kind = failure.kind ?? 'unknown';
+    kinds[kind] = (kinds[kind] ?? 0) + 1;
+  }
+  return Object.entries(kinds)
+    .sort((a, b) => b[1] - a[1])
+    .map(([kind, count]) => `${count} ${kind}`)
+    .join(', ');
+};
+
+/**
+ * Decides one exit code for a finished run. Pure: it reads the report and returns
+ * a verdict, so the CI contract is testable without a browser or a judge.
+ */
+export const gateRun = ({ report, failUnder = null }) => {
+  const threshold = parseFailUnder(failUnder);
+  const perTool = report.invocation?.perTool ?? [];
+  const outstandingFailures = report.harnessFailures ?? [];
+  const coverage = report.coverage ?? null;
+  const missingTrials = coverage?.missingTrials ?? 0;
+  const expectedTrials = coverage?.expectedTrials ?? null;
+
+  const coverageNote =
+    expectedTrials === null
+      ? 'coverage unknown'
+      : `${expectedTrials - missingTrials}/${expectedTrials} planned trials measured`;
+
+  const breaches = (threshold === null ? [] : perTool)
+    .filter((tool) => tool.invocation?.rate !== null && tool.invocation?.rate < threshold)
+    .map((tool) => ({
+      tool: tool.tool,
+      rate: tool.invocation.rate,
+      low: tool.invocation.low,
+      high: tool.invocation.high,
+      trials: tool.trials,
+    }))
+    .sort((a, b) => a.rate - b.rate);
+
+  const nothingToGate = threshold !== null && perTool.length === 0;
+
+  const verdict = {
+    failUnder: threshold,
+    gatedTools: threshold === null ? 0 : perTool.length,
+    missingTrials,
+    outstandingFailures: outstandingFailures.length,
+    breaches,
+  };
+
+  if (missingTrials > 0 || outstandingFailures.length > 0 || nothingToGate) {
+    const reasons = [];
+    if (missingTrials > 0) {
+      const withCause = outstandingFailures.length;
+      reasons.push(
+        `${missingTrials} of ${expectedTrials ?? '?'} planned trials produced no measurement` +
+          (withCause > 0 ? ` (${countKinds(outstandingFailures)})` : ' with no logged cause')
+      );
+    } else if (outstandingFailures.length > 0) {
+      reasons.push(
+        `${outstandingFailures.length} logged harness failure${outstandingFailures.length === 1 ? '' : 's'} outside the measured set (${countKinds(outstandingFailures)})`
+      );
+    }
+    if (nothingToGate) {
+      reasons.push(
+        `no tool trials were measured, so --fail-under ${percent(threshold)} cannot be evaluated`
+      );
+    }
+    return {
+      ...verdict,
+      code: EXIT.incomplete,
+      status: 'incomplete',
+      summary: `INCOMPLETE — ${reasons.join('; ')}. Re-run with --resume; this is not a threshold breach.`,
+    };
+  }
+
+  if (breaches.length > 0) {
+    const worst = breaches[0];
+    return {
+      ...verdict,
+      code: EXIT.breach,
+      status: 'breach',
+      summary:
+        `FAIL — ${breaches.length} of ${perTool.length} tool${perTool.length === 1 ? '' : 's'} below --fail-under ${percent(threshold)}: ` +
+        breaches
+          .map(
+            (breach) =>
+              `\`${breach.tool}\` ${percent(breach.rate)} [${percent(breach.low)}, ${percent(breach.high)}] over ${breach.trials} trials`
+          )
+          .join(', ') +
+        `. ${coverageNote}, so the number is the page's, not the harness's.` +
+        (worst.high >= threshold
+          ? ` Note ${worst.tool}'s interval still reaches ${percent(worst.high)}: the breach is inside the noise at this sample size.`
+          : ''),
+    };
+  }
+
+  if (threshold === null) {
+    return {
+      ...verdict,
+      code: EXIT.pass,
+      status: 'pass',
+      summary: `COMPLETE — ${coverageNote}, no --fail-under given, so nothing was gated.`,
+    };
+  }
+
+  const lowest = [...perTool].sort(
+    (a, b) => (a.invocation.rate ?? 1) - (b.invocation.rate ?? 1)
+  )[0];
+
+  return {
+    ...verdict,
+    code: EXIT.pass,
+    status: 'pass',
+    summary: `PASS — all ${perTool.length} tools at or above --fail-under ${percent(threshold)} (lowest \`${lowest.tool}\` ${percent(lowest.invocation.rate)}), ${coverageNote}.`,
+  };
+};

package/core/gate.test.mjs

@@ -0,0 +1,213 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { buildReport } from '../report/emit.mjs';
+import { EXIT, gateRun, parseFailUnder } from './gate.mjs';
+
+const fixture = { version: '1.3.0', frozen: true, authoring: { modelId: 'author-model' } };
+
+/**
+ * Built through the real emitter rather than hand-shaped, so the gate is tested
+ * against the report contract it will actually be handed. A rename in emit.mjs
+ * should break these tests loudly instead of leaving the gate reading undefined.
+ */
+const reportOf = ({ records = [], coverage = null, harnessFailures = [] } = {}) =>
+  buildReport({
+    fixture,
+    records,
+    harnessFailures,
+    coverage,
+    judge: { model: 'judge-model', baseUrl: 'https://example.invalid/v1' },
+    settings: { url: 'https://example.invalid/', sessions: 1, repeatsPerSession: 1, concurrency: 1 },
+    timing: { startedAt: '2026-08-30T00:00:00.000Z', finishedAt: '2026-08-30T00:01:00.000Z', elapsedMs: 60000 },
+  });
+
+const toolTrials = ({ tool, ok, total, session = 1 }) =>
+  Array.from({ length: total }, (_, index) => ({
+    session,
+    repeat: 1,
+    kind: 'tool',
+    expectedTool: tool,
+    utteranceId: `${tool}-${String(index + 1).padStart(2, '0')}`,
+    tag: 'plain',
+    outcome: index < ok ? 'ok' : 'wrong_tool',
+  }));
+
+const complete = (expectedTrials) => ({ expectedTrials, measuredTrials: expectedTrials, missingTrials: 0, missing: [] });
+
+test('a complete run with every tool above the threshold exits 0', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: [
+        ...toolTrials({ tool: 'top_expenses', ok: 20, total: 20 }),
+        ...toolTrials({ tool: 'filter_rows', ok: 19, total: 20 }),
+      ],
+      coverage: complete(40),
+    }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.pass);
+  assert.equal(verdict.status, 'pass');
+  assert.equal(verdict.gatedTools, 2);
+  assert.deepEqual(verdict.breaches, []);
+  assert.match(verdict.summary, /^PASS —/);
+  assert.match(verdict.summary, /40\/40 planned trials measured/);
+});
+
+test('a rate below --fail-under exits 1 and names the tool, its interval and its trial count', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: [
+        ...toolTrials({ tool: 'top_expenses', ok: 20, total: 20 }),
+        ...toolTrials({ tool: 'sum_by_category', ok: 16, total: 20 }),
+      ],
+      coverage: complete(40),
+    }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.breach);
+  assert.equal(verdict.status, 'breach');
+  assert.equal(verdict.breaches.length, 1);
+  assert.equal(verdict.breaches[0].tool, 'sum_by_category');
+  assert.equal(verdict.breaches[0].trials, 20);
+  assert.match(verdict.summary, /^FAIL —/);
+  assert.match(verdict.summary, /sum_by_category` 80\.0% \[\d+\.\d%, \d+\.\d%\] over 20 trials/);
+});
+
+test('a breach whose interval still reaches the threshold says so, rather than implying certainty', () => {
+  // 17/20 = 85%, and the Wilson upper bound is above 90%, so the breach is inside
+  // the noise. Exit 1 is still correct - the measured rate is the measured rate -
+  // but a report that hid the overlap would invite a wrong conclusion.
+  const verdict = gateRun({
+    report: reportOf({ records: toolTrials({ tool: 'filter_rows', ok: 17, total: 20 }), coverage: complete(20) }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.breach);
+  assert.ok(verdict.breaches[0].high > 0.9, 'this fixture is only interesting if the interval overlaps');
+  assert.match(verdict.summary, /inside the noise at this sample size/);
+});
+
+test('the threshold is compared against the point rate, not the Wilson lower bound', () => {
+  // 20/20 has a lower bound of 83.9%. Gating on the bound would fail a page that
+  // never missed once, purely on sample size, so the point rate is what is gated.
+  const report = reportOf({
+    records: toolTrials({ tool: 'top_expenses', ok: 20, total: 20 }),
+    coverage: complete(20),
+  });
+  const [tool] = report.invocation.perTool;
+  assert.equal(tool.invocation.rate, 1);
+  assert.ok(
+    tool.invocation.low < 0.9,
+    `20/20 has a lower bound of ${tool.invocation.low}; the test is only meaningful while that is under the threshold`
+  );
+
+  assert.equal(gateRun({ report, failUnder: 0.9 }).code, EXIT.pass);
+});
+
+test('a rate exactly at the threshold passes: below the line is a breach, on it is not', () => {
+  const verdict = gateRun({
+    report: reportOf({ records: toolTrials({ tool: 'filter_rows', ok: 18, total: 20 }), coverage: complete(20) }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.pass);
+});
+
+test('planned trials with no measurement exit 2, however healthy the rates look', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: toolTrials({ tool: 'top_expenses', ok: 18, total: 18 }),
+      coverage: { expectedTrials: 20, measuredTrials: 18, missingTrials: 2, missing: ['1:1:top_expenses-19', '1:1:top_expenses-20'] },
+      harnessFailures: [
+        { session: 1, repeat: 1, utteranceId: 'top_expenses-19', kind: 'judge_unavailable', error: 'fetch failed' },
+        { session: 1, repeat: 1, utteranceId: 'top_expenses-20', kind: 'trial_threw', error: 'timed out waiting for Page.loadEventFired' },
+      ],
+    }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.incomplete);
+  assert.equal(verdict.status, 'incomplete');
+  assert.equal(verdict.missingTrials, 2);
+  assert.match(verdict.summary, /2 of 20 planned trials produced no measurement/);
+  assert.match(verdict.summary, /judge_unavailable/);
+  assert.match(verdict.summary, /not a threshold breach/);
+});
+
+test('incomplete outranks a breach: a run with holes cannot certify a regression', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: toolTrials({ tool: 'sum_by_category', ok: 10, total: 18 }),
+      coverage: { expectedTrials: 20, measuredTrials: 18, missingTrials: 2, missing: [] },
+    }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.incomplete);
+  // The breach is still recorded - it is the reason to re-run, not a number to publish.
+  assert.equal(verdict.breaches.length, 1);
+  assert.match(verdict.summary, /with no logged cause/);
+});
+
+test('missing trials exit 2 even with no threshold set — the old behaviour was to exit 1', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: toolTrials({ tool: 'top_expenses', ok: 19, total: 19 }),
+      coverage: { expectedTrials: 20, measuredTrials: 19, missingTrials: 1, missing: ['1:1:top_expenses-20'] },
+    }),
+  });
+
+  assert.equal(verdict.code, EXIT.incomplete);
+  assert.equal(verdict.failUnder, null);
+  assert.equal(verdict.gatedTools, 0);
+});
+
+test('a complete run with no threshold exits 0 and says nothing was gated', () => {
+  const verdict = gateRun({
+    report: reportOf({ records: toolTrials({ tool: 'top_expenses', ok: 12, total: 20 }), coverage: complete(20) }),
+  });
+
+  assert.equal(verdict.code, EXIT.pass);
+  assert.match(verdict.summary, /no --fail-under given, so nothing was gated/);
+  assert.deepEqual(verdict.breaches, []);
+});
+
+test('an outstanding harness failure exits 2 even when coverage was not computed', () => {
+  const verdict = gateRun({
+    report: reportOf({
+      records: toolTrials({ tool: 'top_expenses', ok: 20, total: 20 }),
+      harnessFailures: [{ session: 1, repeat: 1, utteranceId: 'filter_rows-03', kind: 'judge_timeout', error: 'timeout' }],
+    }),
+    failUnder: 0.9,
+  });
+
+  assert.equal(verdict.code, EXIT.incomplete);
+  assert.match(verdict.summary, /outside the measured set \(1 judge_timeout\)/);
+});
+
+test('a threshold with no tool trials at all exits 2 rather than vacuously passing', () => {
+  const verdict = gateRun({ report: reportOf({ coverage: complete(0) }), failUnder: 0.9 });
+
+  assert.equal(verdict.code, EXIT.incomplete);
+  assert.match(verdict.summary, /cannot be evaluated/);
+});
+
+test('parseFailUnder accepts fractions and refuses percentages, which would fail every build', () => {
+  assert.equal(parseFailUnder(undefined), null);
+  assert.equal(parseFailUnder(null), null);
+  assert.equal(parseFailUnder('0.9'), 0.9);
+  assert.equal(parseFailUnder(0), 0);
+  assert.equal(parseFailUnder('1'), 1);
+
+  assert.throws(() => parseFailUnder('90'), /use 0\.9, not 90/);
+  assert.throws(() => parseFailUnder('101'), RangeError);
+  assert.throws(() => parseFailUnder('-0.1'), RangeError);
+  assert.throws(() => parseFailUnder('nine tenths'), RangeError);
+  assert.throws(() => parseFailUnder(true), /needs a rate/);
+});
+
+test('gateRun refuses a nonsensical threshold rather than gating on it', () => {
+  assert.throws(() => gateRun({ report: reportOf({ coverage: complete(0) }), failUnder: 90 }), RangeError);
+});