webmcp-gauge 0.1.0

package/core/sweep.test.mjs

@@ -0,0 +1,162 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { appendFailures, buildPlan, readCheckpoint, readFailures, trialKey, __withDeadline as withDeadline } from './sweep.mjs';
+
+const withTempDir = async (body) => {
+  const dir = await mkdtemp(join(tmpdir(), 'webmcp-gauge-sweep-'));
+  try {
+    return await body(dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+};
+
+const failure = (utteranceId, kind, session = 1) => ({
+  session,
+  repeat: 1,
+  utteranceId,
+  kind,
+  error: `${kind} on ${utteranceId}`,
+});
+
+test('a failure is durable as soon as it is appended, one line at a time', async () => {
+  await withTempDir(async (dir) => {
+    const path = join(dir, 'nested', 'harness-failures.jsonl');
+
+    // Written one at a time, which is how the sweep calls it: the point of the fix
+    // is that a process killed after the first failure still leaves the first
+    // failure on disk.
+    await appendFailures(path, [failure('filter_rows-05', 'judge_truncated')]);
+    const afterOne = await readFailures(path);
+    assert.equal(afterOne.length, 1);
+
+    await appendFailures(path, [failure('top_expenses-11', 'trial_threw')]);
+    const afterTwo = await readFailures(path);
+    assert.deepEqual(
+      afterTwo.map((entry) => entry.utteranceId),
+      ['filter_rows-05', 'top_expenses-11']
+    );
+    assert.equal((await readFile(path, 'utf8')).trim().split('\n').length, 2);
+  });
+});
+
+test('appending nothing writes nothing, and a missing log reads as no failures', async () => {
+  await withTempDir(async (dir) => {
+    const path = join(dir, 'harness-failures.jsonl');
+    await appendFailures(path, []);
+    await appendFailures(null, [failure('a-01', 'trial_threw')]);
+    assert.deepEqual(await readFailures(path), []);
+  });
+});
+
+test('a trial that failed twice across attempts counts once, with its newest kind', async () => {
+  await withTempDir(async (dir) => {
+    const path = join(dir, 'harness-failures.jsonl');
+    await appendFailures(path, [failure('top_expenses-11', 'judge_truncated')]);
+    await appendFailures(path, [failure('top_expenses-11', 'trial_threw')]);
+    await appendFailures(path, [failure('top_expenses-11', 'judge_unavailable', 2)]);
+
+    const entries = await readFailures(path);
+    // Same trial in session 1 twice - one entry, newest kind. Session 2 is a
+    // different trial key and keeps its own entry.
+    assert.equal(entries.length, 2);
+    assert.equal(entries.find((entry) => entry.session === 1).kind, 'trial_threw');
+    assert.equal(entries.find((entry) => entry.session === 2).kind, 'judge_unavailable');
+  });
+});
+
+test('a malformed log line fails loudly rather than silently dropping a failure', async () => {
+  await withTempDir(async (dir) => {
+    const path = join(dir, 'harness-failures.jsonl');
+    await writeFile(path, '{"session":1,"repeat":1,"utteranceId":"a-01"}\nnot json\n', 'utf8');
+    await assert.rejects(() => readFailures(path), SyntaxError);
+  });
+});
+
+test('a trial that never settles is stopped waiting for, and says so with a code', async () => {
+  // The stall this exists to prevent: 90 minutes on one trial with no output, because
+  // a CDP command never answered. A deadline turns that into a non-measurement the
+  // sweep records and --resume retries.
+  const neverSettles = new Promise(() => {});
+  await assert.rejects(
+    () => withDeadline(neverSettles, 20, 'trial x-01'),
+    (error) => {
+      assert.equal(error.code, 'DEADLINE');
+      assert.match(error.message, /trial x-01 did not finish within 20ms/);
+      return true;
+    }
+  );
+});
+
+test('the deadline does not interfere with work that finishes, or with its errors', async () => {
+  assert.equal(await withDeadline(Promise.resolve('measured'), 1000, 'fast'), 'measured');
+
+  // A real failure must arrive as itself, not as a timeout: the sweep classifies
+  // trial_threw and trial_timeout differently and the distinction is diagnostic.
+  await assert.rejects(
+    () => withDeadline(Promise.reject(new Error('page threw')), 1000, 'failing'),
+    (error) => {
+      assert.equal(error.message, 'page threw');
+      assert.equal(error.code, undefined);
+      return true;
+    }
+  );
+});
+
+test('an abandoned trial that rejects later cannot crash the process', async () => {
+  // The worker attaches a bare catch to the attempt for exactly this reason: after
+  // the deadline wins, nothing is awaiting the original promise any more.
+  let failLater;
+  const attempt = new Promise((_, reject) => {
+    failLater = reject;
+  });
+  attempt.catch(() => {});
+
+  await assert.rejects(() => withDeadline(attempt, 10, 'trial y-02'));
+  failLater(new Error('CDP Runtime.evaluate did not answer within 30000ms'));
+  await new Promise((resolve) => setTimeout(resolve, 20));
+});
+
+test('the plan covers every utterance once per repeat, controls included', () => {
+  const fixture = {
+    version: 'test',
+    tools: [
+      { name: 'a_tool', utterances: [{ id: 'a_tool-01' }, { id: 'a_tool-02' }] },
+      { name: 'b_tool', utterances: [{ id: 'b_tool-01' }] },
+    ],
+    controls: { utterances: [{ id: 'control-01' }] },
+  };
+
+  const plan = buildPlan({ fixture, repeatsPerSession: 2 });
+  assert.equal(plan.length, 8);
+  assert.equal(plan.filter((item) => item.kind === 'control').length, 2);
+
+  const withoutControls = buildPlan({ fixture, includeControls: false });
+  assert.equal(withoutControls.length, 3);
+
+  const oneTool = buildPlan({ fixture, tools: ['b_tool'], includeControls: false });
+  assert.deepEqual(oneTool.map((item) => item.utterance.id), ['b_tool-01']);
+});
+
+test('the checkpoint reads back as records plus the trial keys already measured', async () => {
+  await withTempDir(async (dir) => {
+    const path = join(dir, 'sweep.jsonl');
+    await writeFile(
+      path,
+      '{"session":2,"repeat":1,"utteranceId":"a_tool-01","outcome":"ok"}\n{"repeat":1,"utteranceId":"a_tool-02","outcome":"wrong_tool"}\n',
+      'utf8'
+    );
+
+    const { records, keys } = await readCheckpoint(path);
+    assert.equal(records.length, 2);
+    assert.ok(keys.has(trialKey(2, 1, 'a_tool-01')));
+    // A record written before sessions existed defaults to session 1 rather than
+    // being dropped, or a resume would re-run every trial of an older run.
+    assert.ok(keys.has(trialKey(1, 1, 'a_tool-02')));
+
+    assert.deepEqual(await readCheckpoint(join(dir, 'missing.jsonl')), { records: [], keys: new Set() });
+  });
+});

package/core/taxonomy.mjs

@@ -0,0 +1,175 @@
+/**
+ * The outcome taxonomy, and the argument checks that decide between its buckets.
+ *
+ * The taxonomy is the product: a pass/fail number tells a developer nothing,
+ * while `not_selected` vs `bad_args` vs `not_registered` tells them which line to
+ * edit. Every trial lands in exactly one bucket, so these functions return one
+ * outcome and never a set.
+ */
+
+export const OUTCOMES = Object.freeze([
+  'not_supported', // the client has no WebMCP surface at all
+  'not_registered', // the page registered nothing, or not this tool
+  'not_discovered', // the page has it, the browser never surfaced it
+  'not_selected', // the judge chose no tool
+  'wrong_tool', // the judge chose a different tool
+  'bad_args', // right tool, arguments that do not satisfy the request
+  'exec_error', // the call threw
+  'silent_fail', // the call returned and nothing observably happened
+  'ok',
+]);
+
+const isPlainObject = (value) =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+const sameScalar = (a, b) => {
+  if (typeof a === 'number' && typeof b === 'number') return a === b;
+  if (typeof a === 'string' && typeof b === 'string') return a.trim() === b.trim();
+  return a === b;
+};
+
+const constraintHolds = (value, constraint) =>
+  Object.entries(constraint).every(([operator, bound]) => {
+    if (typeof value !== 'number') return false;
+    if (operator === 'gt') return value > bound;
+    if (operator === 'gte') return value >= bound;
+    if (operator === 'lt') return value < bound;
+    if (operator === 'lte') return value <= bound;
+    return false;
+  });
+
+/**
+ * Checks the judge's arguments against one utterance's expectations and the
+ * tool's own schema. Returns every violation rather than the first, because the
+ * reason is what a developer acts on; the caller collapses them to `bad_args`.
+ */
+export const checkArguments = ({ args, expectation = {}, inputSchema }) => {
+  const violations = [];
+  const supplied = isPlainObject(args) ? args : {};
+
+  if (args !== undefined && args !== null && !isPlainObject(args)) {
+    violations.push({ kind: 'not_an_object', detail: typeof args });
+  }
+
+  const schemaKeys = isPlainObject(inputSchema?.properties)
+    ? Object.keys(inputSchema.properties)
+    : null;
+  if (schemaKeys) {
+    for (const key of Object.keys(supplied)) {
+      if (!schemaKeys.includes(key)) violations.push({ kind: 'unknown_key', key });
+    }
+  }
+
+  for (const [key, expected] of Object.entries(expectation.expectedArgs ?? {})) {
+    if (!(key in supplied)) {
+      violations.push({ kind: 'missing_expected_key', key, expected });
+      continue;
+    }
+    if (!sameScalar(supplied[key], expected)) {
+      violations.push({ kind: 'wrong_value', key, expected, actual: supplied[key] });
+    }
+  }
+
+  for (const key of expectation.requiredArgKeys ?? []) {
+    if (!(key in supplied)) violations.push({ kind: 'missing_required_key', key });
+  }
+
+  for (const [key, constraint] of Object.entries(expectation.argConstraints ?? {})) {
+    if (!(key in supplied)) continue; // already reported by requiredArgKeys
+    if (!constraintHolds(supplied[key], constraint)) {
+      violations.push({ kind: 'constraint_violated', key, constraint, actual: supplied[key] });
+    }
+  }
+
+  for (const key of expectation.forbiddenArgKeys ?? []) {
+    if (key in supplied) violations.push({ kind: 'forbidden_key', key, actual: supplied[key] });
+  }
+
+  return { valid: violations.length === 0, violations };
+};
+
+/**
+ * Everything decidable before the tool is called. Returns null when the trial
+ * should proceed to execution.
+ *
+ * `browserToolNames` is optional: without the browser's own view there is no way
+ * to separate "the page never registered it" from "the browser never surfaced
+ * it", and guessing between them would invent a diagnosis.
+ */
+export const classifyBeforeExecution = ({
+  manifest,
+  expectedTool,
+  selection,
+  expectation,
+  browserToolNames = null,
+}) => {
+  if (!manifest?.present) {
+    return { outcome: 'not_supported', reason: 'no modelContext on this client' };
+  }
+
+  const pageToolNames = (manifest.tools ?? []).map((tool) => tool.name);
+
+  if (pageToolNames.length === 0) {
+    return { outcome: 'not_registered', reason: 'page registered no tools' };
+  }
+
+  if (!pageToolNames.includes(expectedTool)) {
+    return {
+      outcome: 'not_registered',
+      reason: `page registered ${pageToolNames.length} tools, none named ${expectedTool}`,
+    };
+  }
+
+  if (browserToolNames && !browserToolNames.includes(expectedTool)) {
+    return {
+      outcome: 'not_discovered',
+      reason: `page registered ${expectedTool} but the browser never surfaced it`,
+    };
+  }
+
+  if (!selection?.tool) {
+    return { outcome: 'not_selected', reason: 'judge chose no tool' };
+  }
+
+  if (selection.tool !== expectedTool) {
+    return { outcome: 'wrong_tool', reason: `judge chose ${selection.tool}` };
+  }
+
+  const schema = manifest.tools.find((tool) => tool.name === expectedTool)?.inputSchema;
+  const argCheck = checkArguments({
+    args: selection.arguments,
+    expectation,
+    inputSchema: schema,
+  });
+
+  if (!argCheck.valid) {
+    return { outcome: 'bad_args', reason: 'arguments rejected', violations: argCheck.violations };
+  }
+
+  return null;
+};
+
+/**
+ * Everything decidable after the call. `silent_fail` needs care: a read-only tool
+ * legitimately changes no DOM, so an empty payload AND an unchanged page is the
+ * only honest signal that nothing happened.
+ */
+export const classifyAfterExecution = ({ execution, before, after }) => {
+  if (!execution?.ok) {
+    return { outcome: 'exec_error', reason: execution?.error ?? 'call failed' };
+  }
+
+  const result = execution.result;
+  const emptyResult =
+    result === null ||
+    result === undefined ||
+    (isPlainObject(result) && Object.keys(result).length === 0) ||
+    (typeof result === 'string' && result.trim() === '');
+  const pageChanged = JSON.stringify(before ?? null) !== JSON.stringify(after ?? null);
+
+  if (emptyResult && !pageChanged) {
+    return { outcome: 'silent_fail', reason: 'empty result and no observable page change' };
+  }
+
+  return { outcome: 'ok', reason: pageChanged ? 'result returned, page changed' : 'result returned' };
+};

package/core/taxonomy.test.mjs

@@ -0,0 +1,198 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  OUTCOMES,
+  checkArguments,
+  classifyAfterExecution,
+  classifyBeforeExecution,
+} from './taxonomy.mjs';
+
+const manifest = {
+  present: true,
+  tools: [
+    {
+      name: 'sum_by_category',
+      inputSchema: { type: 'object', properties: { highlight: { type: 'string' } } },
+    },
+    {
+      name: 'find_anomalies',
+      inputSchema: { type: 'object', properties: { threshold: { type: 'number' } } },
+    },
+  ],
+};
+
+test('the taxonomy is exactly the nine documented outcomes', () => {
+  assert.deepEqual(OUTCOMES, [
+    'not_supported',
+    'not_registered',
+    'not_discovered',
+    'not_selected',
+    'wrong_tool',
+    'bad_args',
+    'exec_error',
+    'silent_fail',
+    'ok',
+  ]);
+});
+
+test('a client with no modelContext is not_supported, not not_registered', () => {
+  const verdict = classifyBeforeExecution({
+    manifest: { present: false },
+    expectedTool: 'sum_by_category',
+    selection: { tool: 'sum_by_category' },
+  });
+  assert.equal(verdict.outcome, 'not_supported');
+});
+
+test('an empty or mismatched page manifest is not_registered', () => {
+  assert.equal(
+    classifyBeforeExecution({
+      manifest: { present: true, tools: [] },
+      expectedTool: 'sum_by_category',
+      selection: null,
+    }).outcome,
+    'not_registered'
+  );
+
+  assert.equal(
+    classifyBeforeExecution({
+      manifest,
+      expectedTool: 'top_expenses',
+      selection: { tool: 'top_expenses' },
+    }).outcome,
+    'not_registered'
+  );
+});
+
+test('not_discovered needs the browser view, and is never guessed without it', () => {
+  const withoutBrowserView = classifyBeforeExecution({
+    manifest,
+    expectedTool: 'sum_by_category',
+    selection: { tool: 'sum_by_category', arguments: {} },
+  });
+  assert.equal(withoutBrowserView, null, 'proceeds to execution rather than inventing a diagnosis');
+
+  const withBrowserView = classifyBeforeExecution({
+    manifest,
+    expectedTool: 'sum_by_category',
+    selection: { tool: 'sum_by_category', arguments: {} },
+    browserToolNames: ['find_anomalies'],
+  });
+  assert.equal(withBrowserView.outcome, 'not_discovered');
+});
+
+test('no selection is not_selected, another tool is wrong_tool', () => {
+  assert.equal(
+    classifyBeforeExecution({ manifest, expectedTool: 'sum_by_category', selection: null }).outcome,
+    'not_selected'
+  );
+  assert.equal(
+    classifyBeforeExecution({
+      manifest,
+      expectedTool: 'sum_by_category',
+      selection: { tool: 'find_anomalies', arguments: {} },
+    }).outcome,
+    'wrong_tool'
+  );
+});
+
+test('expected argument values must match, and misses are bad_args', () => {
+  const verdict = classifyBeforeExecution({
+    manifest,
+    expectedTool: 'sum_by_category',
+    selection: { tool: 'sum_by_category', arguments: { highlight: 'Dining' } },
+    expectation: { expectedArgs: { highlight: 'Groceries' } },
+  });
+  assert.equal(verdict.outcome, 'bad_args');
+  assert.equal(verdict.violations[0].kind, 'wrong_value');
+});
+
+test('a forbidden argument is over-reach, not a near miss', () => {
+  const verdict = classifyBeforeExecution({
+    manifest,
+    expectedTool: 'sum_by_category',
+    selection: { tool: 'sum_by_category', arguments: { highlight: 'Dining' } },
+    expectation: { forbiddenArgKeys: ['highlight'] },
+  });
+  assert.equal(verdict.outcome, 'bad_args');
+  assert.equal(verdict.violations[0].kind, 'forbidden_key');
+});
+
+test('constraint direction is enforced, so "be stricter" answered lower fails', () => {
+  const stricter = { requiredArgKeys: ['threshold'], argConstraints: { threshold: { gt: 2.5 } } };
+
+  assert.equal(
+    classifyBeforeExecution({
+      manifest,
+      expectedTool: 'find_anomalies',
+      selection: { tool: 'find_anomalies', arguments: { threshold: 1 } },
+      expectation: stricter,
+    }).violations[0].kind,
+    'constraint_violated'
+  );
+
+  assert.equal(
+    classifyBeforeExecution({
+      manifest,
+      expectedTool: 'find_anomalies',
+      selection: { tool: 'find_anomalies', arguments: { threshold: 4 } },
+      expectation: stricter,
+    }),
+    null
+  );
+});
+
+test('arguments outside the tool schema are rejected', () => {
+  const { valid, violations } = checkArguments({
+    args: { nonsense: 1 },
+    inputSchema: manifest.tools[0].inputSchema,
+  });
+  assert.equal(valid, false);
+  assert.equal(violations[0].kind, 'unknown_key');
+});
+
+test('a missing required key is reported even when nothing else is expected', () => {
+  const { violations } = checkArguments({
+    args: {},
+    expectation: { requiredArgKeys: ['threshold'] },
+    inputSchema: manifest.tools[1].inputSchema,
+  });
+  assert.deepEqual(violations, [{ kind: 'missing_required_key', key: 'threshold' }]);
+});
+
+test('a throwing call is exec_error', () => {
+  assert.equal(
+    classifyAfterExecution({ execution: { ok: false, error: 'boom' } }).outcome,
+    'exec_error'
+  );
+});
+
+test('silent_fail needs both an empty result and an unchanged page', () => {
+  const unchanged = { highlightClass: '' };
+
+  assert.equal(
+    classifyAfterExecution({ execution: { ok: true, result: null }, before: unchanged, after: unchanged })
+      .outcome,
+    'silent_fail'
+  );
+
+  // A read-only tool legitimately changes no DOM, so a payload alone is enough.
+  assert.equal(
+    classifyAfterExecution({
+      execution: { ok: true, result: { totals: [] } },
+      before: unchanged,
+      after: unchanged,
+    }).outcome,
+    'ok'
+  );
+
+  // And an empty payload with a visible change is a real effect.
+  assert.equal(
+    classifyAfterExecution({
+      execution: { ok: true, result: null },
+      before: unchanged,
+      after: { highlightClass: 'has-highlight' },
+    }).outcome,
+    'ok'
+  );
+});