webmcp-gauge 0.1.0

package/browser/webmcp.test.mjs

@@ -0,0 +1,299 @@
+/**
+ * The browser-side tool view, which is the only way `not_discovered` can ever fire.
+ *
+ * Tested against a fake session because what matters is the accumulation contract,
+ * not Chrome: the WebMCP domain has no command that lists tools, so the view is
+ * assembled from a stream of events and every rule about that assembly - one event
+ * per registration, removals, an unavailable domain, unsubscribing - is logic this
+ * repo owns.
+ */
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { watchBrowserTools, watchBrowserToolsAtBrowser } from './webmcp.mjs';
+
+const fakeSession = ({ available = true, reason = null } = {}) => {
+  const subscribers = new Map();
+  return {
+    subscribe(method, handler) {
+      const handlers = subscribers.get(method) ?? new Set();
+      handlers.add(handler);
+      subscribers.set(method, handlers);
+      return () => handlers.delete(handler);
+    },
+    async enableWebMcpDomain() {
+      return available ? { available: true } : { available: false, reason };
+    },
+    emit(method, params) {
+      for (const handler of [...(subscribers.get(method) ?? [])]) handler(params);
+    },
+    subscriberCount() {
+      let total = 0;
+      for (const handlers of subscribers.values()) total += handlers.size;
+      return total;
+    },
+  };
+};
+
+const tool = (name) => ({ name, description: `${name} does something`, frameId: 'F1' });
+
+test('the view accumulates across one event per registration, which is how Chrome sends them', async () => {
+  const session = fakeSession();
+  const watch = await watchBrowserTools(session);
+
+  // Measured on Chrome 152: each registerTool produces its own toolsAdded event, so
+  // a reader that waits for one event sees one tool and calls the rest missing.
+  session.emit('WebMCP.toolsAdded', { tools: [tool('describe_dataset')] });
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows')] });
+  session.emit('WebMCP.toolsAdded', { tools: [tool('top_expenses'), tool('monthly_trend')] });
+
+  assert.deepEqual(watch.names(), [
+    'describe_dataset',
+    'filter_rows',
+    'top_expenses',
+    'monthly_trend',
+  ]);
+  assert.equal(watch.tools().length, 4);
+  assert.equal(watch.tools()[0].frameId, 'F1', 'the browser knows which frame registered it');
+});
+
+test('a removed tool leaves the view and is remembered as removed', async () => {
+  const session = fakeSession();
+  const watch = await watchBrowserTools(session);
+
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows'), tool('clear_highlights')] });
+  session.emit('WebMCP.toolsRemoved', { tools: [{ name: 'clear_highlights', frameId: 'F1' }] });
+
+  assert.deepEqual(watch.names(), ['filter_rows']);
+  assert.deepEqual(watch.removedNames(), ['clear_highlights']);
+});
+
+test('re-registering a name after removal puts it back exactly once', async () => {
+  const session = fakeSession();
+  const watch = await watchBrowserTools(session);
+
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows')] });
+  session.emit('WebMCP.toolsRemoved', { tools: [{ name: 'filter_rows' }] });
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows')] });
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows')] });
+
+  // Names are the unit of selection, so the view is keyed by name: two tools with
+  // one name are one thing an agent can call, not two.
+  assert.deepEqual(watch.names(), ['filter_rows']);
+});
+
+test('an unavailable domain reports null rather than an empty list', async () => {
+  const session = fakeSession({ available: false, reason: "'WebMCP.enable' wasn't found" });
+  const watch = await watchBrowserTools(session);
+
+  // The distinction the classifier depends on: no view at all must never read as
+  // "the browser surfaced nothing", or every trial on an older build would become
+  // not_discovered.
+  assert.equal(watch.available, false);
+  assert.equal(watch.names(), null);
+  assert.equal(watch.tools(), null);
+  assert.equal(watch.removedNames(), null);
+  assert.match(watch.reason, /WebMCP\.enable/);
+  assert.equal(session.subscriberCount(), 0, 'an unavailable watch must not leave subscribers behind');
+});
+
+test('events without a usable name are ignored rather than crashing the reader', async () => {
+  const session = fakeSession();
+  const watch = await watchBrowserTools(session);
+
+  session.emit('WebMCP.toolsAdded', {});
+  session.emit('WebMCP.toolsAdded', { tools: [{ description: 'no name' }, null] });
+  session.emit('WebMCP.toolsRemoved', { tools: [{ frameId: 'F2' }] });
+
+  assert.deepEqual(watch.names(), []);
+  assert.deepEqual(watch.removedNames(), []);
+});
+
+test('stop() unsubscribes, so a closed trial cannot keep collecting', async () => {
+  const session = fakeSession();
+  const watch = await watchBrowserTools(session);
+
+  session.emit('WebMCP.toolsAdded', { tools: [tool('filter_rows')] });
+  assert.equal(session.subscriberCount(), 2);
+
+  watch.stop();
+  session.emit('WebMCP.toolsAdded', { tools: [tool('late_tool')] });
+
+  assert.equal(session.subscriberCount(), 0);
+  assert.deepEqual(watch.names(), ['filter_rows']);
+});
+
+/**
+ * The browser-endpoint view is tested against a fake socket for the same reason
+ * the host-attached one is tested against a fake session: the accumulation
+ * contract across sessions is this repo's logic, and Chrome is not. The fake
+ * answers CDP commands the way a build does — every command succeeds except
+ * `WebMCP.enable` when the scenario wants a build without the domain — and the
+ * tests drive `attachedToTarget` and the WebMCP events in the shapes Chrome
+ * delivers in flatten mode, tagged with the `sessionId` of the target each
+ * event came from.
+ */
+class FakeSocket {
+  constructor({ enableResult = 'ok' } = {}) {
+    this.enableResult = enableResult;
+    this.sent = [];
+    this.closed = false;
+    this.handlers = new Map();
+    queueMicrotask(() => this.emit('open'));
+  }
+
+  addEventListener(type, handler) {
+    const list = this.handlers.get(type) ?? [];
+    list.push(handler);
+    this.handlers.set(type, list);
+  }
+
+  emit(type, event = {}) {
+    for (const handler of [...(this.handlers.get(type) ?? [])]) handler(event);
+  }
+
+  send(text) {
+    const message = JSON.parse(text);
+    this.sent.push(message);
+    const reply =
+      message.method === 'WebMCP.enable' && this.enableResult !== 'ok'
+        ? { error: { code: -32601, message: this.enableResult } }
+        : { result: {} };
+    queueMicrotask(() => this.emit('message', { data: JSON.stringify({ id: message.id, ...reply }) }));
+  }
+
+  close() {
+    this.closed = true;
+  }
+}
+
+const openWatch = async ({ enableResult = 'ok' } = {}) => {
+  const socket = new FakeSocket({ enableResult });
+  const watch = await watchBrowserToolsAtBrowser('ws://browser', {
+    WebSocket: class {
+      constructor() {
+        return socket;
+      }
+    },
+  });
+  return { watch, socket };
+};
+
+/** Attaches a target the way flattened auto-attach reports one. */
+const attach = (socket, sessionId, type = 'page') =>
+  socket.emit('message', {
+    data: JSON.stringify({
+      method: 'Target.attachedToTarget',
+      params: {
+        sessionId,
+        targetInfo: { targetId: `T-${sessionId}`, type, url: 'http://site.example/' },
+      },
+    }),
+  });
+
+const added = (socket, sessionId, names) =>
+  socket.emit('message', {
+    data: JSON.stringify({
+      method: 'WebMCP.toolsAdded',
+      sessionId,
+      params: { tools: names.map((name) => ({ name, description: `${name} does something`, frameId: 'F1' })) },
+    }),
+  });
+
+const removedOne = (socket, sessionId, name) =>
+  socket.emit('message', {
+    data: JSON.stringify({
+      method: 'WebMCP.toolsRemoved',
+      sessionId,
+      params: { tools: [{ name, frameId: 'F1' }] },
+    }),
+  });
+
+const settle = () => new Promise((done) => setTimeout(done, 0));
+
+test('the browser-endpoint view reaches a cross-site embed the host session cannot hear', async () => {
+  const { watch, socket } = await openWatch();
+
+  attach(socket, 'S1', 'page');
+  attach(socket, 'S2', 'iframe');
+  await settle();
+
+  added(socket, 'S1', ['host_alpha', 'host_beta', 'host_gamma']);
+  added(socket, 'S2', ['widget_ping']);
+
+  // The 2026-09-05 measurement, replayed: 4 tools across 2 sessions where the
+  // host-attached view stops at 3. The counters are what make the number
+  // interpretable — an iframe session did attach, and the union came from both.
+  assert.deepEqual(watch.names(), ['host_alpha', 'host_beta', 'host_gamma', 'widget_ping']);
+  assert.equal(watch.tools().length, 4);
+  assert.equal(watch.available, true);
+  assert.equal(watch.oopiFrames, 1);
+  assert.equal(watch.toolSessionCount, 2);
+
+  // Recursion is the fix, so it is pinned: one arm on the browser session, then
+  // one per attached session. A single arm measured 6 targets with no iframe
+  // among them and saw 3 tools.
+  const arms = socket.sent.filter((message) => message.method === 'Target.setAutoAttach');
+  assert.equal(arms.length, 3, 'auto-attach must be armed on the browser session and on every attached session');
+  assert.equal(arms[1].sessionId, 'S1');
+  assert.equal(arms[2].sessionId, 'S2');
+});
+
+test('a removal from one session leaves the union and is remembered', async () => {
+  const { watch, socket } = await openWatch();
+
+  attach(socket, 'S1', 'page');
+  attach(socket, 'S2', 'iframe');
+  await settle();
+
+  added(socket, 'S1', ['host_alpha', 'host_beta']);
+  added(socket, 'S2', ['widget_ping']);
+  removedOne(socket, 'S2', 'widget_ping');
+
+  assert.deepEqual(watch.names(), ['host_alpha', 'host_beta']);
+  assert.deepEqual(watch.removedNames(), ['widget_ping']);
+});
+
+test('a build whose every enable refuses reports unavailable, not empty', async () => {
+  const { watch, socket } = await openWatch({ enableResult: "'WebMCP.enable' wasn't found" });
+
+  attach(socket, 'S1', 'page');
+  attach(socket, 'S2', 'iframe');
+  await settle();
+
+  added(socket, 'S1', ['host_alpha']);
+
+  // The distinction the classifier depends on travels across sessions too: no
+  // view at all must never read as "the browser surfaced nothing".
+  assert.equal(watch.available, false);
+  assert.equal(watch.names(), null);
+  assert.equal(watch.tools(), null);
+  assert.equal(watch.removedNames(), null);
+  assert.match(watch.reason, /WebMCP\.enable/);
+});
+
+test('a run where no target ever attached reports that, not an empty union', async () => {
+  const { watch } = await openWatch();
+
+  // The same exit-2 distinction browser-scope.mjs had to learn the hard way:
+  // "auto-attach reached nothing" and "the browser cannot see it" are different
+  // findings, and only one of them is about WebMCP.
+  assert.equal(watch.available, false);
+  assert.equal(watch.names(), null);
+  assert.equal(watch.oopiFrames, 0);
+  assert.equal(watch.toolSessionCount, 0);
+  assert.match(watch.reason, /no target session attached/);
+});
+
+test('stop() closes the socket, so a closed capture cannot keep collecting', async () => {
+  const { watch, socket } = await openWatch();
+
+  attach(socket, 'S1', 'page');
+  await settle();
+  added(socket, 'S1', ['host_alpha']);
+
+  watch.stop();
+  assert.equal(socket.closed, true);
+
+  added(socket, 'S1', ['late_tool']);
+  assert.deepEqual(watch.names(), ['host_alpha']);
+});

package/core/args.mjs

@@ -0,0 +1,93 @@
+/**
+ * One option parser for the whole repository, because two were one too many.
+ *
+ * Until 2026-09-03 `bin/` and `probes/` accepted **opposite** syntaxes and neither
+ * complained about the other's:
+ *
+ *   - `bin/webmcp-gauge.mjs` read `--name value` and turned `--fail-under=0.9`
+ *     into a *switch* named `fail-under=0.9`, so `flags['fail-under']` was
+ *     undefined and the build was **not gated at all**. Latent rather than live —
+ *     every documented example and `action.yml` use the space form — but a CI job
+ *     written the other way would have passed silently forever.
+ *   - the probes read `--name=value` and ignored `--serve fixtures/gallery`
+ *     entirely, which walked the *live* gallery while claiming to walk a fixture.
+ *
+ * Same defect twice, in opposite directions: an option that does not arrive leaves
+ * a default in place and says nothing. So both forms are accepted everywhere now,
+ * and the three ways of getting it wrong are refused instead of absorbed:
+ *
+ *   1. an unknown option           — a typo silently kept the default before
+ *   2. a value option with no value
+ *   3. a value handed to a switch
+ *
+ * Returns `{ options, positional, error }` and never throws: each caller owns its
+ * own exit code, and in this repo those codes carry meaning (`core/gate.mjs`).
+ *
+ * One older lesson is kept from the CLI's own parser, which this replaces: argv
+ * must be walked once, in order. Collecting positionals by filtering on "does not
+ * start with `--`" looks equivalent and is not — it swallowed the judge model and
+ * the port as positionals, and the first of them became the subject url.
+ */
+
+/**
+ * @param {string[]} argv
+ * @param {{ values?: string[], switches?: string[], maxPositional?: number }} spec
+ */
+export const parseOptions = (argv, { values = [], switches = [], maxPositional = 0 } = {}) => {
+  const list = Array.isArray(argv) ? argv.map((entry) => String(entry)) : [];
+  const valueNames = new Set(values);
+  const switchNames = new Set(switches);
+
+  const options = {};
+  const positional = [];
+  const fail = (error) => ({ options, positional, error });
+
+  for (let index = 0; index < list.length; index += 1) {
+    const token = list[index];
+
+    if (!token.startsWith('--')) {
+      positional.push(token);
+      if (positional.length > maxPositional) {
+        const previous = index > 0 ? list[index - 1] : null;
+        // The most common way to land here is the syntax that used to be ignored,
+        // so name the fix rather than only the symptom.
+        const hint =
+          previous && previous.startsWith('--') && !previous.includes('=')
+            ? ` — if it is the value for ${previous}, both ${previous}=${token} and ${previous} ${token} are accepted, but ${previous} takes no value`
+            : '';
+        return fail(`unexpected argument '${token}'${hint}`);
+      }
+      continue;
+    }
+
+    const equals = token.indexOf('=');
+    const name = equals === -1 ? token.slice(2) : token.slice(2, equals);
+    const inlineValue = equals === -1 ? null : token.slice(equals + 1);
+
+    if (!valueNames.has(name) && !switchNames.has(name)) {
+      return fail(`unknown option '--${name}'. Known options: ${[...valueNames, ...switchNames].sort().map((known) => `--${known}`).join(', ')}`);
+    }
+
+    if (switchNames.has(name)) {
+      if (inlineValue !== null) return fail(`--${name} is a switch and takes no value, got '--${name}=${inlineValue}'`);
+      options[name] = true;
+      continue;
+    }
+
+    if (inlineValue !== null) {
+      options[name] = inlineValue;
+      continue;
+    }
+
+    const next = list[index + 1];
+    // A following `--something` is the next option, not this one's value: the
+    // alternative swallows options and produces a plausible wrong run.
+    if (next === undefined || next.startsWith('--')) {
+      return fail(`--${name} needs a value, e.g. --${name}=<value>`);
+    }
+    options[name] = next;
+    index += 1;
+  }
+
+  return { options, positional, error: null };
+};

package/core/args.test.mjs

@@ -0,0 +1,85 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { parseOptions } from './args.mjs';
+
+const spec = { values: ['url', 'fail-under', 'serve'], switches: ['json', 'resume'], maxPositional: 1 };
+
+test('both option syntaxes mean the same thing', () => {
+  const equals = parseOptions(['--url=https://example.com', '--fail-under=0.9'], spec);
+  const spaced = parseOptions(['--url', 'https://example.com', '--fail-under', '0.9'], spec);
+  assert.equal(equals.error, null);
+  assert.equal(spaced.error, null);
+  assert.deepEqual(equals.options, spaced.options);
+  assert.equal(equals.options['fail-under'], '0.9');
+});
+
+/**
+ * The two bugs this parser exists for, one per surface, both of which left a
+ * default in place and said nothing: `--fail-under=0.9` was a switch named
+ * `fail-under=0.9` in the CLI, so nothing was gated; `--serve fixtures/gallery`
+ * never reached the probes, so a rehearsal walked the live gallery.
+ */
+test('the equals form reaches the gate, which it did not in the old CLI parser', () => {
+  const { options } = parseOptions(['--fail-under=0.9'], spec);
+  assert.equal(options['fail-under'], '0.9');
+  assert.equal(options['fail-under=0.9'], undefined);
+});
+
+test('the space form reaches the probes, which it did not before', () => {
+  const { options, error } = parseOptions(['--serve', 'fixtures/gallery', '--json'], spec);
+  assert.equal(error, null);
+  assert.equal(options.serve, 'fixtures/gallery');
+  assert.equal(options.json, true);
+});
+
+test('an unknown option is refused and the known ones are listed', () => {
+  const { error } = parseOptions(['--fail-undr=0.9'], spec);
+  assert.match(error, /unknown option '--fail-undr'/);
+  assert.match(error, /--fail-under/);
+});
+
+test('a value option with no value is refused rather than read as true', () => {
+  assert.match(parseOptions(['--url'], spec).error, /--url needs a value/);
+  assert.match(parseOptions(['--url', '--json'], spec).error, /--url needs a value/);
+});
+
+test('a switch handed a value is refused rather than silently truthy', () => {
+  assert.match(parseOptions(['--json=yes'], spec).error, /--json is a switch and takes no value/);
+});
+
+test('a switch does not swallow the token after it', () => {
+  const { options, positional, error } = parseOptions(['--json', 'subject.html'], spec);
+  assert.equal(error, null);
+  assert.equal(options.json, true);
+  assert.deepEqual(positional, ['subject.html']);
+});
+
+test('a positional beyond the declared budget is refused, and names the option it may belong to', () => {
+  const { error } = parseOptions(['--json', 'one', 'two'], spec);
+  assert.match(error, /unexpected argument 'two'/);
+});
+
+test('`--resume yes` is refused, and the message says the switch takes no value', () => {
+  const { error } = parseOptions(['--resume', 'yes'], { ...spec, maxPositional: 0 });
+  assert.match(error, /unexpected argument 'yes'/);
+  assert.match(error, /--resume takes no value/);
+});
+
+test('an equals value keeps everything after the first equals', () => {
+  const { options } = parseOptions(['--url=https://example.com/?a=1&b=2'], spec);
+  assert.equal(options.url, 'https://example.com/?a=1&b=2');
+});
+
+test('a value that looks like a flag is refused rather than consumed', () => {
+  const { error } = parseOptions(['--serve', '--json'], spec);
+  assert.match(error, /--serve needs a value/);
+});
+
+test('empty argv parses to nothing', () => {
+  const { options, positional, error } = parseOptions([], spec);
+  assert.equal(error, null);
+  assert.deepEqual(options, {});
+  assert.deepEqual(positional, []);
+  assert.equal(parseOptions(undefined, spec).error, null);
+});

package/core/capture-seam.test.mjs

@@ -0,0 +1,86 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { pickTarget, toHarvest } from './gallery.mjs';
+import { normalizeTargets } from './cohort.mjs';
+
+/**
+ * The seam between the two halves of the one-day capture.
+ *
+ * `gallery-harvest.mjs` writes a targets file; `cohort-snapshot.mjs` reads it.
+ * Both halves were tested in isolation before gallery-publish day and the join
+ * between them never was — so if `toHarvest`'s row shape and
+ * `normalizeTargets`'s accepted shape ever disagree, the discovery would happen
+ * on the one day the capture cannot be repeated. These tests are that join.
+ */
+
+const harvestOf = (projects) =>
+  toHarvest(
+    projects.map((project) => pickTarget(project)),
+    { source: 'https://example.invalid/project-gallery', harvestedAt: '2026-09-04T00:00:00.000Z' }
+  );
+
+test('a harvest\u2019s targets are accepted by the snapshot without translation', () => {
+  const harvest = harvestOf([
+    {
+      title: 'Project One',
+      devpostUrl: 'https://devpost.com/software/one',
+      links: [{ url: 'https://one.example.com/', label: 'Try it' }, { url: 'https://github.com/o/one', label: 'Code' }],
+    },
+    {
+      title: 'Project Two',
+      devpostUrl: 'https://devpost.com/software/two',
+      links: [{ url: 'https://two.example.com/app', label: 'Demo' }],
+    },
+  ]);
+
+  assert.equal(harvest.counts.usable, 2);
+
+  const normalized = normalizeTargets(harvest.targets);
+  assert.equal(normalized.length, 2);
+  assert.deepEqual(
+    normalized.map((row) => row.url),
+    ['https://one.example.com/', 'https://two.example.com/app']
+  );
+});
+
+test('a submission with no demo link never reaches the snapshot, and keeps its reason', () => {
+  const harvest = harvestOf([
+    { title: 'No Demo', devpostUrl: 'https://devpost.com/software/nodemo', links: [{ url: 'https://github.com/o/nd' }] },
+    { title: 'Has Demo', devpostUrl: 'https://devpost.com/software/hasdemo', links: [{ url: 'https://demo.example.com/' }] },
+  ]);
+
+  assert.equal(harvest.counts.usable, 1);
+  assert.equal(harvest.skipped.length, 1);
+  assert.equal(harvest.skipped[0].reason, 'no demo-class link on the submission');
+  assert.equal(normalizeTargets(harvest.targets).length, 1);
+});
+
+/**
+ * The two halves dedupe on different keys — the harvest by project, the snapshot
+ * by canonical URL — so two submissions pointing at one deployment are two rows
+ * and one capture. That is correct, and it means the census cannot report
+ * "projects" and "captures" as the same number.
+ */
+test('two submissions sharing one deployment are two harvest rows and one capture', () => {
+  const harvest = harvestOf([
+    { title: 'Team A', devpostUrl: 'https://devpost.com/software/a', links: [{ url: 'https://shared.example.com/app' }] },
+    { title: 'Team B', devpostUrl: 'https://devpost.com/software/b', links: [{ url: 'https://shared.example.com/app#try' }] },
+  ]);
+
+  assert.equal(harvest.counts.usable, 2);
+  assert.equal(normalizeTargets(harvest.targets).length, 1);
+});
+
+test('an empty harvest normalizes to nothing rather than throwing', () => {
+  const harvest = harvestOf([]);
+  assert.equal(harvest.counts.usable, 0);
+  assert.deepEqual(normalizeTargets(harvest.targets), []);
+});
+
+/**
+ * The option-syntax half of this story lives in `core/args.test.mjs` now. It began
+ * here on 2026-09-03, when `--serve fixtures/gallery` walked the **live** gallery
+ * because the probes read `--name=value` only — then `bin/` turned out to have the
+ * mirror-image bug, so both went onto one parser and the tests followed it.
+ */