webmcp-gauge 0.1.0

package/core/trial.mjs

@@ -0,0 +1,248 @@
+/**
+ * One trial: one utterance, one expected tool, one outcome.
+ *
+ * Fresh context per utterance is not a detail - conversational carry-over means
+ * trial N contaminates trial N+1 - so the judge is called with a single message
+ * pair and nothing else, and the tab is the caller's to discard afterwards.
+ */
+import { captureManifest, executeTool, observe, watchBrowserTools } from '../browser/webmcp.mjs';
+import { classifyAfterExecution, classifyBeforeExecution } from './taxonomy.mjs';
+
+const SYSTEM_PROMPT = `You are the tool-using layer of a web browser. The page below exposes tools you may call.
+
+Answer with a single JSON object and nothing else - no prose, no code fence:
+{"tool": "<tool name or null>", "arguments": {<arguments object>}}
+
+Rules:
+- Choose the one tool that best serves the user's request, or null if none of them does.
+- Pass only arguments that appear in the chosen tool's inputSchema, and only those the user's words justify.
+- Never invent argument values the user did not imply.`;
+
+const buildUserPrompt = ({ tools, utterance }) =>
+  [
+    'Tools available on this page:',
+    JSON.stringify(
+      tools.map((tool) => ({
+        name: tool.name,
+        description: tool.description,
+        inputSchema: tool.inputSchema,
+        annotations: tool.annotations,
+      })),
+      null,
+      2
+    ),
+    '',
+    'The user says:',
+    utterance,
+  ].join('\n');
+
+/**
+ * Parses the judge's reply without repairing it. A judge that cannot follow the
+ * contract is a measurement, so a parse failure is reported as an unparseable
+ * selection (which classifies as not_selected) rather than retried into shape.
+ */
+export const parseSelection = (content) => {
+  const text = (content ?? '').trim();
+  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  const candidate = (fenced ? fenced[1] : text).trim();
+
+  try {
+    const parsed = JSON.parse(candidate);
+    return {
+      tool: typeof parsed.tool === 'string' && parsed.tool.length > 0 ? parsed.tool : null,
+      arguments: parsed.arguments ?? {},
+      parsed: true,
+      usedFence: Boolean(fenced),
+    };
+  } catch (error) {
+    return { tool: null, arguments: {}, parsed: false, parseError: String(error.message ?? error) };
+  }
+};
+
+export const runTrial = async ({
+  session,
+  judge,
+  url,
+  toolName,
+  utterance,
+  expectation = {},
+  setup = null,
+  browserToolNames = null,
+  fixtureVersion = null,
+  /**
+   * Control mode inverts the question: no tool should be selected at all. The
+   * pipeline is identical up to selection, then stops - executing a tool for a
+   * control would mutate the page to measure nothing.
+   */
+  controlMode = false,
+}) => {
+  const startedAt = new Date().toISOString();
+
+  // Started before navigation, because the browser announces tools through
+  // `WebMCP.toolsAdded` events rather than through any command: a watch attached
+  // afterwards sees nothing and would read as "the browser surfaced none of them".
+  const browserWatch = await watchBrowserTools(session);
+
+  await session.navigate(url);
+  const manifest = await captureManifest(session);
+
+  // Two independent views of the same page. The page's own getTools() says what it
+  // believes it registered; the browser's says what an agent would be offered. A
+  // disagreement is the whole point of the not_discovered outcome, and is invisible
+  // from inside the page.
+  const browserNames = browserToolNames ?? browserWatch.names();
+  const browserTools = browserWatch.tools();
+  browserWatch.stop();
+  const pageNames = (manifest.tools ?? []).map((tool) => tool.name);
+  const browserView = {
+    available: browserWatch.available,
+    reason: browserWatch.reason,
+    toolCount: browserNames?.length ?? null,
+    // More than one frame means tools arrived from an embed. Chrome 152 folds those
+    // into the top frame's getTools(), so a host page's agent surface can include a
+    // widget's tools - worth recording per trial rather than discovering later.
+    frames: browserTools ? new Set(browserTools.map((tool) => tool.frameId)).size : null,
+    registeredButNotSurfaced: browserNames
+      ? pageNames.filter((name) => !browserNames.includes(name))
+      : null,
+    surfacedButNotInPage: browserNames
+      ? browserNames.filter((name) => !pageNames.includes(name))
+      : null,
+  };
+
+  // Utterances like "clear that" have no referent on a clean page, so the fixture
+  // declares the state they presuppose. The seed call is setup, never scored.
+  let seed = null;
+  if (setup?.seedCall && manifest.present) {
+    const seedResult = await executeTool(session, setup.seedCall.tool, setup.seedCall.args ?? {});
+    seed = { call: setup.seedCall, ok: seedResult.ok, error: seedResult.error ?? null };
+  }
+
+  const before = manifest.present ? await observe(session) : null;
+
+  let selection = null;
+  let judgeAnswer = null;
+  let judgeError = null;
+
+  if (manifest.present && (manifest.tools?.length ?? 0) > 0) {
+    try {
+      judgeAnswer = await judge.complete({
+        system: SYSTEM_PROMPT,
+        user: buildUserPrompt({ tools: manifest.tools, utterance: utterance.text }),
+      });
+      selection = parseSelection(judgeAnswer.content);
+    } catch (error) {
+      judgeError = String(error.message ?? error);
+    }
+  }
+
+  const preVerdict = controlMode
+    ? null
+    : classifyBeforeExecution({
+        manifest,
+        expectedTool: toolName,
+        selection,
+        expectation,
+        browserToolNames: browserNames,
+      });
+
+  const record = {
+    trial: {
+      utteranceId: utterance.id,
+      utterance: utterance.text,
+      tag: utterance.tag ?? null,
+      expectedTool: toolName,
+      fixtureVersion,
+      startedAt,
+      url,
+    },
+    client: {
+      cdpPort: session.port,
+      modelContextPresent: Boolean(manifest.present),
+      surface: manifest.surface ?? null,
+      settled: manifest.settled ?? null,
+      settledAtMs: manifest.settledAtMs ?? null,
+      toolCount: manifest.tools?.length ?? 0,
+      browserView,
+    },
+    judge: {
+      model: judgeAnswer?.model ?? judge.id,
+      requested: judge.id,
+      baseUrl: judge.baseUrl,
+      keySource: judge.keySource,
+      elapsedMs: judgeAnswer?.elapsedMs ?? null,
+      usage: judgeAnswer?.usage ?? null,
+      finishReason: judgeAnswer?.finishReason ?? null,
+      // The raw response travels with every trial: a number nobody can audit
+      // back to what the model actually said is not evidence.
+      rawResponse: judgeAnswer?.raw ?? null,
+      error: judgeError,
+    },
+    selection,
+    seed,
+  };
+
+  if (judgeError) {
+    // A judge that could not be reached has told us nothing about the page. Calling
+    // that not_selected would put transport failures inside the metric and quietly
+    // depress every rate; the sweep routes a null outcome to harness failures.
+    return {
+      ...record,
+      outcome: null,
+      harnessFailure: { kind: 'judge_unavailable', detail: judgeError },
+    };
+  }
+
+  if (judgeAnswer?.truncated && !judgeAnswer.content) {
+    const reasoning = judgeAnswer.usage?.completion_tokens_details?.reasoning_tokens ?? null;
+    return {
+      ...record,
+      outcome: null,
+      harnessFailure: {
+        kind: 'judge_truncated',
+        detail: `finish_reason=length with empty content; ${judgeAnswer.usage?.completion_tokens ?? '?'} completion tokens${reasoning === null ? '' : ` of which ${reasoning} reasoning`}. Raise maxTokens.`,
+      },
+    };
+  }
+
+  if (controlMode) {
+    if (!manifest.present) {
+      return { ...record, outcome: 'not_supported', reason: 'no modelContext on this client' };
+    }
+    if ((manifest.tools?.length ?? 0) === 0) {
+      return { ...record, outcome: 'not_registered', reason: 'page registered no tools' };
+    }
+    if (!selection?.tool) {
+      return { ...record, outcome: 'not_selected', reason: 'no tool selected, which is the pass' };
+    }
+    return {
+      ...record,
+      outcome: 'wrong_tool',
+      reason: `control utterance selected ${selection.tool}`,
+    };
+  }
+
+  if (preVerdict) {
+    return { ...record, outcome: preVerdict.outcome, reason: preVerdict.reason, violations: preVerdict.violations ?? null };
+  }
+
+  const execution = await executeTool(session, selection.tool, selection.arguments);
+  const after = await observe(session);
+  const postVerdict = classifyAfterExecution({ execution, before, after });
+
+  return {
+    ...record,
+    execution: {
+      ok: execution.ok,
+      callShape: execution.callShape ?? null,
+      // The rejected signatures are the compatibility data: which shapes a build
+      // refuses, and with what message, is the row worth publishing.
+      attempts: execution.attempts ?? null,
+      error: execution.error ?? null,
+      result: execution.result ?? null,
+    },
+    observation: { before, after, changed: JSON.stringify(before) !== JSON.stringify(after) },
+    outcome: postVerdict.outcome,
+    reason: postVerdict.reason,
+  };
+};

package/core/visibility.mjs

@@ -0,0 +1,163 @@
+/**
+ * Is this repository still invisible to a stranger? Classification rules only.
+ *
+ * The publishing policy is that the remote stays private until the report launch,
+ * so "still private?" is a precondition for every push. It was checked by hand on
+ * 2026-08-29 the right way — two independent signals, both of whose failure modes
+ * are informative — and then by hand again on 2026-09-03 the wrong way, in an
+ * inline one-liner whose catch block printed "404 = private" for **any** thrown
+ * error. DNS was down at that moment, so a transport failure that never received
+ * an HTTP status at all was reported as a private repo. The push was held only
+ * because the second signal named the real cause (`Could not resolve host`).
+ *
+ * That is the same defect as the `d.name`/`d.domain` bug corrected the same day: a
+ * check whose failure path cannot distinguish *no answer* from *the answer I
+ * expected* is not a check. So the rules live here, with tests, and the I/O lives
+ * in `probes/remote-visibility.mjs` — the split this project already uses for
+ * `cohort.mjs`, `gallery.mjs` and `scorecard.mjs`.
+ *
+ * Three verdicts, never two. `indeterminate` is a first-class answer: the only
+ * failure this gate must never produce is a confident "private" it did not measure.
+ *
+ * Exit codes follow the harness contract in `gate.mjs`, for the same reason:
+ *
+ *   0  every signal answered, and all of them say private
+ *   1  a signal definitively says public — a real answer that breaks the policy
+ *   2  cannot answer: a transport failure, an unexpected status, one lone signal,
+ *      or signals that disagree
+ */
+
+export const VISIBILITY_EXIT = { private: 0, public: 1, indeterminate: 2 };
+
+/**
+ * The slug comes from the configured remote rather than a constant, because a
+ * hardcoded `owner/repo` is how a check quietly starts testing a different
+ * repository than the one being pushed. Returns null for anything that is not a
+ * GitHub remote, which the caller must treat as "this signal does not apply"
+ * rather than as a pass.
+ */
+export const parseGitHubSlug = (remoteUrl) => {
+  if (typeof remoteUrl !== 'string' || remoteUrl.trim() === '') return null;
+  const url = remoteUrl.trim();
+
+  const patterns = [
+    /^https?:\/\/(?:[^@/]+@)?github\.com\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/i,
+    /^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?\/?$/i,
+    /^ssh:\/\/git@github\.com(?::\d+)?\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/i,
+  ];
+
+  for (const pattern of patterns) {
+    const match = url.match(pattern);
+    if (match) return { owner: match[1], repo: match[2] };
+  }
+  return null;
+};
+
+/**
+ * Signal 1 — an unauthenticated GitHub API read.
+ *
+ * `status: null` is the case that motivated this file: no response arrived, so
+ * there is nothing to read a verdict off. 403 and 429 are GitHub's rate limits,
+ * which are also not visibility answers. A 404 means "not visible anonymously",
+ * which covers both private and deleted — either way a stranger cannot read it,
+ * and either way a push is not made unsafe by it.
+ */
+export const classifyApiSignal = ({ status = null, transportError = null } = {}) => {
+  if (transportError) {
+    return { verdict: 'indeterminate', detail: `no HTTP response (${transportError})` };
+  }
+  if (status === null || status === undefined) {
+    return { verdict: 'indeterminate', detail: 'no HTTP status was captured' };
+  }
+  if (status === 200) return { verdict: 'public', detail: 'HTTP 200 — readable without credentials' };
+  if (status === 404) return { verdict: 'private', detail: 'HTTP 404 — not visible anonymously' };
+  if (status === 401 || status === 403 || status === 429) {
+    return { verdict: 'indeterminate', detail: `HTTP ${status} — rate limit or auth, not a visibility answer` };
+  }
+  return { verdict: 'indeterminate', detail: `HTTP ${status} — unexpected, treated as no answer` };
+};
+
+const DEMANDS_CREDENTIALS =
+  /could not read Username|could not read Password|terminal prompts disabled|Authentication failed|Invalid username or (?:password|token)|Password authentication is not supported|Permission denied \(publickey\)|Repository not found/i;
+
+const TRANSPORT_FAILURE =
+  /could not resolve host|couldn't resolve host|connection timed out|failed to connect|operation timed out|network is unreachable|connection reset|SSL certificate problem|proxy/i;
+
+/**
+ * Signal 2 — an anonymous `git ls-remote`, credential helper disabled and prompts
+ * off, so a private repo has no way to succeed.
+ *
+ * Order matters. Git reports a DNS failure as `fatal: unable to access '…':
+ * Could not resolve host`, and an HTTP error as `unable to access '…': The
+ * requested URL returned error: 403` — the shared prefix is worthless, so the
+ * transport patterns are checked before anything is concluded, and only a message
+ * that actually demands credentials counts as private.
+ *
+ * Exit 0 with no refs is deliberately indeterminate: an empty *public* repository
+ * answers exactly that way, and this project's remote has a `main` to report.
+ */
+export const classifyLsRemoteSignal = ({ code = null, stdout = '', stderr = '' } = {}) => {
+  const out = String(stdout);
+  const err = String(stderr);
+
+  if (/refs\/heads\//.test(out)) {
+    return { verdict: 'public', detail: 'refs listed anonymously' };
+  }
+  if (TRANSPORT_FAILURE.test(err)) {
+    return { verdict: 'indeterminate', detail: `transport failure (${firstLine(err)})` };
+  }
+  if (DEMANDS_CREDENTIALS.test(err)) {
+    return { verdict: 'private', detail: `credentials demanded (${firstLine(err)})` };
+  }
+  if (code === 0) {
+    return { verdict: 'indeterminate', detail: 'connected and listed no refs — an empty public repo looks like this' };
+  }
+  return { verdict: 'indeterminate', detail: err.trim() === '' ? `exit ${code}, no output` : `exit ${code}: ${firstLine(err)}` };
+};
+
+const firstLine = (text) => String(text).trim().split('\n')[0].slice(0, 200);
+
+/**
+ * One signal cannot audit itself — that is the whole lesson of 2026-09-03, twice
+ * over — so a `private` pass requires at least two signals that all say private.
+ * A single `public` outranks everything: it is a real answer, and it is the one
+ * answer that must stop a push.
+ */
+export const combineSignals = (signals) => {
+  const list = Array.isArray(signals) ? signals : [];
+  if (list.length === 0) {
+    return { verdict: 'indeterminate', exitCode: VISIBILITY_EXIT.indeterminate, reason: 'no signals were collected' };
+  }
+
+  const publics = list.filter((signal) => signal.verdict === 'public');
+  if (publics.length > 0) {
+    return {
+      verdict: 'public',
+      exitCode: VISIBILITY_EXIT.public,
+      reason: `readable without credentials — ${publics.map((signal) => signal.detail).join('; ')}`,
+    };
+  }
+
+  const unknown = list.filter((signal) => signal.verdict !== 'private');
+  if (unknown.length > 0) {
+    return {
+      verdict: 'indeterminate',
+      exitCode: VISIBILITY_EXIT.indeterminate,
+      reason: `cannot conclude — ${unknown.map((signal) => `${signal.name ?? 'signal'}: ${signal.detail}`).join('; ')}`,
+    };
+  }
+
+  if (list.length < 2) {
+    return {
+      verdict: 'indeterminate',
+      exitCode: VISIBILITY_EXIT.indeterminate,
+      reason: 'only one signal answered, and one signal cannot audit itself',
+    };
+  }
+
+  return {
+    verdict: 'private',
+    exitCode: VISIBILITY_EXIT.private,
+    reason: `${list.length} independent signals agree the remote is not readable without credentials`,
+  };
+};

package/core/visibility.test.mjs

@@ -0,0 +1,164 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+  VISIBILITY_EXIT,
+  classifyApiSignal,
+  classifyLsRemoteSignal,
+  combineSignals,
+  parseGitHubSlug,
+} from './visibility.mjs';
+
+/**
+ * The first test is the bug this module exists for: on 2026-09-03 an inline check
+ * printed "404 = private" when DNS was down and no HTTP status existed at all.
+ */
+test('a transport failure is never a private verdict, because no answer arrived', () => {
+  const signal = classifyApiSignal({ transportError: 'getaddrinfo ENOTFOUND api.github.com' });
+  assert.equal(signal.verdict, 'indeterminate');
+  assert.match(signal.detail, /no HTTP response/);
+});
+
+test('a missing status is indeterminate even with no error to report', () => {
+  assert.equal(classifyApiSignal({}).verdict, 'indeterminate');
+  assert.equal(classifyApiSignal({ status: null }).verdict, 'indeterminate');
+});
+
+test('200 is public and 404 is private, and 404 says what it really means', () => {
+  assert.equal(classifyApiSignal({ status: 200 }).verdict, 'public');
+  const notFound = classifyApiSignal({ status: 404 });
+  assert.equal(notFound.verdict, 'private');
+  assert.match(notFound.detail, /not visible anonymously/);
+});
+
+test('a rate limit is not a visibility answer', () => {
+  for (const status of [401, 403, 429]) {
+    const signal = classifyApiSignal({ status });
+    assert.equal(signal.verdict, 'indeterminate', `HTTP ${status} must not conclude`);
+  }
+});
+
+test('an unexpected status is treated as no answer rather than guessed at', () => {
+  assert.equal(classifyApiSignal({ status: 500 }).verdict, 'indeterminate');
+  assert.equal(classifyApiSignal({ status: 301 }).verdict, 'indeterminate');
+});
+
+test('listed refs mean a stranger can read the repository', () => {
+  const signal = classifyLsRemoteSignal({
+    code: 0,
+    stdout: '3486bc838f9e1debabb49cd5d44fd25368067a11\trefs/heads/main\n',
+  });
+  assert.equal(signal.verdict, 'public');
+});
+
+test('a demand for credentials is the private signature', () => {
+  const signal = classifyLsRemoteSignal({
+    code: 128,
+    stderr: "fatal: could not read Username for 'https://github.com': terminal prompts disabled",
+  });
+  assert.equal(signal.verdict, 'private');
+  assert.match(signal.detail, /credentials demanded/);
+});
+
+/**
+ * Observed 2026-09-03 with a credential helper in the environment: the refusal
+ * arrives as GitHub's own wording rather than git's, so matching only git's
+ * phrasing would have downgraded a real private answer to indeterminate.
+ */
+test('GitHub\u2019s own rejection wording counts as the private signature too', () => {
+  for (const stderr of [
+    'remote: Invalid username or token. Password authentication is not supported for Git operations.',
+    "fatal: Authentication failed for 'https://github.com/owner/repo/'",
+    'remote: Repository not found.',
+  ]) {
+    assert.equal(classifyLsRemoteSignal({ code: 128, stderr }).verdict, 'private', stderr);
+  }
+});
+
+/**
+ * Both of these begin `fatal: unable to access`, so the shared prefix decides
+ * nothing and the transport patterns have to be tested before any conclusion.
+ */
+test('a DNS failure is indeterminate, not private, however fatal it looks', () => {
+  const signal = classifyLsRemoteSignal({
+    code: 128,
+    stderr: "fatal: unable to access 'https://github.com/owner/repo/': Could not resolve host: github.com",
+  });
+  assert.equal(signal.verdict, 'indeterminate');
+  assert.match(signal.detail, /transport failure/);
+});
+
+test('a timeout is indeterminate too', () => {
+  const signal = classifyLsRemoteSignal({ code: 128, stderr: 'fatal: unable to access: Connection timed out after 15000 ms' });
+  assert.equal(signal.verdict, 'indeterminate');
+});
+
+test('connecting and listing nothing is indeterminate, because an empty public repo looks the same', () => {
+  const signal = classifyLsRemoteSignal({ code: 0, stdout: '' });
+  assert.equal(signal.verdict, 'indeterminate');
+});
+
+test('an unclassifiable failure reports its exit code instead of picking a side', () => {
+  const signal = classifyLsRemoteSignal({ code: 129, stderr: 'error: unknown option `--nope`' });
+  assert.equal(signal.verdict, 'indeterminate');
+  assert.match(signal.detail, /exit 129/);
+});
+
+test('two private signals are a pass, and that is the only pass', () => {
+  const verdict = combineSignals([
+    { name: 'api', verdict: 'private', detail: 'HTTP 404' },
+    { name: 'ls-remote', verdict: 'private', detail: 'credentials demanded' },
+  ]);
+  assert.equal(verdict.verdict, 'private');
+  assert.equal(verdict.exitCode, VISIBILITY_EXIT.private);
+});
+
+test('one lone private signal does not pass, because one signal cannot audit itself', () => {
+  const verdict = combineSignals([{ name: 'api', verdict: 'private', detail: 'HTTP 404' }]);
+  assert.equal(verdict.verdict, 'indeterminate');
+  assert.equal(verdict.exitCode, VISIBILITY_EXIT.indeterminate);
+  assert.match(verdict.reason, /one signal cannot audit itself/);
+});
+
+test('a single public signal outranks everything else', () => {
+  const verdict = combineSignals([
+    { name: 'api', verdict: 'private', detail: 'HTTP 404' },
+    { name: 'ls-remote', verdict: 'public', detail: 'refs listed anonymously' },
+  ]);
+  assert.equal(verdict.verdict, 'public');
+  assert.equal(verdict.exitCode, VISIBILITY_EXIT.public);
+});
+
+test('one indeterminate signal blocks a pass and names the signal that failed', () => {
+  const verdict = combineSignals([
+    { name: 'api', verdict: 'indeterminate', detail: 'no HTTP response (ENOTFOUND)' },
+    { name: 'ls-remote', verdict: 'private', detail: 'credentials demanded' },
+  ]);
+  assert.equal(verdict.exitCode, VISIBILITY_EXIT.indeterminate);
+  assert.match(verdict.reason, /api: no HTTP response/);
+});
+
+test('no signals at all is indeterminate rather than vacuously private', () => {
+  assert.equal(combineSignals([]).exitCode, VISIBILITY_EXIT.indeterminate);
+  assert.equal(combineSignals(undefined).exitCode, VISIBILITY_EXIT.indeterminate);
+});
+
+test('the slug comes off any spelling of a GitHub remote', () => {
+  const expected = { owner: 'Svishwa2004', repo: 'webmcp-gauge' };
+  for (const url of [
+    'https://github.com/Svishwa2004/webmcp-gauge',
+    'https://github.com/Svishwa2004/webmcp-gauge.git',
+    'https://github.com/Svishwa2004/webmcp-gauge/',
+    'https://token@github.com/Svishwa2004/webmcp-gauge.git',
+    'git@github.com:Svishwa2004/webmcp-gauge.git',
+    'ssh://git@github.com/Svishwa2004/webmcp-gauge.git',
+  ]) {
+    assert.deepEqual(parseGitHubSlug(url), expected, url);
+  }
+});
+
+test('a non-GitHub or unusable remote yields null rather than a guessed slug', () => {
+  for (const url of ['https://gitlab.com/owner/repo.git', 'file:///tmp/repo', '', null, undefined, 'not a url']) {
+    assert.equal(parseGitHubSlug(url), null, String(url));
+  }
+});