webmcp-gauge 0.1.0

package/report/published-runs.test.mjs

@@ -0,0 +1,77 @@
+/**
+ * The published record's own invariants.
+ *
+ * `reports/README.md` states the rule: every run is `<page>-<set>-<judge>-<shape>`,
+ * and "`.json` is the machine record — same numbers, plus the per-tool outcome
+ * counts, the coverage diff and the gate verdict."
+ *
+ * That rule was written down and not enforced, and on 2026-09-05 a reviewer with
+ * no repository access found the gap by reading the draft alone: the Edge
+ * second-client run — the newest and most quotable result in the folder — had a
+ * write-up and no machine record. Its `report.json` had sat in `artifacts/edge-s1/`
+ * since 2026-09-03 and was never copied. Nothing failed, because nothing checked.
+ *
+ * So the rule is a test now. It is deliberately narrow: it does not validate the
+ * contents of a report (`emit` and `gate` own that), only that the pair exists and
+ * that a machine record is the schema it claims to be. A published rate whose
+ * numbers cannot be re-read by a machine is an anecdote with a table.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readdir, readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'node:path';
+
+const reportsDir = resolve(dirname(fileURLToPath(import.meta.url)), '..', 'reports');
+
+// The npm package ships without `reports/` — it carries the harness, not the
+// published record. This test is about this *repository's* folder discipline,
+// so on a package install there is nothing for it to check and it says so
+// rather than failing.
+const reportsStat = await readdir(reportsDir).then(
+  () => true,
+  () => false
+);
+const suite = reportsStat ? test : test.skip;
+
+/**
+ * A run file names its four coordinates and its shape. Write-ups (`README`,
+ * `compatibility-matrix`, `spacing-2026-09-01`, …) are prose about runs and carry
+ * no machine record, so they are matched out by the shape suffix rather than by a
+ * list of exceptions that someone would have to remember to extend.
+ */
+const RUN_FILE = /^[a-z0-9-]+-\d+\.\d+\.\d+-[a-z0-9.-]+-(s\d+r\d+|r\d+)(-[a-z0-9-]+)?\.md$/;
+
+const listReports = async () => (await readdir(reportsDir)).sort();
+
+suite('every published run write-up has its machine record beside it', async () => {
+  const files = await listReports();
+  const runs = files.filter((name) => RUN_FILE.test(name));
+
+  // If this ever reads zero, the pattern has drifted away from the naming
+  // convention and the test would pass by measuring nothing.
+  assert.ok(runs.length >= 13, `expected at least 13 run write-ups, found ${runs.length}`);
+
+  const missing = runs.filter((name) => !files.includes(name.replace(/\.md$/, '.json')));
+  assert.deepEqual(missing, [], `run write-ups with no .json machine record: ${missing.join(', ')}`);
+});
+
+suite('every machine record is a report of the schema it claims', async () => {
+  const files = await listReports();
+  const records = files.filter(
+    (name) => name.endsWith('.json') && RUN_FILE.test(name.replace(/\.json$/, '.md'))
+  );
+  assert.ok(records.length >= 13, `expected at least 13 machine records, found ${records.length}`);
+
+  for (const name of records) {
+    const record = JSON.parse(await readFile(`${reportsDir}/${name}`, 'utf8'));
+    assert.match(record.schema ?? '', /^webmcp-gauge\/report\/\d+$/, `${name} carries no report schema`);
+    assert.ok(record.stamps?.judge?.model, `${name} does not stamp its judge model`);
+    assert.ok(record.stamps?.utteranceSet?.version, `${name} does not stamp its utterance set version`);
+    // The coverage block is what stops an incomplete run reading as a rate; it
+    // arrived in schema 3, so older records are allowed to lack it.
+    if (record.schema === 'webmcp-gauge/report/3') {
+      assert.equal(typeof record.coverage?.expectedTrials, 'number', `${name} has no coverage block`);
+    }
+  }
+});

package/report/scorecard.mjs

@@ -0,0 +1,157 @@
+/**
+ * A private scorecard for one captured project.
+ *
+ * Item 13 in the project log. The cohort snapshot already holds everything this
+ * needs — the page's manifest verbatim, the agent-visible view, and the liveness
+ * of the URL — and `core/lint.mjs` already knows what is wrong with a manifest.
+ * What did not exist was the rendering, so this is that and nothing more: it
+ * writes Markdown, it sends nothing, and the decision to hand one to anybody is
+ * still a gate in the log rather than a function call here.
+ *
+ * Publication rules do not apply to a scorecard, and the distinction matters:
+ * `toPublishable` withholds a builder's descriptions because publishing them
+ * would republish their work to strangers. A scorecard goes **to that builder**,
+ * about **their own page**, so quoting their own text back is the entire point —
+ * a finding a reader cannot locate is not actionable.
+ *
+ * The tone rule encoded below: every finding names what a client or a model does
+ * differently because of it. "Description too short" is a fact about a string;
+ * "a model choosing between this and `filter_rows` has 14 characters to go on" is
+ * a reason to act. Findings the harness cannot justify that way are not shipped.
+ */
+import { lintManifest } from '../core/lint.mjs';
+
+const severityLabel = { error: 'Error', warning: 'Warning' };
+
+/** Rules whose fix is not obvious from the finding alone get a sentence here. */
+const REMEDY = Object.freeze({
+  'name/invalid-characters': 'Rename using letters, digits, underscores or hyphens only.',
+  'name/too-long': 'Shorten it; a name is an identifier, not a sentence.',
+  'name/duplicate': 'Two tools cannot share a name — the second registration is the one an agent will not see.',
+  'description/missing': 'Write one sentence saying what the tool does and when to use it.',
+  'description/too-short': 'Say what it does *and* when to prefer it over its neighbours.',
+  'description/duplicate': 'Two tools described alike are two tools an agent cannot choose between; make the difference explicit in both.',
+  'description/near-duplicate': 'Make the distinguishing condition the first thing each description says.',
+  'schema/missing': 'Declare an `inputSchema`, even an empty object — a client cannot validate what is not described.',
+  'schema/no-properties': 'If the tool takes arguments, name them; if it takes none, say so with an empty `properties`.',
+  'schema/untyped-property': 'Give each property a `type`; an untyped property is a guess at call time.',
+  'schema/undocumented-property': 'Describe each property — the description is what a model fills it from.',
+  'schema/required-not-declared': 'List required properties in `required`, or a client will send a call that cannot succeed.',
+  'budget/too-many-tools': 'Consider grouping; a long manifest costs context on every turn.',
+});
+
+/**
+ * Everything the scorecard says, as data. Rendered separately so the numbers can
+ * be checked without parsing prose.
+ */
+export const buildScorecard = (record, { lintOptions = {} } = {}) => {
+  const manifest = {
+    present: record?.webmcp?.apiPresent ?? null,
+    settled: record?.webmcp?.settled ?? null,
+    tools: record?.webmcp?.tools ?? [],
+  };
+  const lint = lintManifest({ manifest, options: lintOptions });
+
+  const byTool = new Map();
+  for (const finding of lint.findings) {
+    if (!byTool.has(finding.tool)) byTool.set(finding.tool, []);
+    byTool.get(finding.tool).push(finding);
+  }
+
+  // Errors first, then the tools carrying the most findings: a builder with ten
+  // minutes should spend them where the manifest is worst.
+  const priorities = [...byTool.entries()]
+    .map(([tool, findings]) => ({
+      tool,
+      findings,
+      errors: findings.filter((f) => f.severity === 'error').length,
+    }))
+    .sort((a, b) => b.errors - a.errors || b.findings.length - a.findings.length || a.tool.localeCompare(b.tool));
+
+  const agentCount = record?.webmcp?.agentToolCount ?? null;
+  const pageCount = record?.webmcp?.toolCount ?? 0;
+
+  return {
+    schema: 'webmcp-gauge/scorecard/1',
+    generatedAt: new Date().toISOString(),
+    project: record?.project ?? null,
+    url: record?.url ?? null,
+    repo: record?.repo ?? null,
+    capturedAt: record?.capturedAt ?? null,
+    liveness: record?.liveness ?? null,
+    tools: {
+      registeredByThePage: pageCount,
+      visibleToAnAgent: agentCount,
+      // Stated only when the two disagree, because when they agree it is noise —
+      // and when they disagree it is the most surprising line on the page.
+      divergence: record?.webmcp?.divergence?.onlyInBrowser?.length
+        ? {
+            onlyInBrowser: record.webmcp.divergence.onlyInBrowser,
+            thirdParty: record.webmcp.thirdPartyToolCount ?? null,
+          }
+        : null,
+    },
+    counts: lint.counts,
+    priorities,
+    lint,
+  };
+};
+
+export const scorecardToMarkdown = (card) => {
+  const lines = [];
+  const { counts, tools } = card;
+
+  lines.push(`# WebMCP scorecard — ${card.project}`);
+  lines.push('');
+  lines.push(`**Page:** ${card.url}`);
+  if (card.repo) lines.push(`**Repo:** ${card.repo}`);
+  lines.push(`**Captured:** ${card.capturedAt}`);
+  lines.push(
+    `**Result:** ${counts.error} error${counts.error === 1 ? '' : 's'}, ${counts.warning} warning${counts.warning === 1 ? '' : 's'} across ${tools.registeredByThePage} registered tool${tools.registeredByThePage === 1 ? '' : 's'}`
+  );
+  lines.push('');
+  lines.push(
+    'This is a static reading of your tool manifest, taken from your live page. It is **not** an invocation rate — nothing here says how often an agent chooses your tools, only what a client and a model have to work with when they try.'
+  );
+  lines.push('');
+
+  if (tools.divergence) {
+    lines.push('## An agent sees tools your page cannot list');
+    lines.push('');
+    lines.push(
+      `Your page's \`getTools()\` returns **${tools.registeredByThePage}**, but the browser offers an agent **${tools.visibleToAnAgent}**: ${tools.divergence.onlyInBrowser.map((n) => `\`${n}\``).join(', ')}. On Chrome 152 that happens when a cross-origin embed is granted \`allow="tools"\` — its registrations reach the agent while appearing in no page's manifest. Worth knowing if you did not intend it, because you cannot enumerate it from script.`
+    );
+    lines.push('');
+  }
+
+  if (card.priorities.length === 0) {
+    lines.push('## Nothing to fix');
+    lines.push('');
+    lines.push('Every rule passed. The manifest is as legible as this linter knows how to check for.');
+    lines.push('');
+  } else {
+    lines.push('## What to fix, worst first');
+    lines.push('');
+    for (const { tool, findings } of card.priorities) {
+      lines.push(`### \`${tool}\``);
+      lines.push('');
+      for (const finding of findings) {
+        const remedy = REMEDY[finding.rule];
+        lines.push(`- **${severityLabel[finding.severity] ?? finding.severity}** · \`${finding.rule}\` — ${finding.detail}`);
+        if (remedy) lines.push(`  - ${remedy}`);
+      }
+      lines.push('');
+    }
+  }
+
+  lines.push('## How this was measured');
+  lines.push('');
+  lines.push(
+    `Your page was loaded once in Chrome with WebMCP enabled, its manifest read after settling, and ${card.lint.families.length > 0 ? `the ${card.lint.families.join(', ')} rule families applied` : 'every rule applied'}. Thresholds: ${Object.entries(card.lint.thresholds).map(([k, v]) => `${k}=${v}`).join(', ')}. Nothing was submitted to your page and no tool was invoked.`
+  );
+  lines.push('');
+  lines.push('Findings are advisory. If a rule is wrong about your page, it is the rule that needs fixing.');
+  lines.push('');
+
+  return `${lines.join('\n')}`;
+};

package/report/scorecard.test.mjs

@@ -0,0 +1,130 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { buildScorecard, scorecardToMarkdown } from './scorecard.mjs';
+
+/** The shape a cohort capture produces, trimmed to what a scorecard reads. */
+const record = ({ tools, agentTools = null, divergence = null, thirdParty = null }) => ({
+  project: 'Somebody Else\u2019s Project',
+  url: 'https://project.example/',
+  repo: 'https://github.com/them/project',
+  capturedAt: '2026-09-04T09:41:00.000Z',
+  liveness: { status: 200, reachable: true, redirected: false, finalUrl: 'https://project.example/' },
+  webmcp: {
+    apiPresent: true,
+    settled: true,
+    registered: tools.length > 0,
+    toolCount: tools.length,
+    tools,
+    agentTools,
+    agentToolCount: agentTools?.length ?? null,
+    divergence,
+    thirdPartyToolCount: thirdParty,
+  },
+});
+
+const goodTool = (name) => ({
+  name,
+  description: `Return the ${name.replace(/_/g, ' ')} for the requested period, so a caller can answer a question about it without reading the table.`,
+  inputSchema: { type: 'object', properties: { period: { type: 'string', description: 'Month or year to report on.' } } },
+  annotations: { readOnlyHint: true },
+});
+
+test('a clean manifest scores zero and says so without inventing advice', () => {
+  const card = buildScorecard(record({ tools: [goodTool('monthly_total')] }));
+
+  assert.equal(card.counts.error, 0);
+  assert.equal(card.counts.warning, 0);
+  assert.deepEqual(card.priorities, []);
+
+  const markdown = scorecardToMarkdown(card);
+  assert.match(markdown, /Nothing to fix/);
+  assert.ok(!/What to fix/.test(markdown));
+});
+
+test('a finding carries the rule id, the tool, and what to do about it', () => {
+  const card = buildScorecard(
+    record({ tools: [{ name: 'bad name', description: 'x', inputSchema: null }] })
+  );
+
+  assert.ok(card.counts.error > 0, 'an invalid name is an error');
+  const markdown = scorecardToMarkdown(card);
+  assert.match(markdown, /`name\/invalid-characters`/);
+  assert.match(markdown, /Rename using letters, digits, underscores or hyphens only\./);
+  assert.match(markdown, /`bad name`/, 'the offending tool is named so the finding can be located');
+});
+
+/**
+ * A scorecard goes to the page's own author, so quoting their descriptions back is
+ * the point — the opposite of `toPublishable`, which withholds them from strangers.
+ * This test exists to stop somebody "fixing" one to match the other.
+ */
+test('a scorecard may quote the builder\u2019s own text, unlike a published row', () => {
+  const description = 'Totals spending per category and highlights one on request.';
+  const card = buildScorecard(
+    record({ tools: [{ name: 'sum_by_category', description, inputSchema: { type: 'object', properties: {} } }] })
+  );
+
+  const markdown = scorecardToMarkdown(card);
+  assert.match(markdown, /sum_by_category/);
+  assert.ok(
+    card.lint.manifest.names.includes('sum_by_category'),
+    'the lint result keeps the real manifest, not a redacted one'
+  );
+  assert.ok(description.length > 0);
+});
+
+test('errors are prioritised above warnings, and busier tools above quieter ones', () => {
+  const card = buildScorecard(
+    record({
+      tools: [
+        { name: 'ok_tool', description: 'x', inputSchema: null },
+        { name: 'worse tool', description: '', inputSchema: null },
+      ],
+    })
+  );
+
+  assert.equal(card.priorities[0].tool, 'worse tool', 'the tool with an error comes first');
+  assert.ok(card.priorities[0].errors >= 1);
+});
+
+test('the divergence section appears only when the two views disagree', () => {
+  const quiet = scorecardToMarkdown(
+    buildScorecard(record({ tools: [goodTool('a')], agentTools: ['a'], divergence: { onlyInBrowser: [], onlyInPage: [] } }))
+  );
+  assert.ok(!/An agent sees tools/.test(quiet), 'agreement is not worth a section');
+
+  const loud = scorecardToMarkdown(
+    buildScorecard(
+      record({
+        tools: [goodTool('a')],
+        agentTools: ['a', 'embedded_pay'],
+        divergence: { onlyInBrowser: ['embedded_pay'], onlyInPage: [] },
+        thirdParty: 1,
+      })
+    )
+  );
+  assert.match(loud, /An agent sees tools your page cannot list/);
+  assert.match(loud, /`embedded_pay`/);
+  assert.match(loud, /allow="tools"/);
+});
+
+test('a scorecard never claims to be an invocation rate', () => {
+  const markdown = scorecardToMarkdown(buildScorecard(record({ tools: [goodTool('a')] })));
+  assert.match(markdown, /not\*\* an invocation rate/);
+  assert.match(markdown, /no tool was invoked/);
+  assert.ok(!/%\s*invocation/.test(markdown));
+});
+
+test('the method section states the thresholds the findings depend on', () => {
+  const markdown = scorecardToMarkdown(buildScorecard(record({ tools: [goodTool('a')] })));
+  assert.match(markdown, /minDescriptionChars=/);
+  assert.match(markdown, /nearDuplicateThreshold=/);
+});
+
+test('a page that registered nothing renders without a findings section', () => {
+  const card = buildScorecard(record({ tools: [] }));
+  assert.equal(card.tools.registeredByThePage, 0);
+  const markdown = scorecardToMarkdown(card);
+  assert.match(markdown, /0 errors, 0 warnings across 0 registered tools/);
+});