toolmenu 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +312 -35
  2. package/dist/args.d.ts +14 -2
  3. package/dist/args.js +44 -1
  4. package/dist/auth-check.d.ts +27 -0
  5. package/dist/auth-check.js +177 -0
  6. package/dist/auth.d.ts +7 -0
  7. package/dist/auth.js +29 -1
  8. package/dist/auto.d.ts +8 -0
  9. package/dist/auto.js +9 -2
  10. package/dist/breakdown.js +14 -10
  11. package/dist/cli.js +171 -56
  12. package/dist/clients.d.ts +13 -2
  13. package/dist/clients.js +65 -11
  14. package/dist/config.js +6 -0
  15. package/dist/diff.d.ts +35 -1
  16. package/dist/diff.js +57 -12
  17. package/dist/failures.js +4 -3
  18. package/dist/help.d.ts +1 -1
  19. package/dist/help.js +45 -17
  20. package/dist/history.js +5 -0
  21. package/dist/html.d.ts +29 -0
  22. package/dist/html.js +360 -0
  23. package/dist/index.d.ts +3 -2
  24. package/dist/index.js +3 -2
  25. package/dist/init.d.ts +14 -0
  26. package/dist/init.js +20 -6
  27. package/dist/junit.d.ts +17 -0
  28. package/dist/junit.js +87 -0
  29. package/dist/mcp-env.d.ts +6 -0
  30. package/dist/mcp-env.js +6 -0
  31. package/dist/mcp.d.ts +10 -0
  32. package/dist/mcp.js +86 -15
  33. package/dist/menu.d.ts +6 -0
  34. package/dist/menu.js +12 -1
  35. package/dist/options.d.ts +13 -0
  36. package/dist/options.js +4 -0
  37. package/dist/report.d.ts +14 -0
  38. package/dist/report.js +57 -51
  39. package/dist/review.d.ts +48 -0
  40. package/dist/review.js +212 -0
  41. package/dist/rule-docs.json +49 -4
  42. package/dist/rules/clients.d.ts +11 -0
  43. package/dist/rules/clients.js +65 -2
  44. package/dist/rules/determinism.d.ts +3 -0
  45. package/dist/rules/determinism.js +44 -7
  46. package/dist/rules/ids.js +5 -1
  47. package/dist/rules/index.js +5 -2
  48. package/dist/rules/output.d.ts +21 -0
  49. package/dist/rules/output.js +49 -0
  50. package/dist/rules/search.js +1 -1
  51. package/dist/rules/spec.d.ts +19 -0
  52. package/dist/rules/spec.js +86 -1
  53. package/dist/sanitize.d.ts +42 -0
  54. package/dist/sanitize.js +142 -0
  55. package/dist/sarif.d.ts +2 -0
  56. package/dist/sarif.js +2 -2
  57. package/dist/scan.d.ts +9 -2
  58. package/dist/scan.js +46 -24
  59. package/dist/session.d.ts +17 -0
  60. package/dist/session.js +119 -13
  61. package/dist/snapshot.js +2 -1
  62. package/package.json +5 -3
@@ -0,0 +1,48 @@
1
+ import { type DiffOptions, type DiffResult } from './diff.js';
2
+ import { type RuleSettings } from './rules/index.js';
3
+ import { type Finding, type Menu, type Severity } from './types.js';
4
+ import { type Format } from './report.js';
5
+ /**
6
+ * - `unchanged`: the same menu, the same findings.
7
+ * - `safe`: nothing breaks callers and no finding at or above --fail-on is new.
8
+ * - `breaking-released`: breaking changes, and the release versions given allow them.
9
+ * - `breaking`: breaking changes, and no release that allows them (none given, too small, or not checkable).
10
+ * - `review`: nothing breaks, but the new menu has findings at or above --fail-on the old one didn't,
11
+ * or diff's own checks (token budget, version bump) fail.
12
+ */
13
+ export type Verdict = 'unchanged' | 'safe' | 'breaking-released' | 'breaking' | 'review';
14
+ export interface ReviewResult {
15
+ verdict: Verdict;
16
+ /** Whether the new baseline can be accepted as it is: the exit code is 0 when true. */
17
+ accept: boolean;
18
+ diff: DiffResult;
19
+ findings: {
20
+ /** In the new menu only. */
21
+ introduced: Finding[];
22
+ /** In the old menu only. */
23
+ resolved: Finding[];
24
+ /**
25
+ * In both, for the same rule, but saying something else: a summary that now lists
26
+ * more tools, one tool's finding that became a summary. `worse`: louder, or now
27
+ * naming a tool the old one didn't; counted with the new findings for the verdict.
28
+ */
29
+ changed: {
30
+ before: Finding;
31
+ after: Finding;
32
+ worse: boolean;
33
+ }[];
34
+ /** In both, unchanged. */
35
+ kept: number;
36
+ };
37
+ tools: {
38
+ before: number;
39
+ after: number;
40
+ };
41
+ }
42
+ export interface ReviewOptions extends DiffOptions, RuleSettings {
43
+ failOn?: Severity;
44
+ }
45
+ /** The menu rules on a menu file, as far as a file can tell: no second list, no probes. */
46
+ export declare function fileFindings(menu: Menu, settings?: RuleSettings): Finding[];
47
+ export declare function reviewMenus(before: Menu, after: Menu, options?: ReviewOptions): ReviewResult;
48
+ export declare function formatReview(r: ReviewResult, paths: [string, string], format: Format, failOn?: Severity): string;
package/dist/review.js ADDED
@@ -0,0 +1,212 @@
1
+ /*
2
+ * toolmenu review: should this new menu file replace the committed baseline?
3
+ *
4
+ * `diff` says what changed between two menus. Accepting a new baseline is a further
5
+ * question: did the menu get worse for agents (findings it didn't have), better
6
+ * (findings gone), or break callers without a release that allows it? `review`
7
+ * answers that from the two files alone, with one verdict and an exit code to match.
8
+ */
9
+ import { diffMenus } from './diff.js';
10
+ import { MENU_RULES, runRules } from './rules/index.js';
11
+ import { SEVERITY_RANK } from './types.js';
12
+ import { bumpVerdict, classLine, counts, findingLines, mdFindings, releasePair, signed } from './report.js';
13
+ import { mdCode, mdText, plain } from './sanitize.js';
14
+ // Rules that read the wire (the raw tools/list pages, the handshake), which a menu
15
+ // file doesn't hold: run on a file they'd say the same about both menus, or nothing true.
16
+ const WIRE_ONLY = new Set(['spec/schema', 'spec/discover']);
17
+ // diff findings that say what it couldn't compare, not what changed.
18
+ const NOT_COMPARED = new Set(['diff/lists-unread', 'diff/catalog-queries']);
19
+ /** The menu rules on a menu file, as far as a file can tell: no second list, no probes. */
20
+ export function fileFindings(menu, settings = {}) {
21
+ return runRules(MENU_RULES, { menu, pages: [], capabilities: { tools: {} }, usedAuth: false, protocolVersion: menu.server.protocolVersion, era: menu.server.era }, settings).filter((f) => !WIRE_ONLY.has(f.rule));
22
+ }
23
+ /** A finding's identity across two menus: its rule, and the tool or tools it's about. */
24
+ function key(f) {
25
+ const tools = f.tools;
26
+ return `${f.rule}\0${f.tool ?? (tools ? tools.join(',') : '')}`;
27
+ }
28
+ const said = (f) => JSON.stringify([f.severity, f.message, f.detail ?? []]);
29
+ export function reviewMenus(before, after, options = {}) {
30
+ const failOn = options.failOn ?? 'error';
31
+ const atLeast = (f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn];
32
+ const diff = diffMenus(before, after, options);
33
+ const settings = { rules: options.rules, ignore: options.ignore };
34
+ const old = fileFindings(before, settings);
35
+ const now = fileFindings(after, settings);
36
+ // Matched by rule and tool first; what's left of a rule on both sides is the same
37
+ // finding said differently (one tool's finding became a summary of two).
38
+ const oldBy = new Map();
39
+ for (const f of old)
40
+ oldBy.set(key(f), [...(oldBy.get(key(f)) ?? []), f]);
41
+ const unmatched = [];
42
+ const changed = [];
43
+ let kept = 0;
44
+ for (const f of now) {
45
+ const match = oldBy.get(key(f))?.shift();
46
+ if (!match)
47
+ unmatched.push(f);
48
+ else if (said(match) === said(f))
49
+ kept++;
50
+ else
51
+ changed.push({ before: match, after: f, worse: false });
52
+ }
53
+ const left = [...oldBy.values()].flat();
54
+ const introduced = [];
55
+ for (const f of unmatched) {
56
+ const i = left.findIndex((o) => o.rule === f.rule);
57
+ if (i === -1)
58
+ introduced.push(f);
59
+ else
60
+ changed.push({ before: left.splice(i, 1)[0], after: f, worse: false });
61
+ }
62
+ const resolved = left;
63
+ // A changed finding is worse when it's louder, or now names a tool the old one didn't.
64
+ const names = after.tools.map((t) => t.name);
65
+ const text = (f) => [f.message, ...(f.detail ?? []), f.tool ?? '', ...(f.tools ?? [])].join('\n');
66
+ const mentions = (f) => new Set(names.filter((n) => new RegExp(`(?<![\\w-])${n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`).test(text(f))));
67
+ const grew = (c) => {
68
+ if (SEVERITY_RANK[c.after.severity] > SEVERITY_RANK[c.before.severity])
69
+ return true;
70
+ const had = mentions(c.before);
71
+ return [...mentions(c.after)].some((n) => !had.has(n));
72
+ };
73
+ const breaking = diff.findings.filter((f) => f.class === 'breaking' && atLeast(f));
74
+ // The release allows the breaking changes when its bump was checked and is big enough.
75
+ const released = diff.actualBump !== undefined && diff.releaseAs === undefined && !diff.bumpNotChecked;
76
+ // diff's own checks that aren't changes to the menu: the token budget, the version bump.
77
+ const checks = diff.findings.filter((f) => f.class === undefined && atLeast(f));
78
+ for (const c of changed)
79
+ c.worse = grew(c);
80
+ const worse = [...introduced, ...changed.filter((c) => c.worse).map((c) => c.after)].filter(atLeast);
81
+ let verdict;
82
+ if (breaking.length)
83
+ verdict = released ? 'breaking-released' : 'breaking';
84
+ else if (worse.length || checks.length)
85
+ verdict = 'review';
86
+ // Notes that something wasn't compared (lists read on one side only) aren't changes.
87
+ else if (!diff.findings.some((f) => !NOT_COMPARED.has(f.rule)) && !introduced.length && !resolved.length && !changed.length && diff.tokens.delta === 0)
88
+ verdict = 'unchanged';
89
+ else
90
+ verdict = 'safe';
91
+ // A released breaking change still needs the rest to be clean.
92
+ const accept = verdict === 'unchanged' || verdict === 'safe' || (verdict === 'breaking-released' && !worse.length && !checks.length);
93
+ return { verdict, accept, diff, findings: { introduced, resolved, changed, kept }, tools: { before: before.tools.length, after: after.tools.length } };
94
+ }
95
+ const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`;
96
+ function verdictLine(r, failOn) {
97
+ const b = r.diff.classes.breaking.changes;
98
+ const worse = [...r.findings.introduced, ...r.findings.changed.filter((c) => c.worse).map((c) => c.after)].filter((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]).length;
99
+ switch (r.verdict) {
100
+ case 'unchanged':
101
+ return { mark: '✓', text: 'unchanged: the same menu and the same findings. Nothing to accept.' };
102
+ case 'safe':
103
+ return { mark: '✓', text: 'safe to accept: nothing breaks callers, and no new finding at or above ' + failOn + '.' };
104
+ case 'breaking-released':
105
+ return r.accept
106
+ ? { mark: '✓', text: `accept with the release: ${plural(b, 'breaking change')}, and the release's ${r.diff.actualBump} bump allows ${b === 1 ? 'it' : 'them'}.` }
107
+ : { mark: '✗', text: `${plural(b, 'breaking change')} the release allows, but ${worse ? plural(worse, 'new finding') : "diff's own checks"} at or above ${failOn} too.` };
108
+ case 'breaking':
109
+ return {
110
+ mark: '✗',
111
+ text: `don't accept as is: ${plural(b, 'breaking change')}, and ${r.diff.releaseAs ? `the release's ${r.diff.actualBump} bump is too small (release ${r.diff.releaseAs})` : r.diff.bumpNotChecked ? `the bump wasn't checked (${r.diff.bumpNotChecked})` : 'no release versions to check them against (--release old..new)'}.`,
112
+ };
113
+ case 'review':
114
+ return { mark: '!', text: `review before accepting: nothing breaks callers, but ${worse ? `${plural(worse, 'finding')} at or above ${failOn} ${worse === 1 ? 'is' : 'are'} new` : "diff's own checks fail"}.` };
115
+ }
116
+ }
117
+ function nextStep(r, paths) {
118
+ const [base, next] = paths.map(plain);
119
+ switch (r.verdict) {
120
+ case 'unchanged':
121
+ return 'Nothing to do.';
122
+ case 'safe':
123
+ case 'breaking-released':
124
+ return r.accept ? `Commit ${next} as ${base}.` : 'Fix the new findings, or turn their rules down in toolmenu.config.json if they are expected, then review again.';
125
+ case 'breaking':
126
+ return r.diff.releaseAs ? `Release it as ${r.diff.releaseAs}, or keep the old shape for one more release. Then review again with --release.` : 'Keep the old shape for one more release, or pass the release versions (--release old..new) for a major release.';
127
+ case 'review':
128
+ return 'Fix the new findings, or turn their rules down in toolmenu.config.json if they are expected, then review again.';
129
+ }
130
+ }
131
+ export function formatReview(r, paths, format, failOn = 'error') {
132
+ const f = r.findings;
133
+ if (format === 'json') {
134
+ const v = verdictLine(r, failOn);
135
+ return JSON.stringify({
136
+ command: 'review',
137
+ verdict: r.verdict,
138
+ accept: r.accept,
139
+ // The verdict and next step as the text report says them, for a PR comment or a log line.
140
+ summary: `${v.mark} ${v.text}`,
141
+ next: nextStep(r, paths),
142
+ baseline: paths[0],
143
+ candidate: paths[1],
144
+ tools: r.tools,
145
+ counts: { introduced: f.introduced.length, resolved: f.resolved.length, changed: f.changed.length, worse: f.changed.filter((c) => c.worse).length, kept: f.kept },
146
+ findings: f,
147
+ diff: { ...r.diff, counts: counts(r.diff.findings) },
148
+ }, null, 2);
149
+ }
150
+ const v = verdictLine(r, failOn);
151
+ const d = r.diff;
152
+ const breaking = d.findings.filter((x) => x.class === 'breaking');
153
+ const otherDiff = d.findings.filter((x) => x.class !== 'breaking');
154
+ const name = d.after.name ?? d.before.name ?? 'server';
155
+ const findingsLine = `${f.introduced.length} new · ${f.resolved.length} gone · ${f.changed.length} changed · ${f.kept} unchanged`;
156
+ const toolsLine = `${r.tools.before} → ${r.tools.after}`;
157
+ const tokens = `~${d.tokens.before.toLocaleString('en-US')} → ~${d.tokens.after.toLocaleString('en-US')} (${signed(d.tokens.delta)}, estimate)`;
158
+ if (format === 'markdown' || format === 'github') {
159
+ const lines = [
160
+ `### toolmenu review: ${mdCode(paths[1])} as the new ${mdCode(paths[0])}`,
161
+ '',
162
+ `**${v.mark} ${mdText(v.text)}**`,
163
+ '',
164
+ `| | |`,
165
+ `|---|---|`,
166
+ `| Server | ${mdCode(name, { cell: true })} ${releasePair(d, true)} |`,
167
+ `| Changes | ${classLine(d)} |`,
168
+ `| Version | ${bumpVerdict(d, true).line} |`,
169
+ `| Tokens | ${tokens} |`,
170
+ `| Tools | ${toolsLine} |`,
171
+ `| Findings | ${findingsLine} |`,
172
+ '',
173
+ ];
174
+ if (breaking.length)
175
+ lines.push(`#### Breaking (${breaking.length})`, '', ...mdFindings(breaking), '');
176
+ if (f.introduced.length)
177
+ lines.push(`#### New findings (${f.introduced.length})`, '', ...mdFindings(f.introduced), '');
178
+ if (f.changed.length)
179
+ lines.push(`#### Changed findings (${f.changed.length})`, '', ...mdFindings(f.changed.map((c) => c.after)), '');
180
+ if (f.resolved.length)
181
+ lines.push(`<details><summary>Findings gone (${f.resolved.length})</summary>`, '', ...mdFindings(f.resolved), '', '</details>', '');
182
+ if (otherDiff.length)
183
+ lines.push(`<details><summary>Other changes (${otherDiff.length})</summary>`, '', ...mdFindings(otherDiff), '', '</details>', '');
184
+ lines.push(`**→ Next:** ${mdText(nextStep(r, paths))}`);
185
+ return lines.join('\n');
186
+ }
187
+ const lines = [
188
+ `toolmenu review ${plain(paths[1])} as the new ${plain(paths[0])}`,
189
+ ` verdict ${v.mark} ${v.text}`,
190
+ ` server ${plain(name)} ${releasePair(d)}`,
191
+ ` changes ${classLine(d)}`,
192
+ ` version ${bumpVerdict(d).line}`,
193
+ ` tokens ${tokens}`,
194
+ ` tools ${toolsLine}`,
195
+ ` findings ${findingsLine}`,
196
+ '',
197
+ ];
198
+ const section = (title, list) => {
199
+ if (list.length)
200
+ lines.push(`${title}:`, ...findingLines(list).map((l) => (l ? ` ${l}` : l)));
201
+ };
202
+ section('Breaking changes', breaking);
203
+ section('New findings', f.introduced);
204
+ section('Changed findings, as they are now (worse: louder or about more tools)', f.changed.filter((c) => c.worse).map((c) => c.after));
205
+ section('Changed findings, as they are now', f.changed.filter((c) => !c.worse).map((c) => c.after));
206
+ if (f.resolved.length)
207
+ lines.push('Findings gone:', ...f.resolved.map((x) => ` ${x.rule}${x.tool ? ` ${plain(x.tool)}` : ''}`), '');
208
+ if (otherDiff.length)
209
+ lines.push(`Other changes: ${plural(otherDiff.length, 'finding')} from diff (toolmenu diff ${plain(paths[0])} ${plain(paths[1])} lists them).`, '');
210
+ lines.push(`→ Next: ${nextStep(r, paths)}`);
211
+ return lines.join('\n');
212
+ }
@@ -79,10 +79,20 @@
79
79
  "severity": "info",
80
80
  "text": "`snapshot --as name:credential` (repeatable) or `identities` in the config lists the menu once per identity: a header set (HTTP) or env vars (stdio) over `--header`/`--env`. The first identity's menu is saved and checked; each other one, from its own process or connection, is compared with it: tools it adds, lacks, or defines differently, and its tokens. The tool set **MAY** vary by authorization, so this is info. An identity that can't be listed is reported here, and the others are still compared"
81
81
  },
82
+ "menu/client-features": {
83
+ "command": "snapshot",
84
+ "severity": "info",
85
+ "text": "`--client` only. What the client leaves out of what the server offers, from its own docs: GitHub Copilot cloud agent and code review use MCP tools only, not prompts or resources (with `--lists`, the server's prompts and resources are counted); code review also uses only tools marked `readOnlyHint: true`, and the finding lists the tools it skips. The detail names the source and read date"
86
+ },
87
+ "menu/client-limit": {
88
+ "command": "snapshot",
89
+ "severity": "warn",
90
+ "text": "`--client` only. More tools than the client sends the model at once: VS Code (GitHub Copilot Chat) allows 128 per chat request (code.visualstudio.com, agents/run/tools). This server alone is over it, so the user has to turn tools off before it works. Limits that only forum posts report (Cursor's 40) aren't checked"
91
+ },
82
92
  "menu/connection-variance": {
83
93
  "command": "snapshot",
84
94
  "severity": "error",
85
- "text": "The same over HTTP: a second connection with the same credentials gets a different menu. Retried once before reporting"
95
+ "text": "The same over HTTP: a second connection with the same credentials gets a different menu. Retried once before reporting. A difference only in fields the model doesn't read (output schema, annotations, an extra field such as a per-tool `version` hash) is a warning: no prompt cache is lost, but no baseline matches (the `@pipeworx` servers, bench 2026-11-09)"
86
96
  },
87
97
  "menu/capability-dependent": {
88
98
  "command": "snapshot",
@@ -112,17 +122,17 @@
112
122
  "menu/nondeterministic": {
113
123
  "command": "snapshot",
114
124
  "severity": "error",
115
- "text": "Two identical `tools/list` calls return different menus. Reports what differed: order, description, schema, annotations, tool added, tool removed. Order and content changes are both cache-relevant. Spec basis for order: servers **SHOULD** return \"the same ordering across requests when the underlying set of tools has not changed\" (§12). Content changes are toolmenu's own rule (lesson 05), not a spec violation"
125
+ "text": "Two identical `tools/list` calls return different menus. Reports what differed: order, description, schema, annotations, tool added, tool removed. Order and content changes are both cache-relevant. Spec basis for order: servers **SHOULD** return \"the same ordering across requests when the underlying set of tools has not changed\" (§12). Content changes are toolmenu's own rule (lesson 05), not a spec violation. A difference only in fields the model doesn't read (output schema, annotations, an extra field such as a per-tool `version` hash) is a warning: no prompt cache is lost, but no baseline matches (the `@pipeworx` servers, bench 2026-11-09)"
116
126
  },
117
127
  "menu/process-variance": {
118
128
  "command": "snapshot",
119
129
  "severity": "error",
120
- "text": "A second server process, started the same way, serves a different menu, so every restart misses the cache. Seen on mcp-atlassian: a default built from a Python set, in a new order every start (FINDINGS F12). toolmenu runs the main process with `PYTHONHASHSEED=0`, so a Python server's saved menu is reproducible, and the second with another seed. `--processes 1` turns it off"
130
+ "text": "A second server process, started the same way, serves a different menu, so every restart misses the cache. Seen on mcp-atlassian: a default built from a Python set, in a new order every start (FINDINGS F12). toolmenu runs the main process with `PYTHONHASHSEED=0`, so a Python server's saved menu is reproducible, and the second with another seed. `--processes 1` turns it off. A difference only in fields the model doesn't read (output schema, annotations, an extra field such as a per-tool `version` hash) is a warning: no prompt cache is lost, but no baseline matches (the `@pipeworx` servers, bench 2026-11-09)"
121
131
  },
122
132
  "naming/client-name": {
123
133
  "command": "snapshot",
124
134
  "severity": "warn",
125
- "text": "`--client` only. Tool names outside the client's API name format: the Claude API's `^[a-zA-Z0-9_-]{1,128}$` (platform.claude.com define-tools) for `claude-code` and `claude-api-search`, OpenAI's `FunctionToolParam.name` (`^[a-zA-Z0-9_-]+$`, maxLength 128, openai/openai-openapi) for `openai`. The detail names the source and read date"
135
+ "text": "`--client` only. Tool names outside the client's API name format: the Claude API's `^[a-zA-Z0-9_-]{1,128}$` (platform.claude.com define-tools) for `claude-code` and `claude-api-search`, OpenAI's `FunctionToolParam.name` (`^[a-zA-Z0-9_-]+$`, maxLength 128, openai/openai-openapi) for `openai` and `codex`, the Gemini API's `FunctionDeclaration.name` (letters, digits, `_`, `:`, `.`, `-`, at most 128) for `gemini-api`. For `gemini-cli` the client rewrites instead of refusing (other characters become `_`, names over 63 characters are cut in the middle), so the finding says the model sees a name that isn't the server's. The detail names the source and read date"
126
136
  },
127
137
  "naming/route": {
128
138
  "command": "snapshot",
@@ -189,6 +199,11 @@
189
199
  "severity": "warn",
190
200
  "text": "The handshake's `serverInfo` has no `name` or no `version`, so a client (and `diff --server-version-is-release`) can't tell which server or release answered"
191
201
  },
202
+ "spec/tool-name": {
203
+ "command": "snapshot",
204
+ "severity": "warn, info, error",
205
+ "text": "A tool name outside the format the spec recommends from 2025-11-25 (SEP-986: 1 to 128 of `A-Z a-z 0-9 _ - .`); info on older protocols, which gave no format. An error on any protocol when a name has characters that act on a terminal or can't be seen (controls, format and bidi characters, default-ignorables such as variation selectors and Hangul fillers): what's read isn't the name the server takes. Other non-ASCII characters are named by code point, so a lookalike can't hide. Reports show hidden characters as escapes (the same class)"
206
+ },
192
207
  "spec/task-required": {
193
208
  "command": "snapshot",
194
209
  "severity": "info",
@@ -534,6 +549,11 @@
534
549
  "severity": "error",
535
550
  "text": "The same tools mid-session in a different order: a cache miss for everything after the first moved tool"
536
551
  },
552
+ "session/output-schema": {
553
+ "command": "session",
554
+ "severity": "error",
555
+ "text": "A tool that declares an `outputSchema` answered (not `isError`) with no `structuredContent`, or with one that doesn't match the schema. From 2025-06-18 servers **MUST** return conforming structured results; the official SDK client validates them and raises an error where the tool answered. Checked once per tool, on the calls a session makes (more with `--call-everything`). A warning when only `format` checks failed (a URI, a date): many validators skip format, so a client may let the result through"
556
+ },
537
557
  "session/result-type": {
538
558
  "command": "session",
539
559
  "severity": "error",
@@ -593,5 +613,30 @@
593
613
  "command": "session",
594
614
  "severity": "warn, error if no call got through",
595
615
  "text": "Calls that failed before reaching the tool: authentication, something missing on this machine (no Chrome), the network; or an answer, not marked `isError`, that is the API's 401 or 403 passed on as JSON (`{\"status\":401,\"code\":\"unauthorized\"}`, Notion's server). One finding for the run, not one per step, so an expired CI secret doesn't pass a run that tested nothing"
616
+ },
617
+ "auth/discovery": {
618
+ "command": "auth",
619
+ "severity": "error",
620
+ "text": "The server answered 401 and says it uses OAuth (a `WWW-Authenticate` header), but serves no OAuth protected resource metadata: not where the header's `resource_metadata` points, nor at a well-known URI. Servers that use OAuth **MUST** implement RFC 9728. Info when there's neither a header nor metadata (authorization is OPTIONAL in MCP, and the server may take an API key), when only the header is missing or lacks `resource_metadata`, or when the metadata couldn't be read"
621
+ },
622
+ "auth/resource-metadata": {
623
+ "command": "auth",
624
+ "severity": "error",
625
+ "text": "The protected resource metadata lists no `authorization_servers` (it **MUST** list one), or has no `resource`, or a `resource` on another origin or path: RFC 9728 §3.3 says it **MUST** be identical to the URL the client called (metadata named by `WWW-Authenticate`) or the identifier the well-known URI was built from, or the client **MUST NOT** use it. A warning when it's a same-origin parent of that URL, or only a trailing slash differs: the official MCP SDK client accepts those, a client that follows the RFC to the letter doesn't"
626
+ },
627
+ "auth/server-metadata": {
628
+ "command": "auth",
629
+ "severity": "error",
630
+ "text": "No authorization server metadata at the URIs the spec has clients try (RFC 8414, then OpenID Connect Discovery, path-aware), so a client can't find where to send the user"
631
+ },
632
+ "auth/pkce": {
633
+ "command": "auth",
634
+ "severity": "error",
635
+ "text": "The authorization server's metadata doesn't list `S256` in `code_challenge_methods_supported`. Clients **MUST** verify PKCE support from it and refuse to go on without it"
636
+ },
637
+ "auth/registration": {
638
+ "command": "auth",
639
+ "severity": "info",
640
+ "text": "No registration endpoint and no Client ID Metadata Documents: every client needs an app registered ahead of time"
596
641
  }
597
642
  }
@@ -5,3 +5,14 @@ import { type Rule } from './rule.js';
5
5
  * ^[a-zA-Z0-9_-]{1,128}$, OpenAI's function names ^[a-zA-Z0-9_-]+$ up to 128.
6
6
  */
7
7
  export declare const clientName: Rule;
8
+ /**
9
+ * --client: more tools than the client sends the model at once (VS Code: "A chat request
10
+ * can have a maximum of 128 tools enabled at a time"). This server alone is over it, so
11
+ * a user has to turn tools off before the client works with it.
12
+ */
13
+ export declare const clientLimit: Rule;
14
+ /**
15
+ * --client: what the client leaves out of what the server offers (GitHub Copilot cloud agent
16
+ * and code review use MCP tools only; code review only tools marked readOnlyHint: true).
17
+ */
18
+ export declare const clientFeatures: Rule;
@@ -23,10 +23,73 @@ export const clientName = {
23
23
  return [
24
24
  {
25
25
  ...(bad.length === 1 ? { tool: bad[0].tool } : {}),
26
- message: `${count(bad.length, 'tool name')} ${bad.length === 1 ? "doesn't" : "don't"} fit the tool name format of ${rule.value.api}${ctx.client === 'claude-code' ? `, which ${profile.label} sends tools to` : ''} (${rule.value.pattern}, at most ${rule.value.max} characters). MCP allows them, but a request that passes them to the API as they are is refused; a client that rewrites them shows the model a name that isn't the server's.`,
26
+ message: rule.value.rewrites
27
+ ? `${count(bad.length, 'tool name')} ${bad.length === 1 ? "doesn't" : "don't"} fit the tool names ${rule.value.api} keeps as they are (${rule.value.pattern}, at most ${rule.value.max} characters). It ${rule.value.rewrites}, so its model sees a name that isn't the server's, and two names can end up the same.`
28
+ : `${count(bad.length, 'tool name')} ${bad.length === 1 ? "doesn't" : "don't"} fit the tool name format of ${rule.value.api}${ctx.client === 'claude-code' ? `, which ${profile.label} sends tools to` : ''} (${rule.value.pattern}, at most ${rule.value.max} characters). MCP allows them, but a request that passes them to the API as they are is refused; a client that rewrites them shows the model a name that isn't the server's.`,
27
29
  detail: [...bad.slice(0, 12).map((b) => `${b.tool}: ${b.why}`), ...(bad.length > 12 ? [`and ${bad.length - 12} more`] : []), `Source: ${rule.source} (read ${rule.read}).`],
28
- fix: 'Use letters, digits, _ and - in tool names (files_read, not files.read), at most 128 characters.',
30
+ fix: `Use letters, digits, _ and - in tool names (files_read, not files.read), at most ${Math.min(rule.value.max, 128)} characters.`,
29
31
  },
30
32
  ];
31
33
  },
32
34
  };
35
+ /**
36
+ * --client: more tools than the client sends the model at once (VS Code: "A chat request
37
+ * can have a maximum of 128 tools enabled at a time"). This server alone is over it, so
38
+ * a user has to turn tools off before the client works with it.
39
+ */
40
+ export const clientLimit = {
41
+ id: 'menu/client-limit',
42
+ scope: 'agent',
43
+ severity: 'warn',
44
+ summary: 'More tools than the client sends the model at once',
45
+ run(ctx) {
46
+ const profile = profileOf(ctx.client);
47
+ const limit = profile?.toolLimit;
48
+ const n = ctx.menu.tools.length;
49
+ if (!profile || !limit || n <= limit.value)
50
+ return [];
51
+ return [
52
+ {
53
+ message: `${n} tools, over the ${limit.value} ${profile.label} allows in one request, and that's this server alone, before any other server the user has. The user has to turn ${n - limit.value} or more off before it works, and picks which without knowing which ones the agent needs.`,
54
+ detail: [`Source: ${limit.source} (read ${limit.read}).`],
55
+ fix: 'Group the tools into toolsets a user turns on (the GitHub server\'s --toolsets), or fold several into one tool with an action parameter.',
56
+ },
57
+ ];
58
+ },
59
+ };
60
+ /**
61
+ * --client: what the client leaves out of what the server offers (GitHub Copilot cloud agent
62
+ * and code review use MCP tools only; code review only tools marked readOnlyHint: true).
63
+ */
64
+ export const clientFeatures = {
65
+ id: 'menu/client-features',
66
+ scope: 'agent',
67
+ severity: 'info',
68
+ summary: "Parts of the server the client doesn't use",
69
+ run(ctx) {
70
+ const profile = profileOf(ctx.client);
71
+ if (!profile)
72
+ return [];
73
+ const out = [];
74
+ const prompts = ctx.menu.prompts?.length ?? 0, resources = (ctx.menu.resources?.length ?? 0) + (ctx.menu.resourceTemplates?.length ?? 0);
75
+ if (profile.toolsOnly && prompts + resources) {
76
+ out.push({
77
+ message: `${profile.label} uses only MCP tools, not ${profile.toolsOnly.value}, so the server's ${[prompts && count(prompts, 'prompt'), resources && count(resources, 'resource')].filter(Boolean).join(' and ')} aren't available there. Anything an agent needs from them has to come through a tool for it to work in ${profile.label}.`,
78
+ detail: [`Source: ${profile.toolsOnly.source} (read ${profile.toolsOnly.read}).`],
79
+ fix: 'If an agent needs the content, offer it through a tool as well.',
80
+ });
81
+ }
82
+ if (profile.readOnlyOnly) {
83
+ const tools = kept(ctx.menu.tools, ctx);
84
+ const off = tools.filter((t) => t.annotations?.readOnlyHint !== true);
85
+ if (off.length) {
86
+ out.push({
87
+ message: `${profile.label} uses only tools marked readOnlyHint: true, and ${off.length === tools.length ? `none of the ${count(tools.length, 'tool')} is` : `${count(off.length, 'tool')} of ${tools.length} ${off.length === 1 ? "isn't" : "aren't"}`}, so ${off.length === 1 ? "it isn't" : "they aren't"} available there.`,
88
+ detail: [...off.slice(0, 12).map((t) => t.name), ...(off.length > 12 ? [`and ${off.length - 12} more`] : []), `Source: ${profile.readOnlyOnly.source} (read ${profile.readOnlyOnly.read}).`],
89
+ fix: 'Mark the tools that only read with readOnlyHint: true (and only those: clients act on it).',
90
+ });
91
+ }
92
+ }
93
+ return out;
94
+ },
95
+ };
@@ -1,5 +1,8 @@
1
+ import { type ToolChange } from '../compare.js';
1
2
  import type { MenuTool, Severity } from '../types.js';
2
3
  import { type Rule, type RuleFinding } from './rule.js';
4
+ /** The fields that differ, when none of them is one the model reads; undefined otherwise. */
5
+ export declare function outsideModel(before: MenuTool[], after: MenuTool[], changes: ToolChange[]): string[] | undefined;
3
6
  export declare const nondeterministic: Rule;
4
7
  /** What a second process or connection served, for the variance rules. */
5
8
  export interface Probe {
@@ -45,6 +45,35 @@ function places(before, after) {
45
45
  return [...changes.filter((c) => !EDITS.has(c.kind)).map((c) => `${c.kind}:${c.tool}`), ...everySpot(before, after, changes).map((s) => s.where)];
46
46
  }
47
47
  const isTime = (s) => TIMESTAMP.test(String(s.before)) && TIMESTAMP.test(String(s.after));
48
+ /**
49
+ * Changes the model reads: the tool set, its order, names, descriptions and input
50
+ * schemas (and their bytes: key order). Output schemas, annotations and other fields
51
+ * (a per-tool "version" hash: the @pipeworx servers) stay with the client.
52
+ */
53
+ const MODEL_KINDS = new Set(['added', 'removed', 'moved', 'description', 'inputSchema', 'serialization']);
54
+ /** The fields that differ, when none of them is one the model reads; undefined otherwise. */
55
+ export function outsideModel(before, after, changes) {
56
+ if (!changes.length || changes.some((c) => MODEL_KINDS.has(c.kind)))
57
+ return undefined;
58
+ const fields = new Set();
59
+ for (const s of everySpot(before, after, changes)) {
60
+ const rest = s.where.slice(s.tool.length + 1);
61
+ if (rest)
62
+ fields.add(rest.split(/[.[]/)[0]);
63
+ }
64
+ for (const c of changes)
65
+ if (c.kind === 'outputSchema' || c.kind === 'annotations')
66
+ fields.add(c.kind);
67
+ return [...fields].sort();
68
+ }
69
+ /** "version", "version and annotations". */
70
+ function fieldList(fields) {
71
+ return fields.length <= 1 ? fields.join('') : `${fields.slice(0, -1).join(', ')} and ${fields.at(-1)}`;
72
+ }
73
+ /** Why a variance only in fields the model doesn't read still matters. */
74
+ function outsideModelCost(fields) {
75
+ return `It differs only in ${fieldList(fields)}, which clients keep to themselves: the model doesn't read ${fields.length === 1 ? 'it' : 'them'}, so prompt caches aren't affected, but no baseline matches, so every diff and review reports a change.`;
76
+ }
48
77
  /** "list_transactions.end_date.default" or "a.b, c.d and 3 more places". */
49
78
  function placesOf(found) {
50
79
  const shown = found.slice(0, 2).map((s) => s.where).join(', ');
@@ -96,12 +125,16 @@ export const nondeterministic = {
96
125
  const onlyOrder = changes.every((c) => c.kind === 'moved');
97
126
  const found = spots(ctx.menu.tools, ctx.secondList, changes);
98
127
  const tools = [...new Set(changes.map((c) => c.tool))];
128
+ const outside = outsideModel(ctx.menu.tools, ctx.secondList, changes);
99
129
  return [
100
130
  {
101
131
  ...(tools.length === 1 ? { tool: tools[0] } : {}),
102
- message: onlyOrder
103
- ? 'Two identical tools/list calls, back to back, returned the tools in a different order, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. The spec says servers SHOULD keep the order stable.'
104
- : `Two identical tools/list calls, back to back, returned different menus, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. ${whatDiffers(changes, found)}`.trim(),
132
+ ...(outside ? { severity: 'warn' } : {}),
133
+ message: outside
134
+ ? `Two identical tools/list calls, back to back, returned different menus. ${outsideModelCost(outside)} ${whatDiffers(changes, found)}`.trim()
135
+ : onlyOrder
136
+ ? 'Two identical tools/list calls, back to back, returned the tools in a different order, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. The spec says servers SHOULD keep the order stable.'
137
+ : `Two identical tools/list calls, back to back, returned different menus, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. ${whatDiffers(changes, found)}`.trim(),
105
138
  detail: detailFor(ctx.menu.tools, ctx.secondList, changes),
106
139
  fix: fixFor(changes, found, 'call'),
107
140
  },
@@ -146,10 +179,13 @@ export function varianceFinding(main, other, opts) {
146
179
  detail = lines.slice(0, 10).concat(lines.length > 10 ? [`…and ${lines.length - 10} more`] : []);
147
180
  }
148
181
  const reordered = found.some((s) => s.reordered);
182
+ const outside = outsideModel(main, other, why);
149
183
  const message = [
150
- stdio
151
- ? 'A second server process, started the same way, served a different menu, so every restart, and every client that starts its own copy, gets a tool list that no cached prompt matches.'
152
- : `A second connection with the same credentials got a different menu, so clients can't share a cached tool list.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per connection.' : ''}`,
184
+ outside
185
+ ? `A second ${stdio ? 'server process, started the same way,' : 'connection with the same credentials'} served a different menu. ${outsideModelCost(outside)}`
186
+ : stdio
187
+ ? 'A second server process, started the same way, served a different menu, so every restart, and every client that starts its own copy, gets a tool list that no cached prompt matches.'
188
+ : `A second connection with the same credentials got a different menu, so clients can't share a cached tool list.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per connection.' : ''}`,
153
189
  lead,
154
190
  whatDiffers(why, found),
155
191
  reordered
@@ -162,7 +198,8 @@ export function varianceFinding(main, other, opts) {
162
198
  ].filter(Boolean).join(' ');
163
199
  return {
164
200
  rule: stdio ? 'menu/process-variance' : 'menu/connection-variance',
165
- severity: stdio || opts.modern ? 'error' : 'warn',
201
+ // Only in fields the model doesn't read: a warning, not an error (no cache is lost).
202
+ severity: outside ? (stdio || opts.modern ? 'warn' : 'info') : stdio || opts.modern ? 'error' : 'warn',
166
203
  message,
167
204
  detail,
168
205
  fix: fixFor(why, found, stdio ? 'process' : 'connection'),
package/dist/rules/ids.js CHANGED
@@ -71,6 +71,9 @@ export function idParams(tool) {
71
71
  return params;
72
72
  }
73
73
  function looksLikeIdValue(schema) {
74
+ // null, true, false: not a schema an ID's value is described by.
75
+ if (!schema || typeof schema !== 'object')
76
+ return false;
74
77
  if (schema.enum)
75
78
  return false; // a fixed set of values is chosen, not invented
76
79
  const type = schema.type;
@@ -88,7 +91,8 @@ function withOperations(tools) {
88
91
  const out = [...tools];
89
92
  for (const t of tools) {
90
93
  for (const schema of Object.values(t.inputSchema?.properties ?? {})) {
91
- for (const v of Array.isArray(schema.enum) ? schema.enum : []) {
94
+ // A property schema can be anything a server sends (null, true): only an object has an enum.
95
+ for (const v of schema && typeof schema === 'object' && Array.isArray(schema.enum) ? schema.enum : []) {
92
96
  if (typeof v !== 'string' || !/^[a-z]+(?:_[a-z]+)+$/.test(v))
93
97
  continue;
94
98
  // Lists and searches only: a router's get_workflow needs the ID it would hand out.
@@ -2,7 +2,7 @@ import { SEVERITY_RANK } from '../types.js';
2
2
  import { buried, cut, lateInstruction } from './description.js';
3
3
  import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
4
4
  import { authoredIds } from './ids.js';
5
- import { clientName } from './clients.js';
5
+ import { clientFeatures, clientLimit, clientName } from './clients.js';
6
6
  import { invalidCredential } from './auth.js';
7
7
  import { networkDependent } from './network.js';
8
8
  import { capabilityDependent } from './capabilities.js';
@@ -10,7 +10,7 @@ import { listSchema, listVariance } from './lists.js';
10
10
  import { handleLifetime } from './handles.js';
11
11
  import { authVariance, cacheScopePublic } from './identities.js';
12
12
  import { route, sharedWord, vagueId } from './naming.js';
13
- import { cacheHints, deprecated, discover, duplicateName, extensionId, schema, serverInfo, taskRequired, toolsCapability } from './spec.js';
13
+ import { cacheHints, deprecated, discover, duplicateName, extensionId, schema, serverInfo, taskRequired, toolName, toolsCapability } from './spec.js';
14
14
  import { noDryRun, readOnlyMismatch, unannotated } from './write.js';
15
15
  import { gated } from './gated.js';
16
16
  import { xMcpHeader } from './headers.js';
@@ -22,6 +22,7 @@ import { serverCard } from './card.js';
22
22
  export const MENU_RULES = [
23
23
  nondeterministic,
24
24
  duplicateName,
25
+ toolName,
25
26
  gated,
26
27
  processVariance,
27
28
  connectionVariance,
@@ -30,6 +31,8 @@ export const MENU_RULES = [
30
31
  sharedWord,
31
32
  route,
32
33
  clientName,
34
+ clientLimit,
35
+ clientFeatures,
33
36
  findable,
34
37
  authoredIds,
35
38
  handleLifetime,
@@ -0,0 +1,21 @@
1
+ import type { JsonSchema } from '../types.js';
2
+ /** The protocol version output schemas came in. */
3
+ export declare const OUTPUT_SCHEMA_SINCE = "2025-06-18";
4
+ /**
5
+ * What's wrong with a successful result against the tool's outputSchema, or
6
+ * undefined when it conforms (or the schema can't be compiled: then it's the
7
+ * schema that's broken, which spec/schema reports, not the result).
8
+ */
9
+ export declare function outputProblem(schema: JsonSchema, result: {
10
+ structuredContent?: unknown;
11
+ }): string | undefined;
12
+ /**
13
+ * The same, and whether only `format` checks failed (a URI, a date): many validators
14
+ * skip format unless told to, so such a result may well pass a client.
15
+ */
16
+ export declare function outputCheck(schema: JsonSchema, result: {
17
+ structuredContent?: unknown;
18
+ }): {
19
+ problem: string;
20
+ formatOnly: boolean;
21
+ } | undefined;