toolmenu 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +312 -35
  2. package/dist/args.d.ts +14 -2
  3. package/dist/args.js +44 -1
  4. package/dist/auth-check.d.ts +27 -0
  5. package/dist/auth-check.js +177 -0
  6. package/dist/auth.d.ts +7 -0
  7. package/dist/auth.js +29 -1
  8. package/dist/auto.d.ts +8 -0
  9. package/dist/auto.js +9 -2
  10. package/dist/breakdown.js +14 -10
  11. package/dist/cli.js +171 -56
  12. package/dist/clients.d.ts +13 -2
  13. package/dist/clients.js +65 -11
  14. package/dist/config.js +6 -0
  15. package/dist/diff.d.ts +35 -1
  16. package/dist/diff.js +57 -12
  17. package/dist/failures.js +4 -3
  18. package/dist/help.d.ts +1 -1
  19. package/dist/help.js +45 -17
  20. package/dist/history.js +5 -0
  21. package/dist/html.d.ts +29 -0
  22. package/dist/html.js +360 -0
  23. package/dist/index.d.ts +3 -2
  24. package/dist/index.js +3 -2
  25. package/dist/init.d.ts +14 -0
  26. package/dist/init.js +20 -6
  27. package/dist/junit.d.ts +17 -0
  28. package/dist/junit.js +87 -0
  29. package/dist/mcp-env.d.ts +6 -0
  30. package/dist/mcp-env.js +6 -0
  31. package/dist/mcp.d.ts +10 -0
  32. package/dist/mcp.js +86 -15
  33. package/dist/menu.d.ts +6 -0
  34. package/dist/menu.js +12 -1
  35. package/dist/options.d.ts +13 -0
  36. package/dist/options.js +4 -0
  37. package/dist/report.d.ts +14 -0
  38. package/dist/report.js +57 -51
  39. package/dist/review.d.ts +48 -0
  40. package/dist/review.js +212 -0
  41. package/dist/rule-docs.json +49 -4
  42. package/dist/rules/clients.d.ts +11 -0
  43. package/dist/rules/clients.js +65 -2
  44. package/dist/rules/determinism.d.ts +3 -0
  45. package/dist/rules/determinism.js +44 -7
  46. package/dist/rules/ids.js +5 -1
  47. package/dist/rules/index.js +5 -2
  48. package/dist/rules/output.d.ts +21 -0
  49. package/dist/rules/output.js +49 -0
  50. package/dist/rules/search.js +1 -1
  51. package/dist/rules/spec.d.ts +19 -0
  52. package/dist/rules/spec.js +86 -1
  53. package/dist/sanitize.d.ts +42 -0
  54. package/dist/sanitize.js +142 -0
  55. package/dist/sarif.d.ts +2 -0
  56. package/dist/sarif.js +2 -2
  57. package/dist/scan.d.ts +9 -2
  58. package/dist/scan.js +46 -24
  59. package/dist/session.d.ts +17 -0
  60. package/dist/session.js +119 -13
  61. package/dist/snapshot.js +2 -1
  62. package/package.json +5 -3
package/README.md CHANGED
@@ -5,24 +5,34 @@
5
5
  <h1 align="center">toolmenu</h1>
6
6
 
7
7
  <p align="center">
8
- <b>Checks for MCP servers' tool lists: changes that confuse agents, break prompt caches or break clients, caught before your users hit them.</b>
8
+ <b>Tests your MCP server the way agents and clients meet it: the menu they read, the calls they make, the results and sign-in they get back, and every release after. Problems caught before your users hit them.</b>
9
9
  </p>
10
10
 
11
11
  <p align="center">
12
12
 
13
13
  [![npm](https://img.shields.io/npm/v/toolmenu)](https://www.npmjs.com/package/toolmenu)
14
- [![ci](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml/badge.svg)](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
15
- [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
16
- [![MCP: 2026-07-28 and 2025](https://img.shields.io/badge/MCP-2026--07--28%20%C2%B7%202025-6E56CF)](https://modelcontextprotocol.io)
17
- [![toolmenu](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fniksa90%2Ftoolmenu%2Fmain%2Fdocs%2Ftoolmenu-badge.json)](https://github.com/niksa90/toolmenu)
18
- [![toolmenu MCP server on Glama](https://glama.ai/mcp/servers/niksa90/toolmenu/badges/score.svg)](https://glama.ai/mcp/servers/niksa90/toolmenu)
14
+ [![ci](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml/badge.svg)](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
15
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
16
+ [![MCP: 2026-07-28 and 2025](https://img.shields.io/badge/MCP-2026--07--28%20%C2%B7%202025-6E56CF)](https://modelcontextprotocol.io)
17
+ [![toolmenu](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fniksa90%2Ftoolmenu%2Fmain%2Fdocs%2Ftoolmenu-badge.json)](https://github.com/niksa90/toolmenu)
18
+ [![toolmenu MCP server on Glama](https://glama.ai/mcp/servers/niksa90/toolmenu/badges/score.svg)](https://glama.ai/mcp/servers/niksa90/toolmenu)
19
+ [![toolmenu MCP on mcprush](https://mcprush.com/badge/toolmenu-mcp/score.svg)](https://mcprush.com/to/toolmenu-mcp?ref=badge)
20
+ [![Listed on mcpservers.org](https://mcpservers.org/badge.svg)](https://mcpservers.org/servers/niksa90/toolmenu)
19
21
 
20
22
  </p>
21
23
 
22
24
  Your MCP server's tool list is the menu every agent reads before it does anything.
23
- toolmenu checks that menu the way clients and agents will meet it: across calls,
24
- connections, clients and releases. No LLM is involved, so the same inputs always give
25
- the same answer and it fits in CI.
25
+ toolmenu starts there and follows the agent in. It **connects** as a client does (stdio
26
+ or HTTP, every protocol version from 2024-11-05 to 2026-07-28), **reads** the menu,
27
+ **calls** the tools (read-only by default; every tool, writes included, in a sealed
28
+ sandbox with `--call-everything`), **checks** each result against the spec and the tool's
29
+ own `outputSchema`, **walks** the OAuth discovery a client follows before it logs in, and
30
+ **compares** the menu across steps, connections, processes, clients and releases.
31
+
32
+ It isn't only a linter: a linter grades the text of one snapshot, and much of what
33
+ breaks agents only shows when you call the server, reconnect, or ship the next version.
34
+ No LLM is involved, so the same server gives the same answer on every run and it fits
35
+ in CI. The same checks run [every week on ~3,200 public servers](#tested-on-real-servers).
26
36
 
27
37
  ```sh
28
38
  npx toolmenu snapshot -- node dist/server.js # or: npx toolmenu snapshot https://example.com/mcp
@@ -31,28 +41,71 @@ npx toolmenu init -- node dist/server.js # CI: a baseline, a workflow,
31
41
 
32
42
  ![toolmenu session output: tools inserted mid-list, a description edited, each pinned to the step that caused it](https://raw.githubusercontent.com/niksa90/toolmenu/main/docs/demo.svg)
33
43
 
44
+ > **Status: 0.27.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
45
+ > every rule in [docs/RULES.md](docs/RULES.md), protocol versions in [docs/PROTOCOLS.md](docs/PROTOCOLS.md)
46
+ > (written by a test that runs each one), changes in [CHANGELOG.md](CHANGELOG.md).
47
+
48
+ **Contents:** [A finding](#what-a-finding-looks-like) ·
49
+ [What it catches](#what-it-catches) · [Why "when it changes" matters](#why-when-it-changes-matters) ·
50
+ [Tested on real servers](#tested-on-real-servers) · [Start here](#start-here) ·
51
+ [CI](#in-ci-one-pr-comment) · [`session`](#session-the-menu-changing-while-the-agent-works) ·
52
+ [`diff`](#diff-compare-releases) · [`review`](#accepting-a-new-baseline-review) · [`snapshot` rules](#snapshot-rules) ·
53
+ [What it can't catch](#what-toolmenu-cant-catch) · [Options](#options)
54
+
55
+ ## What a finding looks like
56
+
57
+ A server whose `unlock_toolset` tool adds two tools to the menu mid-conversation, run
58
+ with `toolmenu session`:
59
+
60
+ ```
61
+ step 3: call unlock_toolset {"toolset":"audits"} · answered · menu changed · +2 tools · list_changed received
62
+ ERROR session/mid-insert
63
+ +2 tools inserted at position 1 (list_team_audits, get_team_audit). Invalidates the cached prompt:
64
+ the tool list and the conversation after it are processed again.
65
+ cost: the whole tool list (this server's part: ~406 tokens, estimate) and the conversation after it
66
+ → Next: Add new tools at the end of the list, not in the middle; better, list them from the start.
67
+ WARN session/unlock-coverage · unsure
68
+ unlock_toolset looks like it unlocks tools, and the run got through 1 of its 2 toolset values. […]
69
+ not unlocked: "reports"
70
+ → Next: Unlock every value: session --init puts each one in the starter.
71
+ ```
72
+
73
+ Every finding has the same parts: a **level** (`ERROR`, `WARN`, `INFO`; `--fail-on`
74
+ decides which fail the run), a **rule id** you can look up (`toolmenu explain
75
+ session/mid-insert`) or turn off in the config, **where** it happened (the step, the
76
+ tool, the parameter path), **what toolmenu saw**, and a **`→ Next:`** step.
77
+
78
+ How much to trust a finding depends on what it rests on, and the rule id's prefix says which:
79
+
80
+ | Rule family | What it rests on |
81
+ |---|---|
82
+ | `spec/`, `apps/`, `lists/schema` | The protocol: the official JSON schema for the server's version (pinned in `schemas/`), a MUST/SHOULD in the spec, or the MCP Apps extension |
83
+ | `diff/` | Two menus compared. A **breaking** change is one where a call that passed validation before now fails it |
84
+ | `session/`, `menu/`, `lists/variance`, `auth/`, `catalog/` | Behaviour toolmenu observed: the menu changing after a step, between two calls, between two processes or connections, or served to a bad credential. The finding names the step or the call |
85
+ | `naming/`, `ids/`, `description/`, `write/`, `search/` | How agents read the menu: names, words, descriptions. Tuned on real servers, but heuristics |
86
+
87
+ A finding marked **`· unsure`**, in any family, rests on names and words rather than on
88
+ something toolmenu saw happen (above: "looks like it unlocks tools"). It says what was
89
+ seen, not what it means. No LLM is involved anywhere: the same server gives the same
90
+ findings on every run.
91
+
34
92
  ## What it catches
35
93
 
36
94
  | | |
37
95
  |---|---|
38
96
  | **A menu that changes mid-session** | `session` calls your tools and watches the list. A tool added mid-conversation can rewrite the whole cached prompt: one unlock cost 40–57K tokens of cache in my logs. |
97
+ | **Every tool, called** | `session --auto` calls the tools marked read-only; `--call-everything` calls all of them, writes included, with made-up input in a container with no network and no credentials. A call that crashes, hangs or breaks the protocol is reported; a readable tool error is what the spec asks for. |
98
+ | **Results clients reject** | Results are checked against the tool's `outputSchema` (a MUST since 2025-06-18) and, on 2026-07-28, for `resultType`, the way the official SDK client checks them before your agent ever sees the answer. |
99
+ | **Sign-in, before logging in** | `auth check` walks OAuth discovery as a spec-following client does: the 401, protected resource metadata (RFC 9728), authorization server metadata (RFC 8414 or OpenID Connect), PKCE and registration. |
39
100
  | **Breaking releases** | `diff` compares two menus: removed tools, new required parameters, narrowed types, token growth. It also says whether your version bump is big enough. |
40
101
  | **The 2026-07-28 protocol** | The tool set must not vary per connection, and results need `resultType`. Elicitation, multi round-trip input and `server/discover` are checked too, as are the 2025 handshakes. |
41
102
  | **MCP Apps, Tasks, Server Cards** | It reads `ui://` resources as a host does and checks who may see each tool. It catches tasks sent to clients that can't poll them, and Server Cards that contradict the live server. |
42
- | **Your client's view** | `--client claude-code \| claude-api-search \| openai` shows what each client loads, cuts, can find and refuses. |
103
+ | **Your client's view** | `--client` with Claude Code, the Claude API, OpenAI, Codex, VS Code (Copilot Chat), Copilot cloud agent and code review, Gemini CLI, the Gemini API or Antigravity shows what each loads, cuts, can find, refuses, rewrites or leaves out, from each one's own docs. |
43
104
  | **What agents misread** | It flags vague `id`s, IDs no tool returns, instructions buried late in descriptions, and writes without annotations or marked read-only. |
105
+ | **Words routing to the wrong tool** | `routes.yml` pins which tool each keyword must (and must not) reach, the way a keyword or BM25 tool search sees the menu. A description edit that lets `get_audit_trail` win "audit" fails the check, with no model and no eval budget. [More](#snapshot-rules) |
44
106
  | **Menus that drift** | It catches defaults built from the clock or a hash order, menus that differ per process, per connection or offline, and listings that depend on who asks. |
45
107
 
46
- Every finding says what's wrong, where it is and what toolmenu saw, and ends with a `→ Next:` step.
47
- It runs on the 22-server [bench](bench/README.md) and on hosted servers (GitHub, Atlassian, Sentry,
48
- Notion, Supabase, Vercel…). What it has found on real servers is in
49
- [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
50
-
51
- > **Status: 0.25.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
52
- > every rule in [docs/RULES.md](docs/RULES.md), changes in [CHANGELOG.md](CHANGELOG.md).
53
-
54
- <details>
55
- <summary><b>Why "when it changes" matters</b></summary>
108
+ ## Why "when it changes" matters
56
109
 
57
110
  The MCP linters I found grade one snapshot of the menu: tool names, descriptions,
58
111
  schemas. toolmenu also checks **when** the menu changes:
@@ -74,10 +127,47 @@ schemas. toolmenu also checks **when** the menu changes:
74
127
  old versions with today's dependencies, so it records what they serve now
75
128
  (FINDINGS F4).
76
129
 
77
- Run against the reference server, `@modelcontextprotocol/server-everything`, it
78
- reports no false findings ([bench/](bench/README.md)).
130
+ ## Tested on real servers
79
131
 
80
- </details>
132
+ Every rule change is run against real MCP servers before it ships, and every finding
133
+ that changes on one of them is read by hand; a change has to say why in the
134
+ [changelog](CHANGELOG.md). The corpus is pinned to exact versions, so a rerun checks the
135
+ same code ([bench/README.md](bench/README.md) has the commands):
136
+
137
+ | Corpus | Servers | Checked with |
138
+ |---|---|---|
139
+ | The reference server, `@modelcontextprotocol/server-everything` 2.1.0 (2026-07-28) | 1 | `snapshot --lists`, `session --auto`, and a scenario that answers its elicitation demos. **No false findings**: a change that makes one appear is wrong until shown otherwise |
140
+ | Recent stdio servers from the MCP registry that start without credentials | 86 | `snapshot` |
141
+ | The servers across the stdio lists (registry, popular npm, PyPI) that answer the 2026-07-28 handshake | 39 | `session --auto` |
142
+ | Named servers run in a container: GitHub, Atlassian, Sentry, Notion, Supabase, Playwright, MongoDB… ([`bench/servers.txt`](bench/servers.txt)) | 22 | `snapshot`, `session --init`, `session` |
143
+ | Hosted (Streamable HTTP) servers from the registry that need no headers, sampled with a fixed seed | 200 | `snapshot`, read-only `session --auto --max-calls 10` |
144
+ | Hosted servers behind OAuth: Sentry, GitHub, Atlassian (F14) | 3 | `auth login`, then `snapshot` |
145
+ | **Weekly:** Docker's MCP catalog, every local server at the digest the catalog pins, with a dummy value for each setting it declares (F20) | 237 | `snapshot`, `session --auto`, and the sealed pass: 4,117 of the 4,237 tools listed called |
146
+ | **Weekly:** stdio servers from the registry, npm and PyPI (F17, F18); each week rechecks up to 300 new releases and newly listed servers | 1,090 at baseline | `snapshot`, `session --auto`, and a sealed `--call-everything` pass: 15,550 of 16,518 tools called, every one of the rest accounted for |
147
+ | The servers big vendors publish to run locally: GitHub, AWS Labs, Microsoft, PayPal, Stripe, Atlassian, Notion, HashiCorp, Grafana… as npm, PyPI and container images, with dummy credentials (F19) | 98 | `snapshot`, `session --auto`, and the sealed pass: 2,213 of the 2,292 tools listed called |
148
+ | **Weekly:** hosted servers, 109 vendor endpoints (GitHub, Google, Microsoft, Atlassian, Stripe…) and a seeded 2,000 of the registry's | 2,109 | `snapshot`, read-only `session --auto`, and `auth check` on each that answers 401 |
149
+
150
+ What those runs found is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md),
151
+ including what toolmenu itself got wrong and how it was fixed. A few:
152
+
153
+ - **mcp-atlassian serves a different menu every time it starts**: a default built from a
154
+ Python set, in a new order each process, so every restart misses the cache (F12).
155
+ - **Atlassian Rovo served 4 tools to any `Authorization` header**, valid or not, so a
156
+ client whose token expired gets fewer tools instead of a reason to log in again (`auth/invalid-accepted`).
157
+ - **The official filesystem server has reported `0.2.0` for 19 releases**, so a
158
+ server's self-reported version can't be used for semver checks (F5).
159
+ - **Notion's server ships the same 9 unused `$defs` in all 24 tools**: 72% of its menu.
160
+ - **12 of 1,090 stdio servers fail in the official SDK client today** (8 lose every tool,
161
+ 4 lose answers), and 22 of 2,000 hosted ones send a 2026-07-28 tool list that client
162
+ rejects (F18).
163
+ - **PayPal's server builds a new menu on every call**: `list_transactions` defaults its dates to
164
+ the moment the list is built, so no cached prompt matches the next one (F19).
165
+ - **Microsoft Sentinel's sign-in can't pass a client that follows MCP 2025-11-25 to the
166
+ letter**: Entra ID supports PKCE but its OpenID Connect metadata doesn't say so, and the
167
+ spec tells clients to refuse (F18).
168
+
169
+ The weekly run compares each week with the last: menus that changed, servers that fixed
170
+ or gained a finding, and the trend ([bench/results/weekly](bench/results/weekly)).
81
171
 
82
172
  ## Start here
83
173
 
@@ -110,6 +200,7 @@ By hand:
110
200
  ```sh
111
201
  npx toolmenu snapshot -- node dist/server.js # the menu, its findings, menu.json
112
202
  npx toolmenu diff menu.json new-menu.json # what changed between two releases
203
+ npx toolmenu review menu.json new-menu.json # should the new menu replace the baseline?
113
204
  npx toolmenu session --auto -- node dist/server.js # call read-only tools, watch the menu
114
205
  ```
115
206
 
@@ -123,6 +214,12 @@ npx toolmenu auth login https://mcp.example.com/mcp
123
214
  npx toolmenu snapshot https://mcp.example.com/mcp
124
215
  ```
125
216
 
217
+ Before logging in, `toolmenu auth check https://mcp.example.com/mcp` walks the server's
218
+ OAuth discovery the way a client that follows the spec does: the 401 and its
219
+ `WWW-Authenticate`, the protected resource metadata (and whether its `resource` is the
220
+ server's URL), the authorization server's metadata, PKCE and registration. No login, no
221
+ tokens: only the requests a client makes first.
222
+
126
223
  Logins are stored per server in `~/.config/toolmenu/auth` (readable only by you);
127
224
  `auth list` and `auth logout <url>` manage them. A server that doesn't allow
128
225
  dynamic registration needs a pre-registered app: `--client-id` (and
@@ -133,22 +230,22 @@ CI, where nobody can sign in, pass a token as a header.
133
230
  Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
134
231
  and the 2025 `initialize` handshake, through the official TypeScript SDK.
135
232
 
136
- Every finding and error says what's wrong, where, what toolmenu saw, and ends in a
137
- `→ Next:` step. Findings marked `· unsure` are heuristics (names and words): they say
138
- what was seen, not what it means. `toolmenu explain <rule>` says what a rule checks,
139
- its default level and how to change it (every rule is in [docs/RULES.md](docs/RULES.md)).
140
- Every command has its own help: `toolmenu history --help`.
233
+ How to read a finding is [above](#what-a-finding-looks-like). `toolmenu explain <rule>`
234
+ says what a rule checks, its default level and how to change it (every rule is in
235
+ [docs/RULES.md](docs/RULES.md)). Every command has its own help: `toolmenu history --help`.
141
236
 
142
237
  Behind a proxy (`HTTPS_PROXY`), stdio servers get the proxy and CA settings
143
238
  (`HTTPS_PROXY`, `NO_PROXY`, `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`…)
144
239
  along with the minimal environment, and `NODE_USE_ENV_PROXY=1` so a Node server's
145
240
  `fetch` uses the proxy. Without them a server that calls an API can't reach it, and
146
- fails as if it were broken.
241
+ fails as if it were broken. A proxy URL with a password in it
242
+ (`https://user:pass@proxy:8080`) reaches the server that way too: give a server you
243
+ don't trust a proxy without one, or `--env HTTPS_PROXY=` to leave it out.
147
244
 
148
245
  ### Every server you use: `scan`
149
246
 
150
247
  ```sh
151
- npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json
248
+ npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json --allow-env LINEAR_API_KEY
152
249
  npx toolmenu scan .mcp.json --format markdown
153
250
  ```
154
251
 
@@ -165,6 +262,13 @@ findings, like this:
165
262
  linear – – not checked: LINEAR_API_KEY is not set (the config refers to it)
166
263
  ```
167
264
 
265
+ `scan` starts every command the list names, so scan only lists you'd run yourself (not
266
+ a `.mcp.json` from a repository you haven't read). A config's `${VAR}` is filled only
267
+ from the variables you name with `--allow-env` (comma-separated, or `'*'` for all): the
268
+ config decides where each value goes, a URL or a header included, so one from someone
269
+ else could otherwise send your environment to a server of theirs. An entry that uses
270
+ a variable you didn't allow isn't checked, and says which `--allow-env` it needs.
271
+
168
272
  `${VAR}`, `${VAR:-default}` and `${env:VAR}` in the config are filled from the
169
273
  environment. A server whose variable isn't set isn't started with an empty
170
274
  credential: it's a row that says so. A server that couldn't be started or read
@@ -200,6 +304,33 @@ Relative paths resolve from the folder `toolmenu mcp` runs in, or a call's `cwd`
200
304
  clients ask before each tool call unless you allow it. toolmenu's own menu passes its
201
305
  checks with no findings, on both protocols and in a session (the test suite checks).
202
306
 
307
+ ## As a library
308
+
309
+ Everything the CLI does is exported from the `toolmenu` package, with types, for test
310
+ suites and scripts of your own:
311
+
312
+ ```js
313
+ import { snapshot, diffMenus, loadMenu, session, loadScenario } from 'toolmenu';
314
+
315
+ // The menu and its findings, as `toolmenu snapshot` gives them.
316
+ const { menu, findings } = await snapshot({ kind: 'stdio', command: 'node', args: ['dist/server.js'] });
317
+ // or { kind: 'http', url: 'https://example.com/mcp', headers: { Authorization: `Bearer ${token}` } }
318
+
319
+ // What changed since the committed baseline, and the version bump it needs.
320
+ const diff = diffMenus(await loadMenu('menu.json'), menu);
321
+ console.log(diff.suggestedBump, diff.tokens);
322
+ if (diff.findings.some((f) => f.severity === 'error')) process.exitCode = 1;
323
+ ```
324
+
325
+ `snapshot`, `diffMenus`, `session` and `history` take the same options as their
326
+ commands (`SnapshotOptions`, `DiffOptions`, `SessionOptions`, `HistoryOptions`) and
327
+ return the findings the reports print: each `Finding` has its `rule`, `severity`,
328
+ `message`, `tool` and the `fix` shown as `→ Next:`. Lower down, `compareMenus(before,
329
+ after)` returns each tool's change (`ToolChange[]`) and `runRules` runs the menu rules
330
+ on a menu you built. The `menu.json` format is the `Menu` type (`toolmenu: 1`);
331
+ a later format will bump that number. Library functions don't print or exit: that's
332
+ the CLI's job.
333
+
203
334
  ## In CI: one PR comment
204
335
 
205
336
  ```yaml
@@ -216,7 +347,7 @@ jobs:
216
347
  - uses: actions/setup-node@v7
217
348
  with: { node-version: 22 }
218
349
  - run: npm ci && npm run build
219
- - uses: niksa90/toolmenu@v0.25.1
350
+ - uses: niksa90/toolmenu@v0.27.0
220
351
  with:
221
352
  command: node dist/server.js
222
353
  baseline: menu.json # your committed snapshot
@@ -225,7 +356,8 @@ jobs:
225
356
 
226
357
  It snapshots the PR's build, diffs it against the baseline as it is on the PR's base
227
358
  branch (or at the previous tag, on a tag push), runs the scenario, and
228
- posts **one comment that updates on every push**: breaking changes, the token change
359
+ posts **one comment that updates on every push**: first the verdict (can this menu
360
+ replace the baseline as it is?), then breaking changes, the token change
229
361
  ("this PR adds ~1,240 tokens to every conversation"), the suggested version bump,
230
362
  and anything `session` caught. The same report goes to the job summary.
231
363
 
@@ -240,6 +372,7 @@ and anything `session` caught. The same report goes to the job summary.
240
372
  | `baseline-from` | `snapshot` | `session` diffs every tool the scenario saw (the union menu), for servers whose tools appear after an unlock |
241
373
  | `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
242
374
  | `fail-on` | `error` | Fail the job on findings at or above this level |
375
+ | `gate` | `findings` | `review` passes or fails the job on the [review verdict](#accepting-a-new-baseline-review) instead of `diff`'s findings: a breaking change the release bump allows passes, new findings at `fail-on` fail ([more](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#the-verdict-and-gate-review)) |
243
376
  | `comment` | `true` | Post and update the PR comment |
244
377
  | `sarif` | | A folder to also write the findings to as SARIF, for code scanning (below) |
245
378
  | `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
@@ -264,7 +397,7 @@ permissions:
264
397
  security-events: write # to upload SARIF
265
398
  steps:
266
399
  # … checkout, build, as above
267
- - uses: niksa90/toolmenu@v0.25.1
400
+ - uses: niksa90/toolmenu@v0.27.0
268
401
  with:
269
402
  command: node dist/server.js
270
403
  sarif: toolmenu-sarif
@@ -278,6 +411,50 @@ Outside the Action: `toolmenu snapshot --sarif toolmenu.sarif -- node dist/serve
278
411
  writes the file beside the usual report (`diff` and `session` take `--sarif` too), or
279
412
  `--format sarif` prints it instead.
280
413
 
414
+ ### Other CI systems: JUnit test reports
415
+
416
+ GitLab, Jenkins, Azure DevOps, CircleCI and Buildkite show JUnit XML as test results.
417
+ `--junit <path>` writes the findings of `snapshot`, `diff` or `session` as JUnit beside
418
+ the usual report (`--format junit` prints it instead). New in 0.25.2. Each finding is a test case named
419
+ after its tool, in a class named after its rule (`toolmenu.diff.diff/removed`), so the
420
+ CI system tracks it across runs. Findings at or above `--fail-on` are failures, the rest
421
+ pass with their text, so the test report agrees with the exit code. A clean run is one
422
+ passing case.
423
+
424
+ ```yaml
425
+ # .gitlab-ci.yml
426
+ toolmenu:
427
+ image: node:22
428
+ script:
429
+ - npm ci && npm run build
430
+ # Exit 1 means findings: keep going so both reports are written, then fail.
431
+ - npx toolmenu snapshot --out new-menu.json --junit snapshot.xml -- node dist/server.js || code=$?
432
+ - npx toolmenu diff menu.json new-menu.json --junit diff.xml || code=$?
433
+ - exit ${code:-0}
434
+ artifacts:
435
+ when: always
436
+ reports:
437
+ junit: [snapshot.xml, diff.xml]
438
+ ```
439
+
440
+ ### A report to keep: HTML
441
+
442
+ `--html <path>` writes the run of `snapshot`, `diff` or `session` as one HTML page
443
+ beside the usual report: the numbers that matter at the top (errors, warnings, tools,
444
+ tokens; for `diff`, breaking changes and the suggested bump), the menu as a table in
445
+ server order with each tool's share of the tokens, and the findings with their next
446
+ steps. `diff` shows each tool before and after, new and removed tools marked. `session`
447
+ has a row per step with the menu after it, tools inserted, appended or edited marked,
448
+ and links from a step to its findings.
449
+
450
+ The page has no script and loads nothing, so it opens offline: keep it as a CI artifact
451
+ (GitLab `artifacts: paths`, `actions/upload-artifact`) or attach it to an issue for a
452
+ server's maintainer. It follows the viewer's light or dark setting.
453
+
454
+ ```sh
455
+ npx toolmenu diff menu.json new-menu.json --html diff.html
456
+ ```
457
+
281
458
  ### A badge for your README
282
459
 
283
460
  ```sh
@@ -299,6 +476,22 @@ PRs without secrets are skipped, and a skipped check still passes; the `skipped`
299
476
  `session` runs a scripted session (the scenario *is* the agent), lists the menu
300
477
  after every step, and reports each change with the step that caused it.
301
478
 
479
+ ```
480
+ step the scenario (the agent) the menu the client sees toolmenu
481
+ ──── ─────────────────────────────── ─────────────────────────────── ─────────────────────────────
482
+ 1 list search_forms, get_form, … baseline: 5 tools
483
+ 2 call search_forms (unchanged) no change
484
+ 3 call unlock_toolset audits search_forms, list_team_audits, +2 tools at position 1:
485
+ get_team_audit, get_form, … session/mid-insert (error)
486
+ fresh process: not there →
487
+ session/side-effect (warn)
488
+ ```
489
+
490
+ Each step's menu is compared with the one before it, so a change is pinned to the call
491
+ that caused it, with where in the list it starts and what it costs the prompt cache.
492
+ toolmenu then starts a second process (or opens a second connection) to see whether
493
+ the change is global, or only this session's, which 2026-07-28 rules out.
494
+
302
495
  ```yaml
303
496
  # scenario.yml
304
497
  allow_writes: false # refuse to call tools not marked readOnlyHint
@@ -348,6 +541,7 @@ serves, and 23 issue and pull-request tools that only the dynamic mode has.
348
541
  | `session/error-not-flagged` | warn | A tool answered with an HTTP error from the API behind it (`{"status":401,"object":"error",…}`) as a plain result, not marked `isError`, so a client that reads `isError` takes the call for a success (Notion's server). An auth one also counts toward `session/untested`. Tools that take a URL are left out: their answer is the page's (heuristic) |
349
542
  | `session/server-unresponsive` | error | The server stopped answering after a step: even `tools/list` timed out. The run stops there with this one finding, instead of every later step waiting out its timeout (a server that polls a device-code sign-in inside a tool call) |
350
543
  | `session/handle-roundtrip` | warn | With `--create-handles`: the tool that creates a session, basket or cart was called, and the tools that take it back refused the handle it had just returned (info when its result had no handle to find) |
544
+ | `session/output-schema` | error | A tool that declares an `outputSchema` answered with no `structuredContent`, or one that doesn't match it: from 2025-06-18 a **MUST**, and the official SDK client raises an error instead of the answer |
351
545
  | `session/rate-limited` | error | The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it |
352
546
  | `session/scope-unchecked` | info | A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run |
353
547
 
@@ -380,6 +574,14 @@ with why each tool was left out, not a clean pass. On the 22 servers in `bench/`
380
574
  **`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
381
575
  not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
382
576
 
577
+ **`--call-everything`** (with `--auto`, stdio only) calls **every** tool once, writes and
578
+ open-world ones included, with a made-up value (`toolmenu-test`, `1`, a one-item list)
579
+ where the schema gives none. It's for a server in a sandbox with nothing to lose: a
580
+ container with no credentials and no network, as `bench/scripts/batch-full.sh` runs
581
+ them with `SEALED=1`. Every tool's answer is then checked: results against their own
582
+ `outputSchema`, errors flagged as errors, calls that hang. It's refused over HTTP,
583
+ because a hosted server's tools act on someone's real account.
584
+
383
585
  ## `diff`: compare releases
384
586
 
385
587
  ```sh
@@ -458,6 +660,40 @@ with `--server-version-is-release`. `diff` also enforces an optional `tokenBudge
458
660
  (`diff/token-budget`). Order changes are only a notice here: between releases, prompt caches rebuild. Order
459
661
  matters *within* a session.
460
662
 
663
+ ### Accepting a new baseline: `review`
664
+
665
+ `diff` says what changed. Before you commit a refreshed `menu.json`, `review` answers
666
+ the question that follows: should it replace the baseline?
667
+
668
+ ```sh
669
+ npx toolmenu snapshot --out new-menu.json -- node dist/server.js
670
+ npx toolmenu review --release 1.4.0..1.5.0 menu.json new-menu.json
671
+ ```
672
+
673
+ ```
674
+ toolmenu review new-menu.json as the new menu.json
675
+ verdict ✗ don't accept as is: 1 breaking change, and the release's minor bump is too small (release 2.0.0).
676
+ changes 1 breaking (1 tool) · 0 minor · 3 notice (13 tools)
677
+ findings 0 new · 0 gone · 1 changed · 1 unchanged
678
+ ```
679
+
680
+ It runs `diff`, and runs the menu rules on both files, so it also says which findings
681
+ the new menu **adds**, which it **clears**, and which **changed** (a summary that now
682
+ names another tool counts as worse). The verdict is one of:
683
+
684
+ | Verdict | Exit | When |
685
+ |---|---|---|
686
+ | `unchanged` | 0 | The same menu and the same findings |
687
+ | `safe` | 0 | Nothing breaks callers, and no finding at or above `--fail-on` is new or worse |
688
+ | `breaking-released` | 0 | Breaking changes, and `--release` shows a bump that allows them |
689
+ | `breaking` | 1 | Breaking changes, and no release that allows them: none given, too small, or not checkable (calendar versions) |
690
+ | `review` | 1 | Nothing breaks, but findings at or above `--fail-on` are new or worse, or `diff`'s own checks fail (`tokenBudget`) |
691
+
692
+ It reads only the two files, so rules that need the live server (a second process,
693
+ the raw `tools/list`) aren't part of it; `snapshot` runs those. `--format markdown`
694
+ gives the same as a PR comment, `--json` for scripts (`verdict`, `accept`, and the
695
+ findings by kind).
696
+
461
697
  ### Operations behind a search tool (`--catalog`)
462
698
 
463
699
  Big servers increasingly keep most operations out of the menu: a small fixed menu, a
@@ -629,6 +865,26 @@ client doesn't document is left out, not guessed.
629
865
  | `claude-code` | tool names; full definitions for tools marked `_meta["anthropic/alwaysLoad"]` | reads a description up to 4,096 characters | 16,384 through search, 4,096 up front | Claude API's |
630
866
  | `claude-api-search` | nothing from tools you defer (keep the 3–5 used most loaded) | name, description, argument names and descriptions; 5 results by default | none documented | `^[a-zA-Z0-9_-]{1,128}$` |
631
867
  | `openai` | each deferred function's name and description (its schema loads on demand); a deferred MCP server shows only its own name and description | how many results, and which fields, isn't documented, so `search/findable` doesn't run | none documented | `^[a-zA-Z0-9_-]+$`, up to 128 |
868
+ | `codex` | not documented | not documented | none documented | OpenAI's (it sends tools to the Responses API) |
869
+ | `antigravity` | each tool's name; the agent reads a tool's schema before calling it, unless the user marks it `"eager"` | not a search: the agent reads the schema file | none documented | not documented |
870
+ | `gemini-cli` | not documented | not documented | none documented | rewrites: characters outside letters, digits, `_ - . :` become `_`, names over 63 characters are cut in the middle |
871
+ | `gemini-api` | not documented | not documented | none documented | letters, digits, `_ - . :`, up to 128 |
872
+
873
+ Three more check what a client limits rather than what it loads:
874
+
875
+ | `--client` | What it checks |
876
+ |---|---|
877
+ | `vscode` | VS Code (GitHub Copilot Chat) sends at most 128 tools a request: a server over it alone is `menu/client-limit` |
878
+ | `copilot-cloud-agent` | uses MCP tools only, not prompts or resources (`menu/client-features`, with `--lists`) |
879
+ | `copilot-code-review` | the same, and only tools marked `readOnlyHint: true`: the rest are listed |
880
+
881
+ Read, but nothing in them toolmenu can check yet: Copilot CLI defers tools behind a
882
+ search at about 30 (it searches names, descriptions and parameters; how many results
883
+ isn't said); Copilot in JetBrains, Visual Studio, Eclipse and Xcode documents no tool
884
+ limit; JetBrains AI Assistant and Junie document none either (Junie CLI times out any
885
+ request after 60 s, startup and `tools/list` included). Not here, because their docs
886
+ don't state it: Cursor's tool limit (forum posts say 40), Windsurf's (100 in older docs,
887
+ not in the current page), ChatGPT's. A value comes in when the client documents it.
632
888
 
633
889
  The snapshot adds a line such as `as Claude Code: up front ~524 tokens · on demand ~344
634
890
  per tool a search loads (largest: update_contacts, ~2,093)`, and `--json` a `client`
@@ -656,7 +912,10 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
656
912
  --no-write don't write the menu file
657
913
  --routes <path> routes.yml
658
914
  --config <path> config file (default: toolmenu.config.json, if present)
659
- --format <fmt> text (default), json, github (annotations) or markdown
915
+ --format <fmt> text (default), json, github (annotations), markdown, sarif
916
+ (code scanning) or junit (CI test reports)
917
+ --junit <path> snapshot, diff, session: also write the findings as JUnit XML
918
+ --html <path> snapshot, diff, session: also write the report as one HTML page
660
919
  --fail-on <level> error (default), warn or info
661
920
  --header <k: v> HTTP header, repeatable
662
921
  --env <K=V> env var for a stdio server, repeatable (it only gets a
@@ -669,18 +928,36 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
669
928
  --lists <kinds> snapshot: also read and check prompts, resources (comma-separated);
670
929
  saved in the menu file beside the tools
671
930
  --client <name> snapshot: check the menu for one client, from its own docs:
672
- claude-code, claude-api-search or openai
931
+ claude-code, claude-api-search, openai, codex, vscode,
932
+ copilot-cloud-agent, copilot-code-review, gemini-cli,
933
+ gemini-api or antigravity
673
934
  --as <name>:<cred> snapshot: list the menu as this identity too, repeatable:
674
935
  "reader:Authorization: Bearer …" (HTTP) or "admin:ROLE=admin"
675
936
  (stdio). The first is the menu saved; the others are compared
676
937
  --offline-check snapshot (stdio): also list the menu with the network cut off
677
938
  --concurrency <n> scan: servers checked at a time (default: 4)
678
939
  --out-dir <dir> scan: write each server's menu file there, as <name>.json
940
+ --allow-env <names> scan: variables a config's ${VAR} may be filled from (default:
941
+ none; '*' for all)
679
942
  ```
680
943
 
681
944
  Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
682
945
  (for `scan`: a server wasn't checked) or bad usage. `toolmenu <command> --help` lists that command's own options.
683
946
 
947
+ Credentials on the command line (`--header`, `--env`, `--client-secret`) can be read by
948
+ every user of the machine while toolmenu runs (`ps`). To keep them off it, put the
949
+ options in `TOOLMENU_ARGS`, a JSON array of toolmenu options and their values; they go
950
+ before the server command's `--`. A process's environment is readable only by its own
951
+ user (and root), so this keeps them from other users, not from other processes of
952
+ yours. jq builds the array whatever the token holds:
953
+
954
+ ```sh
955
+ TOOLMENU_ARGS=$(jq -cn --arg t "$TOKEN" '["--header", "Authorization: Bearer " + $t]') npx toolmenu snapshot https://example.com/mcp
956
+ ```
957
+
958
+ The GitHub Action passes its `headers` and `env` this way (to toolmenu 0.26.0 and
959
+ later; an older `version` gets them as arguments, with a warning).
960
+
684
961
  ## Limitations
685
962
 
686
963
  - **Server order isn't final client order.** Clients merge tools from several servers
package/dist/args.d.ts CHANGED
@@ -19,6 +19,16 @@ export declare function fits(value: unknown, schema: JsonSchema | undefined): bo
19
19
  * tool that needs one is skipped, not called with junk.
20
20
  */
21
21
  export declare function synthesize(schema: JsonSchema | undefined, name?: string): Synthesized;
22
+ /** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
23
+ export declare const INVENTED = "toolmenu-test";
24
+ /**
25
+ * A value of the right type where the schema itself vouches for none, for
26
+ * `session --auto --call-everything`: the schema's own value first (synthesize),
27
+ * then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
28
+ * required fields). A pattern it can't satisfy gets the made-up string anyway: the
29
+ * tool's answer to a value it refuses is part of what's checked.
30
+ */
31
+ export declare function invent(schema: JsonSchema | undefined, name?: string, depth?: number): unknown;
22
32
  /**
23
33
  * A value the user gave (--value), made to fit the parameter: `5` is the number 5
24
34
  * for an integer and the string "5" for a string; JSON that doesn't fit the type
@@ -29,7 +39,7 @@ export declare function fitValue(given: {
29
39
  value: unknown;
30
40
  }, schema: JsonSchema | undefined): unknown;
31
41
  /** Where a tool's argument came from: the user (--value) or the schema. */
32
- export type ArgSource = 'value' | 'schema';
42
+ export type ArgSource = 'value' | 'schema' | 'invented';
33
43
  /**
34
44
  * Arguments for a tool's required parameters, or the ones it can't fill. `given`:
35
45
  * values the user supplied for this tool, by parameter name; they come first, and
@@ -38,7 +48,9 @@ export type ArgSource = 'value' | 'schema';
38
48
  export declare function synthesizeArgs(schema: JsonSchema | undefined, given?: Record<string, {
39
49
  raw?: string;
40
50
  value: unknown;
41
- }>): {
51
+ }>, options?: {
52
+ invent?: boolean;
53
+ }): {
42
54
  ok: true;
43
55
  args: Record<string, unknown>;
44
56
  sources: Record<string, ArgSource>;
package/dist/args.js CHANGED
@@ -107,6 +107,45 @@ export function synthesize(schema, name = '') {
107
107
  return { ok: false };
108
108
  }
109
109
  }
110
+ /** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
111
+ export const INVENTED = 'toolmenu-test';
112
+ /**
113
+ * A value of the right type where the schema itself vouches for none, for
114
+ * `session --auto --call-everything`: the schema's own value first (synthesize),
115
+ * then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
116
+ * required fields). A pattern it can't satisfy gets the made-up string anyway: the
117
+ * tool's answer to a value it refuses is part of what's checked.
118
+ */
119
+ export function invent(schema, name = '', depth = 0) {
120
+ const own = synthesize(schema, name);
121
+ if (own.ok)
122
+ return own.value;
123
+ if (!schema || typeof schema !== 'object' || depth > 8)
124
+ return INVENTED;
125
+ const branches = (schema.anyOf ?? schema.oneOf);
126
+ const type = Array.isArray(schema.type) ? schema.type.find((t) => t !== 'null') : schema.type;
127
+ if (!type && Array.isArray(branches) && branches.length)
128
+ return invent(branches.find((b) => b && typeof b === 'object' && b.type !== 'null') ?? branches[0], name, depth + 1);
129
+ switch (type) {
130
+ case 'boolean':
131
+ return false;
132
+ case 'integer':
133
+ case 'number':
134
+ return 1;
135
+ case 'array':
136
+ return [invent(schema.items, name, depth + 1)];
137
+ case 'object': {
138
+ const out = {};
139
+ for (const p of schema.required ?? [])
140
+ out[p] = invent(schema.properties?.[p], p, depth + 1);
141
+ return out;
142
+ }
143
+ case 'null':
144
+ return null;
145
+ default:
146
+ return INVENTED;
147
+ }
148
+ }
110
149
  /**
111
150
  * A value the user gave (--value), made to fit the parameter: `5` is the number 5
112
151
  * for an integer and the string "5" for a string; JSON that doesn't fit the type
@@ -127,7 +166,7 @@ export function fitValue(given, schema) {
127
166
  * values the user supplied for this tool, by parameter name; they come first, and
128
167
  * one for an optional parameter is passed too.
129
168
  */
130
- export function synthesizeArgs(schema, given = {}) {
169
+ export function synthesizeArgs(schema, given = {}, options = {}) {
131
170
  const args = {};
132
171
  const sources = {};
133
172
  const missing = [];
@@ -143,6 +182,10 @@ export function synthesizeArgs(schema, given = {}) {
143
182
  args[param] = v.value;
144
183
  sources[param] = 'schema';
145
184
  }
185
+ else if (options.invent) {
186
+ args[param] = invent(schema?.properties?.[param], param);
187
+ sources[param] = 'invented';
188
+ }
146
189
  else
147
190
  missing.push(param);
148
191
  }