toolmenu 0.25.2 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +248 -29
  2. package/dist/args.d.ts +14 -2
  3. package/dist/args.js +44 -1
  4. package/dist/auth-check.d.ts +27 -0
  5. package/dist/auth-check.js +177 -0
  6. package/dist/auth.d.ts +7 -0
  7. package/dist/auth.js +29 -1
  8. package/dist/auto.d.ts +8 -0
  9. package/dist/auto.js +9 -2
  10. package/dist/breakdown.js +14 -10
  11. package/dist/cli.js +150 -48
  12. package/dist/clients.d.ts +13 -2
  13. package/dist/clients.js +65 -11
  14. package/dist/config.js +6 -0
  15. package/dist/diff.d.ts +35 -1
  16. package/dist/diff.js +57 -12
  17. package/dist/failures.js +4 -3
  18. package/dist/help.d.ts +1 -1
  19. package/dist/help.js +43 -16
  20. package/dist/history.js +5 -0
  21. package/dist/html.d.ts +29 -0
  22. package/dist/html.js +360 -0
  23. package/dist/index.d.ts +3 -2
  24. package/dist/index.js +3 -2
  25. package/dist/init.d.ts +14 -0
  26. package/dist/init.js +20 -6
  27. package/dist/junit.d.ts +6 -1
  28. package/dist/junit.js +12 -3
  29. package/dist/mcp-env.d.ts +6 -0
  30. package/dist/mcp-env.js +6 -0
  31. package/dist/mcp.d.ts +10 -0
  32. package/dist/mcp.js +76 -15
  33. package/dist/menu.d.ts +6 -0
  34. package/dist/menu.js +12 -1
  35. package/dist/options.d.ts +10 -0
  36. package/dist/options.js +3 -0
  37. package/dist/report.d.ts +14 -0
  38. package/dist/report.js +57 -51
  39. package/dist/review.d.ts +48 -0
  40. package/dist/review.js +212 -0
  41. package/dist/rule-docs.json +49 -4
  42. package/dist/rules/clients.d.ts +11 -0
  43. package/dist/rules/clients.js +65 -2
  44. package/dist/rules/determinism.d.ts +3 -0
  45. package/dist/rules/determinism.js +44 -7
  46. package/dist/rules/ids.js +5 -1
  47. package/dist/rules/index.js +5 -2
  48. package/dist/rules/output.d.ts +21 -0
  49. package/dist/rules/output.js +49 -0
  50. package/dist/rules/search.js +1 -1
  51. package/dist/rules/spec.d.ts +19 -0
  52. package/dist/rules/spec.js +86 -1
  53. package/dist/sanitize.d.ts +42 -0
  54. package/dist/sanitize.js +142 -0
  55. package/dist/scan.d.ts +9 -2
  56. package/dist/scan.js +46 -24
  57. package/dist/session.d.ts +17 -0
  58. package/dist/session.js +119 -13
  59. package/dist/snapshot.js +2 -1
  60. package/package.json +5 -3
package/README.md CHANGED
@@ -5,7 +5,7 @@
5
5
  <h1 align="center">toolmenu</h1>
6
6
 
7
7
  <p align="center">
8
- <b>Checks for MCP servers' tool lists: changes that confuse agents, break prompt caches or break clients, caught before your users hit them.</b>
8
+ <b>Tests your MCP server the way agents and clients meet it: the menu they read, the calls they make, the results and sign-in they get back, and every release after. Problems caught before your users hit them.</b>
9
9
  </p>
10
10
 
11
11
  <p align="center">
@@ -22,9 +22,17 @@
22
22
  </p>
23
23
 
24
24
  Your MCP server's tool list is the menu every agent reads before it does anything.
25
- toolmenu checks that menu the way clients and agents will meet it: across calls,
26
- connections, clients and releases. No LLM is involved, so the same inputs always give
27
- the same answer and it fits in CI.
25
+ toolmenu starts there and follows the agent in. It **connects** as a client does (stdio
26
+ or HTTP, every protocol version from 2024-11-05 to 2026-07-28), **reads** the menu,
27
+ **calls** the tools (read-only by default; every tool, writes included, in a sealed
28
+ sandbox with `--call-everything`), **checks** each result against the spec and the tool's
29
+ own `outputSchema`, **walks** the OAuth discovery a client follows before it logs in, and
30
+ **compares** the menu across steps, connections, processes, clients and releases.
31
+
32
+ It isn't only a linter: a linter grades the text of one snapshot, and much of what
33
+ breaks agents only shows when you call the server, reconnect, or ship the next version.
34
+ No LLM is involved, so the same server gives the same answer on every run and it fits
35
+ in CI. The same checks run [every week on ~3,200 public servers](#tested-on-real-servers).
28
36
 
29
37
  ```sh
30
38
  npx toolmenu snapshot -- node dist/server.js # or: npx toolmenu snapshot https://example.com/mcp
@@ -33,29 +41,71 @@ npx toolmenu init -- node dist/server.js # CI: a baseline, a workflow,
33
41
 
34
42
  ![toolmenu session output: tools inserted mid-list, a description edited, each pinned to the step that caused it](https://raw.githubusercontent.com/niksa90/toolmenu/main/docs/demo.svg)
35
43
 
44
+ > **Status: 0.27.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
45
+ > every rule in [docs/RULES.md](docs/RULES.md), protocol versions in [docs/PROTOCOLS.md](docs/PROTOCOLS.md)
46
+ > (written by a test that runs each one), changes in [CHANGELOG.md](CHANGELOG.md).
47
+
48
+ **Contents:** [A finding](#what-a-finding-looks-like) ·
49
+ [What it catches](#what-it-catches) · [Why "when it changes" matters](#why-when-it-changes-matters) ·
50
+ [Tested on real servers](#tested-on-real-servers) · [Start here](#start-here) ·
51
+ [CI](#in-ci-one-pr-comment) · [`session`](#session-the-menu-changing-while-the-agent-works) ·
52
+ [`diff`](#diff-compare-releases) · [`review`](#accepting-a-new-baseline-review) · [`snapshot` rules](#snapshot-rules) ·
53
+ [What it can't catch](#what-toolmenu-cant-catch) · [Options](#options)
54
+
55
+ ## What a finding looks like
56
+
57
+ A server whose `unlock_toolset` tool adds two tools to the menu mid-conversation, run
58
+ with `toolmenu session`:
59
+
60
+ ```
61
+ step 3: call unlock_toolset {"toolset":"audits"} · answered · menu changed · +2 tools · list_changed received
62
+ ERROR session/mid-insert
63
+ +2 tools inserted at position 1 (list_team_audits, get_team_audit). Invalidates the cached prompt:
64
+ the tool list and the conversation after it are processed again.
65
+ cost: the whole tool list (this server's part: ~406 tokens, estimate) and the conversation after it
66
+ → Next: Add new tools at the end of the list, not in the middle; better, list them from the start.
67
+ WARN session/unlock-coverage · unsure
68
+ unlock_toolset looks like it unlocks tools, and the run got through 1 of its 2 toolset values. […]
69
+ not unlocked: "reports"
70
+ → Next: Unlock every value: session --init puts each one in the starter.
71
+ ```
72
+
73
+ Every finding has the same parts: a **level** (`ERROR`, `WARN`, `INFO`; `--fail-on`
74
+ decides which fail the run), a **rule id** you can look up (`toolmenu explain
75
+ session/mid-insert`) or turn off in the config, **where** it happened (the step, the
76
+ tool, the parameter path), **what toolmenu saw**, and a **`→ Next:`** step.
77
+
78
+ How much to trust a finding depends on what it rests on, and the rule id's prefix says which:
79
+
80
+ | Rule family | What it rests on |
81
+ |---|---|
82
+ | `spec/`, `apps/`, `lists/schema` | The protocol: the official JSON schema for the server's version (pinned in `schemas/`), a MUST/SHOULD in the spec, or the MCP Apps extension |
83
+ | `diff/` | Two menus compared. A **breaking** change is one where a call that passed validation before now fails it |
84
+ | `session/`, `menu/`, `lists/variance`, `auth/`, `catalog/` | Behaviour toolmenu observed: the menu changing after a step, between two calls, between two processes or connections, or served to a bad credential. The finding names the step or the call |
85
+ | `naming/`, `ids/`, `description/`, `write/`, `search/` | How agents read the menu: names, words, descriptions. Tuned on real servers, but heuristics |
86
+
87
+ A finding marked **`· unsure`**, in any family, rests on names and words rather than on
88
+ something toolmenu saw happen (above: "looks like it unlocks tools"). It says what was
89
+ seen, not what it means. No LLM is involved anywhere: the same server gives the same
90
+ findings on every run.
91
+
36
92
  ## What it catches
37
93
 
38
94
  | | |
39
95
  |---|---|
40
96
  | **A menu that changes mid-session** | `session` calls your tools and watches the list. A tool added mid-conversation can rewrite the whole cached prompt: one unlock cost 40–57K tokens of cache in my logs. |
97
+ | **Every tool, called** | `session --auto` calls the tools marked read-only; `--call-everything` calls all of them, writes included, with made-up input in a container with no network and no credentials. A call that crashes, hangs or breaks the protocol is reported; a readable tool error is what the spec asks for. |
98
+ | **Results clients reject** | Results are checked against the tool's `outputSchema` (a MUST since 2025-06-18) and, on 2026-07-28, for `resultType`, the way the official SDK client checks them before your agent ever sees the answer. |
99
+ | **Sign-in, before logging in** | `auth check` walks OAuth discovery as a spec-following client does: the 401, protected resource metadata (RFC 9728), authorization server metadata (RFC 8414 or OpenID Connect), PKCE and registration. |
41
100
  | **Breaking releases** | `diff` compares two menus: removed tools, new required parameters, narrowed types, token growth. It also says whether your version bump is big enough. |
42
101
  | **The 2026-07-28 protocol** | The tool set must not vary per connection, and results need `resultType`. Elicitation, multi round-trip input and `server/discover` are checked too, as are the 2025 handshakes. |
43
102
  | **MCP Apps, Tasks, Server Cards** | It reads `ui://` resources as a host does and checks who may see each tool. It catches tasks sent to clients that can't poll them, and Server Cards that contradict the live server. |
44
- | **Your client's view** | `--client claude-code \| claude-api-search \| openai` shows what each client loads, cuts, can find and refuses. |
103
+ | **Your client's view** | `--client` with Claude Code, the Claude API, OpenAI, Codex, VS Code (Copilot Chat), Copilot cloud agent and code review, Gemini CLI, the Gemini API or Antigravity shows what each loads, cuts, can find, refuses, rewrites or leaves out, from each one's own docs. |
45
104
  | **What agents misread** | It flags vague `id`s, IDs no tool returns, instructions buried late in descriptions, and writes without annotations or marked read-only. |
46
105
  | **Words routing to the wrong tool** | `routes.yml` pins which tool each keyword must (and must not) reach, the way a keyword or BM25 tool search sees the menu. A description edit that lets `get_audit_trail` win "audit" fails the check, with no model and no eval budget. [More](#snapshot-rules) |
47
106
  | **Menus that drift** | It catches defaults built from the clock or a hash order, menus that differ per process, per connection or offline, and listings that depend on who asks. |
48
107
 
49
- Every finding says what's wrong, where it is and what toolmenu saw, and ends with a `→ Next:` step.
50
- It runs on the 22-server [bench](bench/README.md) and on hosted servers (GitHub, Atlassian, Sentry,
51
- Notion, Supabase, Vercel…). What it has found on real servers is in
52
- [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
53
-
54
- > **Status: 0.25.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
55
- > every rule in [docs/RULES.md](docs/RULES.md), changes in [CHANGELOG.md](CHANGELOG.md).
56
-
57
- <details>
58
- <summary><b>Why "when it changes" matters</b></summary>
108
+ ## Why "when it changes" matters
59
109
 
60
110
  The MCP linters I found grade one snapshot of the menu: tool names, descriptions,
61
111
  schemas. toolmenu also checks **when** the menu changes:
@@ -77,10 +127,47 @@ schemas. toolmenu also checks **when** the menu changes:
77
127
  old versions with today's dependencies, so it records what they serve now
78
128
  (FINDINGS F4).
79
129
 
80
- Run against the reference server, `@modelcontextprotocol/server-everything`, it
81
- reports no false findings ([bench/](bench/README.md)).
130
+ ## Tested on real servers
131
+
132
+ Every rule change is run against real MCP servers before it ships, and every finding
133
+ that changes on one of them is read by hand; a change has to say why in the
134
+ [changelog](CHANGELOG.md). The corpus is pinned to exact versions, so a rerun checks the
135
+ same code ([bench/README.md](bench/README.md) has the commands):
82
136
 
83
- </details>
137
+ | Corpus | Servers | Checked with |
138
+ |---|---|---|
139
+ | The reference server, `@modelcontextprotocol/server-everything` 2.1.0 (2026-07-28) | 1 | `snapshot --lists`, `session --auto`, and a scenario that answers its elicitation demos. **No false findings**: a change that makes one appear is wrong until shown otherwise |
140
+ | Recent stdio servers from the MCP registry that start without credentials | 86 | `snapshot` |
141
+ | The servers across the stdio lists (registry, popular npm, PyPI) that answer the 2026-07-28 handshake | 39 | `session --auto` |
142
+ | Named servers run in a container: GitHub, Atlassian, Sentry, Notion, Supabase, Playwright, MongoDB… ([`bench/servers.txt`](bench/servers.txt)) | 22 | `snapshot`, `session --init`, `session` |
143
+ | Hosted (Streamable HTTP) servers from the registry that need no headers, sampled with a fixed seed | 200 | `snapshot`, read-only `session --auto --max-calls 10` |
144
+ | Hosted servers behind OAuth: Sentry, GitHub, Atlassian (F14) | 3 | `auth login`, then `snapshot` |
145
+ | **Weekly:** Docker's MCP catalog, every local server at the digest the catalog pins, with a dummy value for each setting it declares (F20) | 237 | `snapshot`, `session --auto`, and the sealed pass: 4,117 of the 4,237 tools listed called |
146
+ | **Weekly:** stdio servers from the registry, npm and PyPI (F17, F18); each week rechecks up to 300 new releases and newly listed servers | 1,090 at baseline | `snapshot`, `session --auto`, and a sealed `--call-everything` pass: 15,550 of 16,518 tools called, every one of the rest accounted for |
147
+ | The servers big vendors publish to run locally: GitHub, AWS Labs, Microsoft, PayPal, Stripe, Atlassian, Notion, HashiCorp, Grafana… as npm, PyPI and container images, with dummy credentials (F19) | 98 | `snapshot`, `session --auto`, and the sealed pass: 2,213 of the 2,292 tools listed called |
148
+ | **Weekly:** hosted servers, 109 vendor endpoints (GitHub, Google, Microsoft, Atlassian, Stripe…) and a seeded 2,000 of the registry's | 2,109 | `snapshot`, read-only `session --auto`, and `auth check` on each that answers 401 |
149
+
150
+ What those runs found is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md),
151
+ including what toolmenu itself got wrong and how it was fixed. A few:
152
+
153
+ - **mcp-atlassian serves a different menu every time it starts**: a default built from a
154
+ Python set, in a new order each process, so every restart misses the cache (F12).
155
+ - **Atlassian Rovo served 4 tools to any `Authorization` header**, valid or not, so a
156
+ client whose token expired gets fewer tools instead of a reason to log in again (`auth/invalid-accepted`).
157
+ - **The official filesystem server has reported `0.2.0` for 19 releases**, so a
158
+ server's self-reported version can't be used for semver checks (F5).
159
+ - **Notion's server ships the same 9 unused `$defs` in all 24 tools**: 72% of its menu.
160
+ - **12 of 1,090 stdio servers fail in the official SDK client today** (8 lose every tool,
161
+ 4 lose answers), and 22 of 2,000 hosted ones send a 2026-07-28 tool list that client
162
+ rejects (F18).
163
+ - **PayPal's server builds a new menu on every call**: `list_transactions` defaults its dates to
164
+ the moment the list is built, so no cached prompt matches the next one (F19).
165
+ - **Microsoft Sentinel's sign-in can't pass a client that follows MCP 2025-11-25 to the
166
+ letter**: Entra ID supports PKCE but its OpenID Connect metadata doesn't say so, and the
167
+ spec tells clients to refuse (F18).
168
+
169
+ The weekly run compares each week with the last: menus that changed, servers that fixed
170
+ or gained a finding, and the trend ([bench/results/weekly](bench/results/weekly)).
84
171
 
85
172
  ## Start here
86
173
 
@@ -113,6 +200,7 @@ By hand:
113
200
  ```sh
114
201
  npx toolmenu snapshot -- node dist/server.js # the menu, its findings, menu.json
115
202
  npx toolmenu diff menu.json new-menu.json # what changed between two releases
203
+ npx toolmenu review menu.json new-menu.json # should the new menu replace the baseline?
116
204
  npx toolmenu session --auto -- node dist/server.js # call read-only tools, watch the menu
117
205
  ```
118
206
 
@@ -126,6 +214,12 @@ npx toolmenu auth login https://mcp.example.com/mcp
126
214
  npx toolmenu snapshot https://mcp.example.com/mcp
127
215
  ```
128
216
 
217
+ Before logging in, `toolmenu auth check https://mcp.example.com/mcp` walks the server's
218
+ OAuth discovery the way a client that follows the spec does: the 401 and its
219
+ `WWW-Authenticate`, the protected resource metadata (and whether its `resource` is the
220
+ server's URL), the authorization server's metadata, PKCE and registration. No login, no
221
+ tokens: only the requests a client makes first.
222
+
129
223
  Logins are stored per server in `~/.config/toolmenu/auth` (readable only by you);
130
224
  `auth list` and `auth logout <url>` manage them. A server that doesn't allow
131
225
  dynamic registration needs a pre-registered app: `--client-id` (and
@@ -136,22 +230,22 @@ CI, where nobody can sign in, pass a token as a header.
136
230
  Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
137
231
  and the 2025 `initialize` handshake, through the official TypeScript SDK.
138
232
 
139
- Every finding and error says what's wrong, where, what toolmenu saw, and ends in a
140
- `→ Next:` step. Findings marked `· unsure` are heuristics (names and words): they say
141
- what was seen, not what it means. `toolmenu explain <rule>` says what a rule checks,
142
- its default level and how to change it (every rule is in [docs/RULES.md](docs/RULES.md)).
143
- Every command has its own help: `toolmenu history --help`.
233
+ How to read a finding is [above](#what-a-finding-looks-like). `toolmenu explain <rule>`
234
+ says what a rule checks, its default level and how to change it (every rule is in
235
+ [docs/RULES.md](docs/RULES.md)). Every command has its own help: `toolmenu history --help`.
144
236
 
145
237
  Behind a proxy (`HTTPS_PROXY`), stdio servers get the proxy and CA settings
146
238
  (`HTTPS_PROXY`, `NO_PROXY`, `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`…)
147
239
  along with the minimal environment, and `NODE_USE_ENV_PROXY=1` so a Node server's
148
240
  `fetch` uses the proxy. Without them a server that calls an API can't reach it, and
149
- fails as if it were broken.
241
+ fails as if it were broken. A proxy URL with a password in it
242
+ (`https://user:pass@proxy:8080`) reaches the server that way too: give a server you
243
+ don't trust a proxy without one, or `--env HTTPS_PROXY=` to leave it out.
150
244
 
151
245
  ### Every server you use: `scan`
152
246
 
153
247
  ```sh
154
- npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json
248
+ npx toolmenu scan ~/Library/Application\ Support/Claude/claude_desktop_config.json --allow-env LINEAR_API_KEY
155
249
  npx toolmenu scan .mcp.json --format markdown
156
250
  ```
157
251
 
@@ -168,6 +262,13 @@ findings, like this:
168
262
  linear – – not checked: LINEAR_API_KEY is not set (the config refers to it)
169
263
  ```
170
264
 
265
+ `scan` starts every command the list names, so scan only lists you'd run yourself (not
266
+ a `.mcp.json` from a repository you haven't read). A config's `${VAR}` is filled only
267
+ from the variables you name with `--allow-env` (comma-separated, or `'*'` for all): the
268
+ config decides where each value goes, a URL or a header included, so one from someone
269
+ else could otherwise send your environment to a server of theirs. An entry that uses
270
+ a variable you didn't allow isn't checked, and says which `--allow-env` it needs.
271
+
171
272
  `${VAR}`, `${VAR:-default}` and `${env:VAR}` in the config are filled from the
172
273
  environment. A server whose variable isn't set isn't started with an empty
173
274
  credential: it's a row that says so. A server that couldn't be started or read
@@ -246,7 +347,7 @@ jobs:
246
347
  - uses: actions/setup-node@v7
247
348
  with: { node-version: 22 }
248
349
  - run: npm ci && npm run build
249
- - uses: niksa90/toolmenu@v0.25.2
350
+ - uses: niksa90/toolmenu@v0.27.0
250
351
  with:
251
352
  command: node dist/server.js
252
353
  baseline: menu.json # your committed snapshot
@@ -255,7 +356,8 @@ jobs:
255
356
 
256
357
  It snapshots the PR's build, diffs it against the baseline as it is on the PR's base
257
358
  branch (or at the previous tag, on a tag push), runs the scenario, and
258
- posts **one comment that updates on every push**: breaking changes, the token change
359
+ posts **one comment that updates on every push**: first the verdict (can this menu
360
+ replace the baseline as it is?), then breaking changes, the token change
259
361
  ("this PR adds ~1,240 tokens to every conversation"), the suggested version bump,
260
362
  and anything `session` caught. The same report goes to the job summary.
261
363
 
@@ -270,6 +372,7 @@ and anything `session` caught. The same report goes to the job summary.
270
372
  | `baseline-from` | `snapshot` | `session` diffs every tool the scenario saw (the union menu), for servers whose tools appear after an unlock |
271
373
  | `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
272
374
  | `fail-on` | `error` | Fail the job on findings at or above this level |
375
+ | `gate` | `findings` | `review` passes or fails the job on the [review verdict](#accepting-a-new-baseline-review) instead of `diff`'s findings: a breaking change the release bump allows passes, new findings at `fail-on` fail ([more](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#the-verdict-and-gate-review)) |
273
376
  | `comment` | `true` | Post and update the PR comment |
274
377
  | `sarif` | | A folder to also write the findings to as SARIF, for code scanning (below) |
275
378
  | `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
@@ -294,7 +397,7 @@ permissions:
294
397
  security-events: write # to upload SARIF
295
398
  steps:
296
399
  # … checkout, build, as above
297
- - uses: niksa90/toolmenu@v0.25.2
400
+ - uses: niksa90/toolmenu@v0.27.0
298
401
  with:
299
402
  command: node dist/server.js
300
403
  sarif: toolmenu-sarif
@@ -334,6 +437,24 @@ toolmenu:
334
437
  junit: [snapshot.xml, diff.xml]
335
438
  ```
336
439
 
440
+ ### A report to keep: HTML
441
+
442
+ `--html <path>` writes the run of `snapshot`, `diff` or `session` as one HTML page
443
+ beside the usual report: the numbers that matter at the top (errors, warnings, tools,
444
+ tokens; for `diff`, breaking changes and the suggested bump), the menu as a table in
445
+ server order with each tool's share of the tokens, and the findings with their next
446
+ steps. `diff` shows each tool before and after, new and removed tools marked. `session`
447
+ has a row per step with the menu after it, tools inserted, appended or edited marked,
448
+ and links from a step to its findings.
449
+
450
+ The page has no script and loads nothing, so it opens offline: keep it as a CI artifact
451
+ (GitLab `artifacts: paths`, `actions/upload-artifact`) or attach it to an issue for a
452
+ server's maintainer. It follows the viewer's light or dark setting.
453
+
454
+ ```sh
455
+ npx toolmenu diff menu.json new-menu.json --html diff.html
456
+ ```
457
+
337
458
  ### A badge for your README
338
459
 
339
460
  ```sh
@@ -355,6 +476,22 @@ PRs without secrets are skipped, and a skipped check still passes; the `skipped`
355
476
  `session` runs a scripted session (the scenario *is* the agent), lists the menu
356
477
  after every step, and reports each change with the step that caused it.
357
478
 
479
+ ```
480
+ step the scenario (the agent) the menu the client sees toolmenu
481
+ ──── ─────────────────────────────── ─────────────────────────────── ─────────────────────────────
482
+ 1 list search_forms, get_form, … baseline: 5 tools
483
+ 2 call search_forms (unchanged) no change
484
+ 3 call unlock_toolset audits search_forms, list_team_audits, +2 tools at position 1:
485
+ get_team_audit, get_form, … session/mid-insert (error)
486
+ fresh process: not there →
487
+ session/side-effect (warn)
488
+ ```
489
+
490
+ Each step's menu is compared with the one before it, so a change is pinned to the call
491
+ that caused it, with where in the list it starts and what it costs the prompt cache.
492
+ toolmenu then starts a second process (or opens a second connection) to see whether
493
+ the change is global, or only this session's, which 2026-07-28 rules out.
494
+
358
495
  ```yaml
359
496
  # scenario.yml
360
497
  allow_writes: false # refuse to call tools not marked readOnlyHint
@@ -404,6 +541,7 @@ serves, and 23 issue and pull-request tools that only the dynamic mode has.
404
541
  | `session/error-not-flagged` | warn | A tool answered with an HTTP error from the API behind it (`{"status":401,"object":"error",…}`) as a plain result, not marked `isError`, so a client that reads `isError` takes the call for a success (Notion's server). An auth one also counts toward `session/untested`. Tools that take a URL are left out: their answer is the page's (heuristic) |
405
542
  | `session/server-unresponsive` | error | The server stopped answering after a step: even `tools/list` timed out. The run stops there with this one finding, instead of every later step waiting out its timeout (a server that polls a device-code sign-in inside a tool call) |
406
543
  | `session/handle-roundtrip` | warn | With `--create-handles`: the tool that creates a session, basket or cart was called, and the tools that take it back refused the handle it had just returned (info when its result had no handle to find) |
544
+ | `session/output-schema` | error | A tool that declares an `outputSchema` answered with no `structuredContent`, or one that doesn't match it: from 2025-06-18 a **MUST**, and the official SDK client raises an error instead of the answer |
407
545
  | `session/rate-limited` | error | The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it |
408
546
  | `session/scope-unchecked` | info | A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run |
409
547
 
@@ -436,6 +574,14 @@ with why each tool was left out, not a clean pass. On the 22 servers in `bench/`
436
574
  **`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
437
575
  not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
438
576
 
577
+ **`--call-everything`** (with `--auto`, stdio only) calls **every** tool once, writes and
578
+ open-world ones included, with a made-up value (`toolmenu-test`, `1`, a one-item list)
579
+ where the schema gives none. It's for a server in a sandbox with nothing to lose: a
580
+ container with no credentials and no network, as `bench/scripts/batch-full.sh` runs
581
+ them with `SEALED=1`. Every tool's answer is then checked: results against their own
582
+ `outputSchema`, errors flagged as errors, calls that hang. It's refused over HTTP,
583
+ because a hosted server's tools act on someone's real account.
584
+
439
585
  ## `diff`: compare releases
440
586
 
441
587
  ```sh
@@ -514,6 +660,40 @@ with `--server-version-is-release`. `diff` also enforces an optional `tokenBudge
514
660
  (`diff/token-budget`). Order changes are only a notice here: between releases, prompt caches rebuild. Order
515
661
  matters *within* a session.
516
662
 
663
+ ### Accepting a new baseline: `review`
664
+
665
+ `diff` says what changed. Before you commit a refreshed `menu.json`, `review` answers
666
+ the question that follows: should it replace the baseline?
667
+
668
+ ```sh
669
+ npx toolmenu snapshot --out new-menu.json -- node dist/server.js
670
+ npx toolmenu review --release 1.4.0..1.5.0 menu.json new-menu.json
671
+ ```
672
+
673
+ ```
674
+ toolmenu review new-menu.json as the new menu.json
675
+ verdict ✗ don't accept as is: 1 breaking change, and the release's minor bump is too small (release 2.0.0).
676
+ changes 1 breaking (1 tool) · 0 minor · 3 notice (13 tools)
677
+ findings 0 new · 0 gone · 1 changed · 1 unchanged
678
+ ```
679
+
680
+ It runs `diff`, and runs the menu rules on both files, so it also says which findings
681
+ the new menu **adds**, which it **clears**, and which **changed** (a summary that now
682
+ names another tool counts as worse). The verdict is one of:
683
+
684
+ | Verdict | Exit | When |
685
+ |---|---|---|
686
+ | `unchanged` | 0 | The same menu and the same findings |
687
+ | `safe` | 0 | Nothing breaks callers, and no finding at or above `--fail-on` is new or worse |
688
+ | `breaking-released` | 0 | Breaking changes, and `--release` shows a bump that allows them |
689
+ | `breaking` | 1 | Breaking changes, and no release that allows them: none given, too small, or not checkable (calendar versions) |
690
+ | `review` | 1 | Nothing breaks, but findings at or above `--fail-on` are new or worse, or `diff`'s own checks fail (`tokenBudget`) |
691
+
692
+ It reads only the two files, so rules that need the live server (a second process,
693
+ the raw `tools/list`) aren't part of it; `snapshot` runs those. `--format markdown`
694
+ gives the same as a PR comment, `--json` for scripts (`verdict`, `accept`, and the
695
+ findings by kind).
696
+
517
697
  ### Operations behind a search tool (`--catalog`)
518
698
 
519
699
  Big servers increasingly keep most operations out of the menu: a small fixed menu, a
@@ -685,6 +865,26 @@ client doesn't document is left out, not guessed.
685
865
  | `claude-code` | tool names; full definitions for tools marked `_meta["anthropic/alwaysLoad"]` | reads a description up to 4,096 characters | 16,384 through search, 4,096 up front | Claude API's |
686
866
  | `claude-api-search` | nothing from tools you defer (keep the 3–5 used most loaded) | name, description, argument names and descriptions; 5 results by default | none documented | `^[a-zA-Z0-9_-]{1,128}$` |
687
867
  | `openai` | each deferred function's name and description (its schema loads on demand); a deferred MCP server shows only its own name and description | how many results, and which fields, isn't documented, so `search/findable` doesn't run | none documented | `^[a-zA-Z0-9_-]+$`, up to 128 |
868
+ | `codex` | not documented | not documented | none documented | OpenAI's (it sends tools to the Responses API) |
869
+ | `antigravity` | each tool's name; the agent reads a tool's schema before calling it, unless the user marks it `"eager"` | not a search: the agent reads the schema file | none documented | not documented |
870
+ | `gemini-cli` | not documented | not documented | none documented | rewrites: characters outside letters, digits, `_ - . :` become `_`, names over 63 characters are cut in the middle |
871
+ | `gemini-api` | not documented | not documented | none documented | letters, digits, `_ - . :`, up to 128 |
872
+
873
+ Three more check what a client limits rather than what it loads:
874
+
875
+ | `--client` | What it checks |
876
+ |---|---|
877
+ | `vscode` | VS Code (GitHub Copilot Chat) sends at most 128 tools a request: a server over it alone is `menu/client-limit` |
878
+ | `copilot-cloud-agent` | uses MCP tools only, not prompts or resources (`menu/client-features`, with `--lists`) |
879
+ | `copilot-code-review` | the same, and only tools marked `readOnlyHint: true`: the rest are listed |
880
+
881
+ Read, but nothing in them toolmenu can check yet: Copilot CLI defers tools behind a
882
+ search at about 30 (it searches names, descriptions and parameters; how many results
883
+ isn't said); Copilot in JetBrains, Visual Studio, Eclipse and Xcode documents no tool
884
+ limit; JetBrains AI Assistant and Junie document none either (Junie CLI times out any
885
+ request after 60 s, startup and `tools/list` included). Not here, because their docs
886
+ don't state it: Cursor's tool limit (forum posts say 40), Windsurf's (100 in older docs,
887
+ not in the current page), ChatGPT's. A value comes in when the client documents it.
688
888
 
689
889
  The snapshot adds a line such as `as Claude Code: up front ~524 tokens · on demand ~344
690
890
  per tool a search loads (largest: update_contacts, ~2,093)`, and `--json` a `client`
@@ -715,6 +915,7 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
715
915
  --format <fmt> text (default), json, github (annotations), markdown, sarif
716
916
  (code scanning) or junit (CI test reports)
717
917
  --junit <path> snapshot, diff, session: also write the findings as JUnit XML
918
+ --html <path> snapshot, diff, session: also write the report as one HTML page
718
919
  --fail-on <level> error (default), warn or info
719
920
  --header <k: v> HTTP header, repeatable
720
921
  --env <K=V> env var for a stdio server, repeatable (it only gets a
@@ -727,18 +928,36 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
727
928
  --lists <kinds> snapshot: also read and check prompts, resources (comma-separated);
728
929
  saved in the menu file beside the tools
729
930
  --client <name> snapshot: check the menu for one client, from its own docs:
730
- claude-code, claude-api-search or openai
931
+ claude-code, claude-api-search, openai, codex, vscode,
932
+ copilot-cloud-agent, copilot-code-review, gemini-cli,
933
+ gemini-api or antigravity
731
934
  --as <name>:<cred> snapshot: list the menu as this identity too, repeatable:
732
935
  "reader:Authorization: Bearer …" (HTTP) or "admin:ROLE=admin"
733
936
  (stdio). The first is the menu saved; the others are compared
734
937
  --offline-check snapshot (stdio): also list the menu with the network cut off
735
938
  --concurrency <n> scan: servers checked at a time (default: 4)
736
939
  --out-dir <dir> scan: write each server's menu file there, as <name>.json
940
+ --allow-env <names> scan: variables a config's ${VAR} may be filled from (default:
941
+ none; '*' for all)
737
942
  ```
738
943
 
739
944
  Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
740
945
  (for `scan`: a server wasn't checked) or bad usage. `toolmenu <command> --help` lists that command's own options.
741
946
 
947
+ Credentials on the command line (`--header`, `--env`, `--client-secret`) can be read by
948
+ every user of the machine while toolmenu runs (`ps`). To keep them off it, put the
949
+ options in `TOOLMENU_ARGS`, a JSON array of toolmenu options and their values; they go
950
+ before the server command's `--`. A process's environment is readable only by its own
951
+ user (and root), so this keeps them from other users, not from other processes of
952
+ yours. jq builds the array whatever the token holds:
953
+
954
+ ```sh
955
+ TOOLMENU_ARGS=$(jq -cn --arg t "$TOKEN" '["--header", "Authorization: Bearer " + $t]') npx toolmenu snapshot https://example.com/mcp
956
+ ```
957
+
958
+ The GitHub Action passes its `headers` and `env` this way (to toolmenu 0.26.0 and
959
+ later; an older `version` gets them as arguments, with a warning).
960
+
742
961
  ## Limitations
743
962
 
744
963
  - **Server order isn't final client order.** Clients merge tools from several servers
package/dist/args.d.ts CHANGED
@@ -19,6 +19,16 @@ export declare function fits(value: unknown, schema: JsonSchema | undefined): bo
19
19
  * tool that needs one is skipped, not called with junk.
20
20
  */
21
21
  export declare function synthesize(schema: JsonSchema | undefined, name?: string): Synthesized;
22
+ /** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
23
+ export declare const INVENTED = "toolmenu-test";
24
+ /**
25
+ * A value of the right type where the schema itself vouches for none, for
26
+ * `session --auto --call-everything`: the schema's own value first (synthesize),
27
+ * then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
28
+ * required fields). A pattern it can't satisfy gets the made-up string anyway: the
29
+ * tool's answer to a value it refuses is part of what's checked.
30
+ */
31
+ export declare function invent(schema: JsonSchema | undefined, name?: string, depth?: number): unknown;
22
32
  /**
23
33
  * A value the user gave (--value), made to fit the parameter: `5` is the number 5
24
34
  * for an integer and the string "5" for a string; JSON that doesn't fit the type
@@ -29,7 +39,7 @@ export declare function fitValue(given: {
29
39
  value: unknown;
30
40
  }, schema: JsonSchema | undefined): unknown;
31
41
  /** Where a tool's argument came from: the user (--value) or the schema. */
32
- export type ArgSource = 'value' | 'schema';
42
+ export type ArgSource = 'value' | 'schema' | 'invented';
33
43
  /**
34
44
  * Arguments for a tool's required parameters, or the ones it can't fill. `given`:
35
45
  * values the user supplied for this tool, by parameter name; they come first, and
@@ -38,7 +48,9 @@ export type ArgSource = 'value' | 'schema';
38
48
  export declare function synthesizeArgs(schema: JsonSchema | undefined, given?: Record<string, {
39
49
  raw?: string;
40
50
  value: unknown;
41
- }>): {
51
+ }>, options?: {
52
+ invent?: boolean;
53
+ }): {
42
54
  ok: true;
43
55
  args: Record<string, unknown>;
44
56
  sources: Record<string, ArgSource>;
package/dist/args.js CHANGED
@@ -107,6 +107,45 @@ export function synthesize(schema, name = '') {
107
107
  return { ok: false };
108
108
  }
109
109
  }
110
+ /** Made-up values are marked as toolmenu's, so they're easy to find in a server's logs. */
111
+ export const INVENTED = 'toolmenu-test';
112
+ /**
113
+ * A value of the right type where the schema itself vouches for none, for
114
+ * `session --auto --call-everything`: the schema's own value first (synthesize),
115
+ * then a made-up one ("toolmenu-test", 1, a one-item list, an object of invented
116
+ * required fields). A pattern it can't satisfy gets the made-up string anyway: the
117
+ * tool's answer to a value it refuses is part of what's checked.
118
+ */
119
+ export function invent(schema, name = '', depth = 0) {
120
+ const own = synthesize(schema, name);
121
+ if (own.ok)
122
+ return own.value;
123
+ if (!schema || typeof schema !== 'object' || depth > 8)
124
+ return INVENTED;
125
+ const branches = (schema.anyOf ?? schema.oneOf);
126
+ const type = Array.isArray(schema.type) ? schema.type.find((t) => t !== 'null') : schema.type;
127
+ if (!type && Array.isArray(branches) && branches.length)
128
+ return invent(branches.find((b) => b && typeof b === 'object' && b.type !== 'null') ?? branches[0], name, depth + 1);
129
+ switch (type) {
130
+ case 'boolean':
131
+ return false;
132
+ case 'integer':
133
+ case 'number':
134
+ return 1;
135
+ case 'array':
136
+ return [invent(schema.items, name, depth + 1)];
137
+ case 'object': {
138
+ const out = {};
139
+ for (const p of schema.required ?? [])
140
+ out[p] = invent(schema.properties?.[p], p, depth + 1);
141
+ return out;
142
+ }
143
+ case 'null':
144
+ return null;
145
+ default:
146
+ return INVENTED;
147
+ }
148
+ }
110
149
  /**
111
150
  * A value the user gave (--value), made to fit the parameter: `5` is the number 5
112
151
  * for an integer and the string "5" for a string; JSON that doesn't fit the type
@@ -127,7 +166,7 @@ export function fitValue(given, schema) {
127
166
  * values the user supplied for this tool, by parameter name; they come first, and
128
167
  * one for an optional parameter is passed too.
129
168
  */
130
- export function synthesizeArgs(schema, given = {}) {
169
+ export function synthesizeArgs(schema, given = {}, options = {}) {
131
170
  const args = {};
132
171
  const sources = {};
133
172
  const missing = [];
@@ -143,6 +182,10 @@ export function synthesizeArgs(schema, given = {}) {
143
182
  args[param] = v.value;
144
183
  sources[param] = 'schema';
145
184
  }
185
+ else if (options.invent) {
186
+ args[param] = invent(schema?.properties?.[param], param);
187
+ sources[param] = 'invented';
188
+ }
146
189
  else
147
190
  missing.push(param);
148
191
  }
@@ -0,0 +1,27 @@
1
+ import type { Finding } from './types.js';
2
+ export interface AuthCheck {
3
+ url: string;
4
+ /** What the unauthenticated initialize got: the status, and the WWW-Authenticate header. */
5
+ status?: number;
6
+ wwwAuthenticate?: string;
7
+ /** Where the protected resource metadata was found, and how. */
8
+ metadataUrl?: string;
9
+ foundBy?: 'header' | 'well-known path' | 'well-known root';
10
+ resource?: string;
11
+ authorizationServers?: string[];
12
+ /** The first authorization server's metadata URL, and what it says. */
13
+ serverMetadataUrl?: string;
14
+ pkce?: string[];
15
+ registration?: 'dynamic' | 'client-id-metadata' | 'both' | 'none';
16
+ findings: Finding[];
17
+ }
18
+ /** `resource_metadata="…"` from a WWW-Authenticate header. */
19
+ export declare function resourceMetadataOf(header: string | undefined): string | undefined;
20
+ /** The well-known URIs RFC 9728 and the spec name, path-specific first. */
21
+ export declare function wellKnownResource(url: string): {
22
+ path: string;
23
+ root: string;
24
+ };
25
+ /** The spec's order for an authorization server's metadata: RFC 8414 then OpenID Connect, path-aware. */
26
+ export declare function serverMetadataUrls(issuer: string): string[];
27
+ export declare function authCheck(url: string, timeoutMs?: number): Promise<AuthCheck>;