toolmenu 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +311 -0
  3. package/dist/cli.d.ts +2 -0
  4. package/dist/cli.js +283 -0
  5. package/dist/compare.d.ts +32 -0
  6. package/dist/compare.js +108 -0
  7. package/dist/config.d.ts +19 -0
  8. package/dist/config.js +27 -0
  9. package/dist/connect.d.ts +66 -0
  10. package/dist/connect.js +122 -0
  11. package/dist/diff.d.ts +58 -0
  12. package/dist/diff.js +404 -0
  13. package/dist/history.d.ts +88 -0
  14. package/dist/history.js +219 -0
  15. package/dist/index.d.ts +10 -0
  16. package/dist/index.js +9 -0
  17. package/dist/menu.d.ts +9 -0
  18. package/dist/menu.js +57 -0
  19. package/dist/report.d.ts +11 -0
  20. package/dist/report.js +278 -0
  21. package/dist/routes.d.ts +22 -0
  22. package/dist/routes.js +76 -0
  23. package/dist/rules/description.d.ts +28 -0
  24. package/dist/rules/description.js +158 -0
  25. package/dist/rules/determinism.d.ts +2 -0
  26. package/dist/rules/determinism.js +35 -0
  27. package/dist/rules/ids.d.ts +11 -0
  28. package/dist/rules/ids.js +138 -0
  29. package/dist/rules/index.d.ts +11 -0
  30. package/dist/rules/index.js +42 -0
  31. package/dist/rules/naming.d.ts +4 -0
  32. package/dist/rules/naming.js +145 -0
  33. package/dist/rules/rule.d.ts +33 -0
  34. package/dist/rules/rule.js +1 -0
  35. package/dist/rules/spec.d.ts +5 -0
  36. package/dist/rules/spec.js +102 -0
  37. package/dist/rules/write.d.ts +3 -0
  38. package/dist/rules/write.js +55 -0
  39. package/dist/semver.d.ts +11 -0
  40. package/dist/semver.js +41 -0
  41. package/dist/session.d.ts +90 -0
  42. package/dist/session.js +519 -0
  43. package/dist/snapshot.d.ts +16 -0
  44. package/dist/snapshot.js +42 -0
  45. package/dist/types.d.ts +49 -0
  46. package/dist/types.js +1 -0
  47. package/dist/version.d.ts +2 -0
  48. package/dist/version.js +3 -0
  49. package/dist/words.d.ts +14 -0
  50. package/dist/words.js +63 -0
  51. package/package.json +57 -0
  52. package/schemas/2024-11-05.json +2077 -0
  53. package/schemas/2025-03-26.json +2139 -0
  54. package/schemas/2025-06-18.json +2517 -0
  55. package/schemas/2025-11-25.json +4058 -0
  56. package/schemas/2026-07-28.json +3963 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 niksa90
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,311 @@
1
+ # toolmenu
2
+
3
+ [![npm](https://img.shields.io/npm/v/toolmenu)](https://www.npmjs.com/package/toolmenu)
4
+ [![ci](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml/badge.svg)](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
5
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
6
+
7
+ **Lint your MCP server's tool menu for changes that confuse agents or break caches.**
8
+
9
+ The MCP linters I found grade one snapshot of the menu an agent reads: tool names,
10
+ descriptions, schemas. toolmenu observes how the menu behaves across calls,
11
+ connections and releases, and checks **when** it changes:
12
+
13
+ - **Mid-session.** A menu that changes after the conversation has started can
14
+ invalidate the cached conversation: most clients put the tool list at the start of
15
+ the prompt, so any change to it, even a tool added at the end, means everything
16
+ after it is processed again. In my logs, one mid-conversation unlock rewrote
17
+ 40–57K tokens of cached prefix (from my own logs; more on this, with a benchmark
18
+ of the setups, in [this write-up](https://niksa.me/agent-tools-design-lessons)). `session` catches it, pins it to the step that
19
+ caused it, and says where in the list it happened.
20
+ - **Ready for 2026-07-28.** That protocol version says the tool set **must not** vary
21
+ per connection or as a side effect of other requests (the 2025 versions allow it).
22
+ Per-session "unlock" designs, like the one I built, are ruled out. `session` tells you whether yours is one of them.
23
+ - **Between releases.** `diff` finds breaking changes and token growth, and checks
24
+ the version bump.
25
+
26
+ ![toolmenu session output: tools inserted mid-list, a description edited, each pinned to the step that caused it](https://raw.githubusercontent.com/niksa90/toolmenu/main/docs/demo.svg)
27
+
28
+ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
29
+ (`history` installs old versions with today's dependencies, so it records what they
30
+ serve now, which can differ from what they shipped with: FINDINGS F4.)
31
+
32
+ > **Status: 0.7, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
33
+ > real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
34
+
35
+ ## Start here
36
+
37
+ Needs Node 22 or later. The GitHub Action brings its own if the runner's Node is older,
38
+ for toolmenu only: your server and the job's later steps keep the job's Node.
39
+ Your server can be in any language: toolmenu starts it (stdio) or connects to it (HTTP).
40
+
41
+ ```sh
42
+ # write a starter scenario from your server's menu, then run it
43
+ npx toolmenu session --init -- node dist/server.js
44
+ npx toolmenu session --scenario scenario.yml -- node dist/server.js
45
+
46
+ # snapshot the menu and commit menu.json as your baseline
47
+ npx toolmenu snapshot -- node dist/server.js
48
+ ```
49
+
50
+ HTTP servers work the same way: `npx toolmenu snapshot https://example.com/mcp --header "Authorization: Bearer $TOKEN"`.
51
+ Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
52
+ and the 2025 `initialize` handshake, through the official TypeScript SDK.
53
+
54
+ ## In CI: one PR comment
55
+
56
+ ```yaml
57
+ # .github/workflows/toolmenu.yml
58
+ on: pull_request
59
+ permissions:
60
+ contents: read
61
+ pull-requests: write
62
+ jobs:
63
+ toolmenu:
64
+ runs-on: ubuntu-latest
65
+ steps:
66
+ - uses: actions/checkout@v7
67
+ - uses: actions/setup-node@v7
68
+ with: { node-version: 22 }
69
+ - run: npm ci && npm run build
70
+ - uses: niksa90/toolmenu@v0.7.0
71
+ with:
72
+ command: node dist/server.js
73
+ baseline: menu.json # your committed snapshot
74
+ scenario: scenario.yml # optional: run a session too
75
+ ```
76
+
77
+ It snapshots the PR's build, diffs it against the baseline as it is on the PR's base
78
+ branch (or at the previous tag, on a tag push), runs the scenario, and
79
+ posts **one comment that updates on every push**: breaking changes, the token change
80
+ ("this PR adds ~1,240 tokens to every conversation"), the suggested version bump,
81
+ and anything `session` caught. The same report goes to the job summary.
82
+
83
+ | Input | Default | |
84
+ |---|---|---|
85
+ | `command` or `url` | | How to start the server over stdio, or its Streamable HTTP endpoint |
86
+ | `headers` | | HTTP headers for a `url` server, one `Name: value` per line. Use a secret for keys: `x-api-key: ${{ secrets.MCP_API_KEY }}` |
87
+ | `env` | | Environment variables for a `command` server, one `KEY=value` per line |
88
+ | `baseline` | `menu.json` | The committed snapshot to diff against |
89
+ | `scenario` | | A scenario to run with `session` |
90
+ | `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
91
+ | `fail-on` | `error` | Fail the job on findings at or above this level |
92
+ | `comment` | `true` | Post and update the PR comment |
93
+ | `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
94
+
95
+ Server behind auth, or PRs from forks and Dependabot? See [docs/github-action.md](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md).
96
+ PRs without secrets are skipped, and a skipped check still passes; the `skipped` output tells you.
97
+
98
+ ## `session`: the menu changing while the agent works
99
+
100
+ `session` runs a scripted session (the scenario *is* the agent), lists the menu
101
+ after every step, and reports each change with the step that caused it.
102
+
103
+ ```yaml
104
+ # scenario.yml
105
+ allow_writes: false # refuse to call tools not marked readOnlyHint
106
+ steps:
107
+ - list
108
+ - call: search_forms
109
+ args: { query: onboarding }
110
+ - call: unlock_toolset
111
+ args: { toolset: audits }
112
+ - wait_for: tools_list_changed # optional; timeout_ms defaults to 5000
113
+ ```
114
+
115
+ `--init` writes a starter scenario from the live menu: it calls every read-only tool
116
+ that needs no arguments, then repeats one. If a tool looks like it unlocks more tools
117
+ (a `domains` or `toolset` parameter with an enum, "unlock" or "capabilities" in its
118
+ name or description), the starter unlocks two values for real and repeats the first.
119
+ Anything it can't fill in is left as a commented-out step. A scenario is only as good
120
+ as its steps, so the starter is a floor, not a ceiling.
121
+
122
+ | Rule | Default | Catches |
123
+ |---|---|---|
124
+ | `session/mid-insert`, `session/reorder`, `session/remove`, `session/edit` | error | The menu changing mid-session anywhere but the end, with the estimated tokens of the tool list from the change on: a floor, since the conversation after the tool list is processed again too |
125
+ | `session/append` | warn | Tools added at the end of the list. Still a cache miss for the conversation when your client sends tools at the start of the prompt (Claude's Messages API does); cache-safe only when the client adds new tools after the cached content, as tool search (deferred loading) does |
126
+ | `session/connection-local` | error on 2026-07-28 (HTTP) | A change only this connection sees. 2026-07-28: the tool set MUST NOT vary "per-connection or as a side effect of other requests on the connection" |
127
+ | `session/side-effect` | warn on 2026-07-28 (stdio) | The same, where stdio can't tell per-connection from global |
128
+ | `session/connection-variance` | error/warn | A second connection with the same credentials gets a different menu |
129
+ | `session/unannounced` | warn | The menu changed without `notifications/tools/list_changed`, although the server declared `listChanged` |
130
+ | `session/refused`, `session/step-failed` | error | A write the scenario didn't allow, or a call that failed |
131
+ | `session/tool-error` | warn | A tool that answered with an error (`isError`, an expired token): the run tested less than it looks |
132
+
133
+ **`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
134
+ not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
135
+
136
+ ## `diff`: compare releases
137
+
138
+ ```sh
139
+ npx toolmenu diff menu.json new-menu.json
140
+ ```
141
+
142
+ ```
143
+ $ npx toolmenu diff --release 2026.1.14..2026.8.31 fs-2026.1.14.json fs-2026.8.31.json
144
+ toolmenu diff secure-filesystem-server 2026.1.14 → 2026.8.31
145
+ 14 → 14 tools · ~2,638 → ~2,821 tokens (+183, estimate): this release adds ~183 tokens to every conversation that loads the menu
146
+ 1 breaking · 0 minor · 15 notice · suggested bump: major · actual: not checked (calendar version)
147
+
148
+ ERROR diff/safety-hint
149
+ move_file was additive-only and is now destructive.
150
+ …
151
+ ```
152
+
153
+ That's two real releases of the official filesystem server. Its calendar versions
154
+ promise nothing about compatibility, so the bump isn't judged, but the breaking change
155
+ is still an error. Changes are classified
156
+ the way OpenAPI breaking-change checkers do it:
157
+
158
+ | Class | Changes | Rules |
159
+ |---|---|---|
160
+ | **breaking** (error) | tool removed or renamed, parameter removed, new required parameter, narrower or different type, enum narrowed, a tool becoming less safe (no longer read-only, or additive → destructive) | `diff/tool-removed`, `diff/tool-renamed`, `diff/param-removed`, `diff/param-required`, `diff/param-type`, `diff/enum-narrowed`, `diff/safety-hint` |
161
+ | **minor** (info) | tool added, new optional parameter, parameter now optional, type or enum widened | `diff/tool-added`, `diff/param-added`, `diff/param-relaxed`, `diff/type-widened`, `diff/enum-widened` |
162
+ | **notice** (info) | optional parameter dropped (extra properties still allowed), description changed (with the text diff: it changes what the agent does), output schema, annotations, other fields, order | `diff/param-dropped`, `diff/description`, `diff/schema-other`, `diff/annotations`, `diff/other`, `diff/order` |
163
+
164
+ It also reports the token change per tool and suggests a semver bump. To check the
165
+ bump, pass the **release** versions: `--release 1.4.0..1.5.0` (npm, a git tag).
166
+ `diff/version-bump` then warns when the release's bump is smaller (`0.0.x` promises
167
+ nothing and `0.x` may break in a minor), and `diff/version-backwards` when the version
168
+ goes down. Calendar versions and prereleases aren't judged. The version in the snapshot is what the
169
+ server reports (`serverInfo.version`), which is often not the release: the filesystem
170
+ server has said `0.2.0` for 19 releases (FINDINGS F5). It's shown, and only checked
171
+ with `--server-version-is-release`. `diff` also enforces an optional `tokenBudget`
172
+ (`diff/token-budget`). Order changes are only a notice here: between releases, prompt caches rebuild. Order
173
+ matters *within* a session.
174
+
175
+ ## `history`: a package's releases, researched
176
+
177
+ ```sh
178
+ npx toolmenu history @modelcontextprotocol/server-filesystem --versions 19 --arg /tmp --csv fs.csv
179
+ ```
180
+
181
+ It installs each published version into a temp directory (install scripts off),
182
+ snapshots it, and diffs it against the previous version that worked. For every
183
+ version it records the protocol, the menu, the findings and the **dependency
184
+ versions that actually got installed**: an old version installed today resolves its
185
+ dependencies today, and can serve a different menu than it shipped with.
186
+
187
+ It's best effort: packages that need env vars, arguments or auth are recorded as
188
+ failed, with a reason, never skipped. **It installs and runs third-party code. Run it
189
+ in a container.**
190
+
191
+ ## `snapshot` rules
192
+
193
+ Every rule comes from a failure I actually hit building 115 tools for one MCP
194
+ server, or from the spec itself. If a rule can't point to one, it doesn't ship.
195
+
196
+ | Rule | Default | Catches |
197
+ |---|---|---|
198
+ | `menu/nondeterministic` | error | Two identical `tools/list` calls returning different menus (order, descriptions, schemas, annotations) |
199
+ | `spec/schema` | error | A `tools/list` result that fails the official schema for its protocol version |
200
+ | `naming/route` | error | A `routes.yml` expectation broke: a keyword now matches the wrong tool at least as well as the right one |
201
+ | `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions: 2,048 characters in Claude Code, or your `descriptionLimit`. From a real failure: a client cut at 280 characters, and the line that decided routing was at 1,222 |
202
+ | `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary. The model gets a prefix that can read as complete |
203
+ | `naming/vague-id` | warn | A parameter called just `id` that doesn't say *which* thing |
204
+ | `ids/authored` | warn | An ID the agent must supply that no tool appears to return, so the agent may invent it (heuristic) |
205
+ | `write/unannotated` | warn | A tool named like a write (`delete_`, `send_`…) with no annotations |
206
+ | `naming/shared-word` | info | Nouns that name different things across tools (`list_team_audits` vs `get_audit_trail`), as one summary |
207
+ | `write/no-dry-run` | info | Destructive tools with no dry-run parameter or preview tool, as one summary |
208
+ | `spec/discover` | info | The server only speaks the 2025 protocol, so 2026-07-28 rules are skipped |
209
+ | `spec/deprecated`, `spec/cache-hints` | info | 2026-07-28: deprecated features still advertised; `ttlMs: 0`, or `cacheScope: "public"` on a server that took credentials |
210
+
211
+ The naming and ID rules are heuristics, tuned on real servers (FINDINGS F7 and
212
+ F10) and a private 115-tool one. They're the extras; the change checks are the point.
213
+
214
+ **Pin routing with `routes.yml`.** Two tools fighting over one word needs a regression
215
+ test, not a rename you'll forget:
216
+
217
+ ```yaml
218
+ audit:
219
+ must_match: [list_team_audits]
220
+ must_not_match: [get_audit_trail]
221
+ audit trail:
222
+ must_not_match: [get_store_scores] # on its own: must not match at all
223
+ ```
224
+
225
+ Matching is keyword matching, the way a keyword or BM25 tool search sees the menu: a
226
+ tool matches if every word of the route is in its name or description (the exact
227
+ phrase scores higher). It can't read "not", and neither can those searches: a
228
+ disclaimer like "not the audit trail" puts "audit trail" into exactly the tool you
229
+ meant to steer away from, and toolmenu says so.
230
+
231
+ **Config** (optional `toolmenu.config.json`):
232
+
233
+ ```json
234
+ {
235
+ "rules": { "naming/shared-word": "off", "ids/authored": "error" },
236
+ "ignore": ["debug_*"],
237
+ "routes": "routes.yml",
238
+ "tokenBudget": 4000,
239
+ "descriptionLimit": "claude-code",
240
+ "fullDescriptions": ["search_capabilities", "lookup_*"]
241
+ }
242
+ ```
243
+
244
+ `descriptionLimit` is where your client cuts tool descriptions: a number, or a client
245
+ whose cut has been reported: `"claude-code"` (2,048, the default: reported in
246
+ [anthropics/claude-code#87650](https://github.com/anthropics/claude-code/issues/87650);
247
+ the [Claude Code changelog](https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md)
248
+ lists `CLAUDE_CODE_MAX_MCP_DESCRIPTION_LENGTH` to change it from 2.1.280) or
249
+ `"amazon-q"` (10,024, reported in
250
+ [makenotion/notion-mcp-server#145](https://github.com/makenotion/notion-mcp-server/issues/145)).
251
+ Clients that send descriptions in full don't need a setting beyond that. If your own
252
+ client cuts shorter, as mine did at 280, set its number. `fullDescriptions` lists the
253
+ tools it sends uncut, so the description rules skip them.
254
+
255
+ ## What toolmenu can't catch
256
+
257
+ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
258
+
259
+ - **Routing to a feature with no tools.** If the agent reaches for something your
260
+ menu doesn't have, no menu check can see the gap.
261
+ - **What the client does with the menu.** Truncating descriptions, sorting or
262
+ filtering tools, merging several servers. The description rules use Claude Code's
263
+ reported cut unless you set yours (`descriptionLimit`); they can't see your client.
264
+ - **Your code and your logs.** A capability check that's answered with one yes/no
265
+ for two different behaviours, or a log line that reads the same whether something
266
+ happened or not. Those live in the client or the server's code.
267
+ - **Whether the agent actually picks the right tool.** That needs evals with a model.
268
+ toolmenu stays deterministic on purpose.
269
+
270
+ ## Options
271
+
272
+ ```
273
+ --out <path> where to write the menu (default: menu.json)
274
+ --no-write don't write the menu file
275
+ --routes <path> routes.yml
276
+ --config <path> config file (default: toolmenu.config.json, if present)
277
+ --format <fmt> text (default), json, github (annotations) or markdown
278
+ --fail-on <level> error (default), warn or info
279
+ --header <k: v> HTTP header, repeatable
280
+ --env <K=V> env var for a stdio server, repeatable (it only gets a
281
+ minimal environment otherwise)
282
+ --timeout <ms> per-request timeout (default: 30000)
283
+ ```
284
+
285
+ Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
286
+ or bad usage. `toolmenu --help` lists the `session` and `history` options.
287
+
288
+ ## Limitations
289
+
290
+ - **Server order isn't final client order.** Clients merge tools from several servers
291
+ and may sort or filter them. A stable server menu is necessary for cache hits, not
292
+ sufficient. toolmenu measures what the server controls.
293
+ - **Token counts are estimates** (`o200k_base`). Vendors tokenize, cache and bill
294
+ differently.
295
+ - **Heuristic rules say so** in their messages.
296
+ - Not a security scanner or a full conformance suite. Other tools do those well.
297
+
298
+ ## About
299
+
300
+ Built by [Niksa](https://niksa.me) while running a 115-tool MCP server and watching
301
+ agents trip over its menu. Every rule started as one of those failures, and the
302
+ research behind them is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
303
+
304
+ Found a false positive, or a failure toolmenu should catch?
305
+ [Open an issue](https://github.com/niksa90/toolmenu/issues) with the menu (a
306
+ `menu.json` is enough) and what you expected. Changes are listed in
307
+ [CHANGELOG.md](https://github.com/niksa90/toolmenu/blob/main/CHANGELOG.md).
308
+
309
+ ## License
310
+
311
+ MIT
package/dist/cli.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export declare function main(argv: string[]): Promise<number>;
package/dist/cli.js ADDED
@@ -0,0 +1,283 @@
1
+ #!/usr/bin/env node
2
+ import { writeFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ import { parseArgs } from 'node:util';
5
+ import { loadConfig } from './config.js';
6
+ import { diffMenus } from './diff.js';
7
+ import { loadMenu } from './menu.js';
8
+ import { history, historyCsv } from './history.js';
9
+ import { formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
10
+ import { loadScenario, session, starterScenario } from './session.js';
11
+ import { connect, listTools } from './connect.js';
12
+ import { buildMenu } from './menu.js';
13
+ import { existsSync } from 'node:fs';
14
+ import { loadRoutes } from './routes.js';
15
+ import { snapshot } from './snapshot.js';
16
+ import { SEVERITY_RANK } from './types.js';
17
+ import { VERSION } from './version.js';
18
+ const HELP = `toolmenu ${VERSION}
19
+ Lint your MCP server's tool menu for changes that confuse agents or break caches.
20
+
21
+ Usage:
22
+ toolmenu snapshot [options] -- <command> [args...] stdio server
23
+ toolmenu snapshot [options] <url> Streamable HTTP server
24
+ toolmenu diff [options] <old.json> <new.json> compare two snapshots
25
+ toolmenu history [options] <npm-package> snapshot and diff published versions
26
+ toolmenu session --scenario <file> [options] -- <command> | <url>
27
+ run a scripted session, watch the menu
28
+
29
+ Commands:
30
+ snapshot establish the menu: list tools twice, write menu.json, run the menu rules
31
+ diff compare releases: breaking changes, token change, semver bump
32
+ session observe the menu changing while a scripted session runs
33
+ history research release history: install, snapshot and diff published npm versions
34
+
35
+ Options:
36
+ --out <path> where to write the menu (default: menu.json)
37
+ --no-write don't write the menu file
38
+ --routes <path> routes.yml: pin which words route to which tools
39
+ --config <path> config file (default: toolmenu.config.json, if present)
40
+ --format <fmt> text (default), json, github (annotations) or markdown (PR comments)
41
+ --json same as --format json
42
+ --fail-on <level> error (default), warn or info
43
+ --header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
44
+ --env <K=V> environment variable for a stdio server, repeatable. The server
45
+ only gets a minimal environment (PATH, HOME, ...) plus these
46
+ --timeout <ms> per-request timeout (default: 30000)
47
+ -h, --help show this help
48
+ -v, --version show the version
49
+
50
+ diff options:
51
+ --release <old>..<new> the release versions (npm, git tag) to check the bump
52
+ against, e.g. 1.4.0..1.5.0. Without it, the bump isn't
53
+ checked: serverInfo.version is often not the release
54
+ --server-version-is-release check the bump against serverInfo.version instead
55
+
56
+ session options:
57
+ --scenario <path> scenario.yml: the steps to run (list, call, wait_for)
58
+ --plan print the steps without connecting or running anything
59
+ --init write a starter scenario from the server's menu (to --scenario,
60
+ default scenario.yml; never overwrites)
61
+
62
+ history options (best effort; installs and runs third-party code, so use a container):
63
+ --versions <n|all> number of published versions to inspect (default: 10)
64
+ --include-prereleases include versions like 1.2.0-beta.1
65
+ --arg <value> argument for the server's bin, repeatable (use --arg=--flag for flags)
66
+ --bin <name> which bin to run when the package has several
67
+ --cmd <template> command instead of the bin; {bin} and {dir} are filled in
68
+ --allow-scripts run install scripts (off by default)
69
+ --install-timeout <ms> per-version install timeout (default: 180000)
70
+ --csv <path> also write the dataset as CSV
71
+ --keep-installs keep each version's install directory
72
+ --out <dir> where menus and history.json go (default: toolmenu-history/<package>)
73
+
74
+ Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
75
+ `;
76
+ class UsageError extends Error {
77
+ }
78
+ export async function main(argv) {
79
+ const dash = argv.indexOf('--');
80
+ const before = dash === -1 ? argv : argv.slice(0, dash);
81
+ const command = dash === -1 ? undefined : argv.slice(dash + 1);
82
+ const { values, positionals } = parseArgs({
83
+ args: before,
84
+ allowPositionals: true,
85
+ options: {
86
+ out: { type: 'string' },
87
+ 'no-write': { type: 'boolean' },
88
+ routes: { type: 'string' },
89
+ config: { type: 'string' },
90
+ format: { type: 'string' },
91
+ json: { type: 'boolean' },
92
+ 'fail-on': { type: 'string' },
93
+ header: { type: 'string', multiple: true },
94
+ env: { type: 'string', multiple: true },
95
+ timeout: { type: 'string' },
96
+ versions: { type: 'string' },
97
+ release: { type: 'string' },
98
+ 'server-version-is-release': { type: 'boolean' },
99
+ 'include-prereleases': { type: 'boolean' },
100
+ arg: { type: 'string', multiple: true },
101
+ bin: { type: 'string' },
102
+ cmd: { type: 'string' },
103
+ 'allow-scripts': { type: 'boolean' },
104
+ 'install-timeout': { type: 'string' },
105
+ csv: { type: 'string' },
106
+ 'keep-installs': { type: 'boolean' },
107
+ scenario: { type: 'string' },
108
+ plan: { type: 'boolean' },
109
+ init: { type: 'boolean' },
110
+ help: { type: 'boolean', short: 'h' },
111
+ version: { type: 'boolean', short: 'v' },
112
+ },
113
+ });
114
+ if (values.version) {
115
+ process.stdout.write(`${VERSION}\n`);
116
+ return 0;
117
+ }
118
+ const [sub, ...rest] = positionals;
119
+ if (values.help || !sub) {
120
+ process.stdout.write(HELP);
121
+ return sub || values.help ? 0 : 2;
122
+ }
123
+ if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
124
+ throw new UsageError(`Unknown command "${sub}". Try --help.`);
125
+ const format = (values.json ? 'json' : values.format ?? 'text');
126
+ if (!['text', 'json', 'github', 'markdown'].includes(format))
127
+ throw new UsageError(`--format must be text, json, github or markdown`);
128
+ const failOn = (values['fail-on'] ?? 'error');
129
+ if (!(failOn in SEVERITY_RANK))
130
+ throw new UsageError(`--fail-on must be error, warn or info`);
131
+ const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
132
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
133
+ throw new UsageError(`--timeout must be a positive number of milliseconds`);
134
+ const config = await loadConfig(values.config);
135
+ if (sub === 'diff') {
136
+ if (command)
137
+ throw new UsageError('diff compares two menu files; it doesn\'t start a server. Run snapshot first.');
138
+ if (rest.length !== 2)
139
+ throw new UsageError('diff needs two menu files: toolmenu diff <old.json> <new.json>');
140
+ const [before, after] = await Promise.all(rest.map((path) => loadMenu(path)));
141
+ const release = values.release ? parseRelease(values.release) : undefined;
142
+ const result = diffMenus(before, after, {
143
+ rules: config.rules,
144
+ ignore: config.ignore,
145
+ tokenBudget: config.tokenBudget,
146
+ release,
147
+ serverVersionIsRelease: values['server-version-is-release'] ?? config.serverVersionIsRelease,
148
+ });
149
+ process.stdout.write(formatDiff(result, before.tools.length, after.tools.length, format) + '\n');
150
+ return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
151
+ }
152
+ if (sub === 'history') {
153
+ if (command)
154
+ throw new UsageError('history takes an npm package name, not a command. Use --cmd for unusual entry points.');
155
+ if (rest.length !== 1)
156
+ throw new UsageError('history needs one npm package name: toolmenu history <package>');
157
+ const versions = values.versions === 'all' ? Infinity : values.versions ? Number(values.versions) : 10;
158
+ if (versions !== Infinity && (!Number.isInteger(versions) || versions < 1))
159
+ throw new UsageError('--versions must be a whole number of versions, 1 or more, or "all"');
160
+ const installTimeoutMs = values['install-timeout'] ? Number(values['install-timeout']) : 180_000;
161
+ if (!Number.isFinite(installTimeoutMs) || installTimeoutMs <= 0)
162
+ throw new UsageError('--install-timeout must be a positive number of milliseconds');
163
+ const pkg = rest[0];
164
+ const outDir = values.out ?? join('toolmenu-history', pkg.replace(/^@/, '').replace(/\//g, '__'));
165
+ process.stderr.write(`toolmenu history: installs and runs ${pkg} from npm, version by version. Run it in a container.\n`);
166
+ const result = await history(pkg, {
167
+ versions,
168
+ includePrereleases: values['include-prereleases'],
169
+ args: values.arg,
170
+ env: parsePairs(values.env ?? [], '=', '--env'),
171
+ bin: values.bin,
172
+ cmd: values.cmd,
173
+ allowScripts: values['allow-scripts'],
174
+ installTimeoutMs,
175
+ timeoutMs,
176
+ outDir,
177
+ keepInstalls: values['keep-installs'],
178
+ onProgress: format === 'text' ? (m) => process.stderr.write(m + '\n') : undefined,
179
+ });
180
+ if (values.csv)
181
+ await writeFile(values.csv, historyCsv(result));
182
+ process.stdout.write(formatHistory(result, format, outDir) + '\n');
183
+ return result.rows.length > 0 && result.rows.every((r) => r.status === 'failed') ? 1 : 0;
184
+ }
185
+ if (sub === 'session' && values.init) {
186
+ const path = values.scenario ?? 'scenario.yml';
187
+ if (existsSync(path))
188
+ throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
189
+ const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
190
+ const conn = await connect(initTarget, { timeoutMs });
191
+ try {
192
+ const list = await listTools(conn, { timeoutMs });
193
+ const menu = buildMenu(list.tools, { name: conn.server.name, version: conn.server.version, protocolVersion: conn.protocolVersion });
194
+ await writeFile(path, starterScenario(menu));
195
+ process.stdout.write(`wrote ${path}: a starter scenario for ${menu.tools.length} tools. Fill in the TODOs, then run toolmenu session --scenario ${path}\n`);
196
+ }
197
+ finally {
198
+ await conn.close().catch(() => { });
199
+ }
200
+ return 0;
201
+ }
202
+ if (sub === 'session') {
203
+ if (!values.scenario)
204
+ throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
205
+ let scenario;
206
+ try {
207
+ scenario = await loadScenario(values.scenario);
208
+ }
209
+ catch (error) {
210
+ throw new UsageError(error instanceof Error ? error.message : String(error));
211
+ }
212
+ if (values.plan) {
213
+ process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
214
+ return 0;
215
+ }
216
+ const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
217
+ const result = await session(sessionTarget, scenario, { timeoutMs, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
218
+ const output = formatSession(result, format);
219
+ if (output)
220
+ process.stdout.write(output + '\n');
221
+ return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
222
+ }
223
+ const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
224
+ const routesPath = values.routes ?? config.routes;
225
+ const routes = routesPath ? await loadRoutes(routesPath) : undefined;
226
+ const { menu, findings } = await snapshot(target, { routes, timeoutMs, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
227
+ const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
228
+ if (outPath)
229
+ await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
230
+ const output = formatSnapshot(menu, findings, format, outPath);
231
+ if (output)
232
+ process.stdout.write(output + '\n');
233
+ return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
234
+ }
235
+ function parseTarget(positionals, command, headers, env) {
236
+ if (command) {
237
+ if (command.length === 0)
238
+ throw new UsageError('Nothing after "--": give the command that starts the server.');
239
+ if (positionals.length)
240
+ throw new UsageError(`Unexpected "${positionals[0]}" before "--".`);
241
+ return { kind: 'stdio', command: command[0], args: command.slice(1), env: parsePairs(env, '=', '--env') };
242
+ }
243
+ const [url, ...extra] = positionals;
244
+ if (!url)
245
+ throw new UsageError('Give a server URL, or "-- <command>" for a stdio server.');
246
+ if (extra.length)
247
+ throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
248
+ if (!/^https?:\/\//.test(url))
249
+ throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
250
+ return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
251
+ }
252
+ function parsePairs(items, separator, flag) {
253
+ const pairs = {};
254
+ for (const item of items) {
255
+ const i = item.indexOf(separator);
256
+ if (i <= 0)
257
+ throw new UsageError(`${flag} "${item}" should look like KEY${separator}VALUE`);
258
+ pairs[item.slice(0, i).trim()] = item.slice(i + 1).trim();
259
+ }
260
+ return pairs;
261
+ }
262
+ // Exit only once stdout and stderr have drained: on a pipe, writes can be
263
+ // asynchronous, and exiting early truncates a large --json report. (An explicit
264
+ // exit is still needed: a stdio server or HTTP socket can keep the loop alive.)
265
+ function exitWhenFlushed(code) {
266
+ process.exitCode = code;
267
+ // A reader that closed the pipe early (| head) isn't an error: keep the exit code.
268
+ for (const stream of [process.stdout, process.stderr]) {
269
+ stream.on('error', (error) => process.exit(error.code === 'EPIPE' ? code : 2));
270
+ }
271
+ process.stdout.write('', () => process.stderr.write('', () => process.exit(code)));
272
+ }
273
+ main(process.argv.slice(2)).then(exitWhenFlushed, (error) => {
274
+ const message = error instanceof Error ? error.message : String(error);
275
+ process.stderr.write(`toolmenu: ${message}\n`);
276
+ exitWhenFlushed(2);
277
+ });
278
+ function parseRelease(value) {
279
+ const m = /^(.+?)\.\.(.+)$/.exec(value);
280
+ if (!m)
281
+ throw new UsageError(`--release expects <old>..<new>, e.g. 1.4.0..1.5.0 (got "${value}")`);
282
+ return { before: m[1], after: m[2] };
283
+ }
@@ -0,0 +1,32 @@
1
+ import type { MenuTool } from './types.js';
2
+ export type ChangeKind = 'added' | 'removed' | 'moved' | 'description' | 'inputSchema' | 'outputSchema' | 'annotations' | 'other'
3
+ /** Same content, different bytes: key or property order changed. */
4
+ | 'serialization';
5
+ export interface ToolChange {
6
+ kind: ChangeKind;
7
+ tool: string;
8
+ /** Position in the newer menu (in the older one for `removed`). */
9
+ position: number;
10
+ }
11
+ /** JSON with sorted object keys, so key order never counts as a change. */
12
+ export declare function canonical(value: unknown): string;
13
+ /** Which parts of one tool changed. Empty when the definitions are identical. */
14
+ export declare function toolDiff(before: MenuTool, after: MenuTool): ChangeKind[];
15
+ export declare function compareMenus(before: MenuTool[], after: MenuTool[]): ToolChange[];
16
+ export interface CacheBreak {
17
+ /** First position where the newer menu stops matching the older one. */
18
+ position: number;
19
+ /**
20
+ * Estimated tokens of the newer tool list from that position to the end.
21
+ * A floor, not the cost: most clients send the tool list at the start of the
22
+ * prompt (Claude's Messages API renders tools first), so everything after the
23
+ * list, the system prompt and the whole conversation, is processed again too.
24
+ */
25
+ tokensAffected: number;
26
+ }
27
+ /**
28
+ * Where the tool list itself stops matching between `before` and `after`.
29
+ * Null when `after` only appends to `before`. That keeps the tool list a valid
30
+ * prefix of itself, not the conversation's cache: see session/append.
31
+ */
32
+ export declare function cacheBreak(before: MenuTool[], after: MenuTool[]): CacheBreak | null;