toolmenu 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +150 -30
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +61 -18
  6. package/dist/auto.d.ts +71 -7
  7. package/dist/auto.js +267 -13
  8. package/dist/breakdown.d.ts +35 -6
  9. package/dist/breakdown.js +179 -23
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +116 -178
  13. package/dist/compare.js +20 -5
  14. package/dist/config.d.ts +5 -1
  15. package/dist/config.js +14 -0
  16. package/dist/connect.d.ts +4 -0
  17. package/dist/connect.js +132 -11
  18. package/dist/diff.d.ts +35 -1
  19. package/dist/diff.js +1132 -121
  20. package/dist/difference.d.ts +2 -0
  21. package/dist/difference.js +13 -12
  22. package/dist/explain.d.ts +103 -0
  23. package/dist/explain.js +434 -0
  24. package/dist/failures.d.ts +55 -0
  25. package/dist/failures.js +103 -0
  26. package/dist/help.d.ts +9 -0
  27. package/dist/help.js +198 -0
  28. package/dist/history.d.ts +13 -0
  29. package/dist/history.js +86 -6
  30. package/dist/init.d.ts +100 -5
  31. package/dist/init.js +520 -26
  32. package/dist/menu.js +10 -2
  33. package/dist/options.d.ts +135 -0
  34. package/dist/options.js +48 -0
  35. package/dist/probe.d.ts +15 -3
  36. package/dist/probe.js +35 -11
  37. package/dist/report.d.ts +1 -1
  38. package/dist/report.js +229 -99
  39. package/dist/rules/description.d.ts +7 -2
  40. package/dist/rules/description.js +89 -43
  41. package/dist/rules/determinism.d.ts +3 -1
  42. package/dist/rules/determinism.js +146 -11
  43. package/dist/rules/ids.d.ts +1 -1
  44. package/dist/rules/ids.js +9 -4
  45. package/dist/rules/index.d.ts +6 -0
  46. package/dist/rules/index.js +15 -5
  47. package/dist/rules/naming.d.ts +1 -1
  48. package/dist/rules/naming.js +66 -19
  49. package/dist/rules/rule.d.ts +18 -0
  50. package/dist/rules/rule.js +25 -1
  51. package/dist/rules/spec.d.ts +2 -0
  52. package/dist/rules/spec.js +61 -8
  53. package/dist/rules/write.d.ts +1 -1
  54. package/dist/rules/write.js +42 -10
  55. package/dist/semver.js +1 -1
  56. package/dist/session.d.ts +56 -7
  57. package/dist/session.js +800 -94
  58. package/dist/snapshot.d.ts +2 -2
  59. package/dist/snapshot.js +25 -7
  60. package/dist/types.d.ts +14 -0
  61. package/dist/words.d.ts +5 -0
  62. package/dist/words.js +14 -0
  63. package/package.json +1 -1
package/README.md CHANGED
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
29
29
  (`history` installs old versions with today's dependencies, so it records what they
30
30
  serve now, which can differ from what they shipped with: FINDINGS F4.)
31
31
 
32
- > **Status: 0.11, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
32
+ > **Status: 0.12, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
33
33
  > real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
34
34
 
35
35
  ## Start here
@@ -46,8 +46,17 @@ Commit both, open a pull request, and toolmenu comments on it.
46
46
  npx toolmenu init -- node dist/server.js # add --with-session to also run session --auto
47
47
  ```
48
48
 
49
- Credentials you pass with `--env` go into the workflow as `${{ secrets.… }}`, never as
50
- values; `init` says which secrets to add. It never overwrites a file.
49
+ Every value you pass with `--env` or `--header` goes into the workflow as
50
+ `${{ secrets.… }}`, never as text (a name like `DATABASE_URL` doesn't say it holds a
51
+ password); `init` says which secrets to add. It never overwrites a file.
52
+
53
+ Run it anywhere in the repository. `.github/workflows/toolmenu.yml` goes to the git
54
+ root, and the server starts in CI from the folder you ran `init` in, which is also
55
+ where `menu.json` is saved. That's how a monorepo package works: run `init` in
56
+ `packages/server`. Paths to files in the repository are made relative, so they work on
57
+ the runner. `init` lists anything that won't work there (a file outside the
58
+ repository, a `localhost` URL, a virtualenv the runner doesn't have), each with what
59
+ to do.
51
60
 
52
61
  By hand:
53
62
 
@@ -74,6 +83,11 @@ dynamic registration needs a pre-registered app: `--client-id` (and
74
83
  Both protocol generations are supported: 2026-07-28 (`server/discover`, stateless)
75
84
  and the 2025 `initialize` handshake, through the official TypeScript SDK.
76
85
 
86
+ Every finding and error says what's wrong, where, what toolmenu saw, and ends in a
87
+ `→ Next:` step. Findings marked `· unsure` are heuristics (names and words): they say
88
+ what was seen, not what it means. Every command has its own help:
89
+ `toolmenu history --help`.
90
+
77
91
  ## In CI: one PR comment
78
92
 
79
93
  ```yaml
@@ -90,7 +104,7 @@ jobs:
90
104
  - uses: actions/setup-node@v7
91
105
  with: { node-version: 22 }
92
106
  - run: npm ci && npm run build
93
- - uses: niksa90/toolmenu@v0.11.0
107
+ - uses: niksa90/toolmenu@v0.13.0
94
108
  with:
95
109
  command: node dist/server.js
96
110
  baseline: menu.json # your committed snapshot
@@ -141,9 +155,19 @@ steps:
141
155
  `--init` writes a starter scenario from the live menu: it calls every read-only tool
142
156
  that needs no arguments, then repeats one. If a tool looks like it unlocks more tools
143
157
  (a `domains` or `toolset` parameter with an enum, "unlock" or "capabilities" in its
144
- name or description), the starter unlocks two values for real and repeats the first.
145
- Anything it can't fill in is left as a commented-out step. A scenario is only as good
146
- as its steps, so the starter is a floor, not a ceiling.
158
+ name, a description that says it enables or loads tools), the starter unlocks
159
+ **every** value, listing after each, then repeats the first, so the session sees every
160
+ tool behind the unlock. The values come from the parameter's enum or, without one,
161
+ from the server's own read-only listing (`list_toolsets`). Lookups that only describe
162
+ toolsets (`list_toolsets`, `get_toolset_tools`) aren't unlocks, and one not marked
163
+ read-only is suggested, never called. Anything it can't fill in is left as a
164
+ commented-out step. An unlock nothing in the menu names, like joining a room that
165
+ brings more tools, needs a step you add yourself. A scenario is only as good as its
166
+ steps, so the starter is a floor, not a ceiling.
167
+
168
+ On github-mcp-server 1.0.5 with `--dynamic-toolsets` (3 tools at the start, 19
169
+ toolsets), unlocking every toolset reaches 107 tools: all 81 that `--toolsets all`
170
+ serves, and 23 issue and pull-request tools that only the dynamic mode has.
147
171
 
148
172
  | Rule | Default | Catches |
149
173
  |---|---|---|
@@ -155,17 +179,36 @@ as its steps, so the starter is a floor, not a ceiling.
155
179
  | `session/unannounced` | warn | The menu changed without `notifications/tools/list_changed`, although the server declared `listChanged` |
156
180
  | `session/untested` | warn, error if no call got through | Calls that failed before reaching the tool: authentication, something missing on this machine (no Chrome), the network. One finding for the run, not one per step, so an expired CI secret doesn't pass a run that tested nothing |
157
181
  | `session/refused`, `session/step-failed` | error | A write the scenario didn't allow, or a call that failed for another reason |
158
- | `session/tool-error` | warn | A tool that answered with an error for another reason (`isError`): the run tested less than it looks |
182
+ | `session/tool-error` | warn | A tool that answered with an error for another reason (`isError`): the run tested less than it looks. The same error at several steps is one finding that names them |
183
+ | `session/known-variance` | info | A mid-session change only in values that already vary from one tools/list to the next: same cause as the finding it names, not counted again |
184
+ | `session/assumed-read-only` | info | Tools called on your word (`--assume-read-only`, `assume_read_only`) although the server doesn't mark them readOnlyHint |
185
+ | `session/unlock-coverage` | warn | An unlock whose enum lists its values, called with only some of them: the tools behind the rest were never seen, so a baseline from the session misses them. With `--union-out` or `--auto`, an unlock never called at all too, with why `--auto` skipped it |
186
+ | `session/nothing-called` | warn | `--auto` called no tools (all open-world, missing values, or not read-only): the run only listed the menu |
187
+ | `session/session-lost` | error | The server ended the session after toolmenu opened a second connection with the same credentials (servers with one session per client). The run stops there; rerun with `--processes 1` |
188
+ | `session/rate-limited` | error | The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it |
189
+ | `session/scope-unchecked` | info | A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run |
159
190
 
160
191
  **No scenario? `session --auto`** builds the steps from the menu: every read-only tool
161
192
  whose required arguments the schema itself gives (a `default`, `examples`, an `enum`,
162
- a type or format), cheapest first, then the first call again. It never guesses an ID:
163
- a tool that needs `owner`, `repo` or an issue key is skipped, and the report says
164
- which values were missing (`--save-scenario auto.yml` writes the steps, with the
165
- skipped tools commented out, to fill in). Tools marked `openWorldHint: true` (web
193
+ a type or format), cheapest first, then every value of each read-only unlock (outside
194
+ `--max-calls`, and the first value twice: a second identical unlock should change
195
+ nothing), then the first call again. It never guesses an ID: a tool that needs
196
+ `owner`, `repo` or an issue key is skipped, and the report names the flag that fills
197
+ it: `--value repo_path=/src/app` fills that parameter on every tool that requires it,
198
+ `--value get_issue.issue_key=ABC-1` on one tool, `--values-file values.yml` takes a map
199
+ of either. Servers that don't mark their tools `readOnlyHint` (DeepWiki, every
200
+ 2024-11-05 server) get nothing called unless you vouch for them by name:
201
+ `--assume-read-only read_wiki_structure,ask_wiki_question`. Only exact names are
202
+ accepted: a pattern would also cover tools a later release adds. A tool marked or
203
+ named as a write is never called, and the report says which tools ran on your word.
204
+ `--save-scenario auto.yml` writes the steps, with the skipped tools commented out, for
205
+ you to fill in. Tools marked `openWorldHint: true` (web
166
206
  search, fetch, scraping, which can cost API credits) are called only with
167
- `--open-world`. On the 22 servers in `bench/`, `--auto` reaches 60 of 262 read-only
168
- tools, 84 with `--open-world`.
207
+ `--open-world`, except unlocks: changing the menu is what a session watches, and an
208
+ unlock spends no search credits, so it runs even on a server that marks every tool
209
+ open-world. A run that ends up calling nothing is `session/nothing-called` (warn),
210
+ with why each tool was left out, not a clean pass. On the 22 servers in `bench/`,
211
+ `--auto` reaches 60 of 262 read-only tools, 84 with `--open-world`.
169
212
 
170
213
  **`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
171
214
  not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
@@ -179,11 +222,16 @@ npx toolmenu diff menu.json new-menu.json
179
222
  ```
180
223
  $ npx toolmenu diff --release 2026.1.14..2026.8.31 fs-2026.1.14.json fs-2026.8.31.json
181
224
  toolmenu diff secure-filesystem-server 2026.1.14 → 2026.8.31
182
- 14 → 14 tools · ~1,640 → ~1,664 tokens (+24, estimate): this release adds ~24 tokens to every conversation that loads the menu
183
- 1 breaking · 0 minor · 15 notice · suggested bump: major · actual: not checked (calendar version)
225
+ changes 1 breaking (1 tool) · 0 minor · 3 notice (13 tools)
226
+ version suggested bump: major · not checked (calendar version)
227
+ tokens ~1,640 → ~1,664 (+24, estimate): this release adds ~24 tokens to every conversation that loads the menu
228
+ tools 14 → 14
184
229
 
185
230
  ERROR diff/safety-hint
186
- move_file was additive-only and is now destructive.
231
+ `move_file` was marked additive-only and is now destructive (destructiveHint false → true; unset means destructive). Clients that auto-approve additive tools may now ask first.
232
+ → Next: If the tool still never deletes or overwrites, set destructiveHint: false; if it does now, say so in the release notes.
233
+ INFO diff/annotations
234
+ Annotations changed the same way (openWorldHint (unset) → false) in 13 of 14 tools: read_file, read_text_file, read_media_file, read_multiple_files, write_file, edit_file, … (+7).
187
235
  …
188
236
  ```
189
237
 
@@ -194,15 +242,49 @@ the way OpenAPI breaking-change checkers do it:
194
242
 
195
243
  | Class | Changes | Rules |
196
244
  |---|---|---|
197
- | **breaking** (error) | tool removed or renamed, parameter removed, new required parameter, narrower or different type, enum narrowed, a tool becoming less safe (no longer read-only, or additive → destructive) | `diff/tool-removed`, `diff/tool-renamed`, `diff/param-removed`, `diff/param-required`, `diff/param-type`, `diff/enum-narrowed`, `diff/safety-hint` |
198
- | **minor** (info) | tool added, new optional parameter, parameter now optional, type or enum widened | `diff/tool-added`, `diff/param-added`, `diff/param-relaxed`, `diff/type-widened`, `diff/enum-widened` |
199
- | **notice** (info) | optional parameter dropped (extra properties still allowed), description changed (with the text diff: it changes what the agent does), output schema, annotations, other fields, order | `diff/param-dropped`, `diff/description`, `diff/schema-other`, `diff/annotations`, `diff/other`, `diff/order` |
245
+ | **breaking** (error) | tool removed or renamed, parameter removed, new required parameter, narrower or different type, enum narrowed, unlisted properties now rejected (`additionalProperties: false` added), a tool becoming less safe (no longer read-only, or additive → destructive) | `diff/tool-removed`, `diff/tool-renamed`, `diff/param-removed`, `diff/param-required`, `diff/param-type`, `diff/enum-narrowed`, `diff/properties-closed`, `diff/safety-hint` |
246
+ | **minor** (info) | tool added, new optional parameter, parameter now optional, type or enum widened, unlisted properties now accepted (`additionalProperties: false` removed) | `diff/tool-added`, `diff/param-added`, `diff/param-relaxed`, `diff/type-widened`, `diff/enum-widened`, `diff/properties-opened` |
247
+ | **notice** (info) | optional parameter dropped (extra properties still allowed), description changed (with the text diff: it changes what the agent does), output schema, annotations, other fields, order, a schema restructured but accepting the same input, only the declared `$schema` dialect changed | `diff/param-dropped`, `diff/description`, `diff/schema-other`, `diff/annotations`, `diff/other`, `diff/order`, `diff/schema-equivalent`, `diff/schema-dialect` |
248
+
249
+ The same rules apply inside parameters: fields of an object parameter, array items,
250
+ and the options of an `anyOf`/`oneOf` union (zod's unions, discriminated unions and
251
+ `.nullable()`) are compared at every depth, with their path (`gen.body.text`,
252
+ `search.filters[].field`, `gen.block(kind="image").url`). Union options are paired by
253
+ the value of a discriminator they all fix; without one (a plain `z.union` of objects),
254
+ identical options first, so a reorder is no change, then by type and shared property
255
+ names. An option gone is breaking, a new one widens. The same change reached through
256
+ one shared definition at several places is one finding that lists them (`places` in
257
+ `--json`). Local `$ref`s (`#/$defs/…`, `#/definitions/…`) are expanded first,
258
+ recursive ones one level deep, so a schema is compared by what it accepts, not how
259
+ it's spelled: moving a repeated block into `$defs` is one `diff/schema-equivalent`
260
+ notice with its token change ("accepts the same input: ~899 tokens fewer"), and a
261
+ breaking change inside `$defs` is breaking. A schema change no rule classifies is
262
+ never silent: it's a `diff/schema-other` "review it".
263
+
264
+ On mongodb-mcp-server 2.1.2 → 3.0.0 this finds `aggregate.pipeline[]…$vectorSearch.numCandidates`
265
+ and `.limit` narrowed from `number` to `integer` inside a union, where the whole
266
+ `pipeline` used to be one "review it".
267
+
268
+ Different spellings of the same schema are one schema (`additionalProperties` absent,
269
+ `true` or `{}`; zod 4's safe-integer bounds), and a change every tool shares is said
270
+ once for the menu: a server moving from zod 3 to zod 4 reads as its real changes (a
271
+ field that became required, the patterns zod 4 adds to `email` and `uuid`) plus one
272
+ `diff/properties-opened` line ("27 tools now accept properties they don't list … most
273
+ likely a schema generator upgrade"), not a "review it" per object.
274
+
275
+ The same change in many tools is reported once, with the tools it touches (`pageId`
276
+ is new and required in 25 of 29 tools, on chrome-devtools-mcp 1.8.0); counts are per
277
+ change. When a tool loses a parameter and gains a required one with a close name,
278
+ `diff/param-renamed` (warn, unsure) points out the likely rename; both changes stay
279
+ breaking. Every breaking finding's next step names the version to release when the
280
+ bump is too small.
200
281
 
201
282
  It also reports the token change per tool and suggests a semver bump. To check the
202
283
  bump, pass the **release** versions: `--release 1.4.0..1.5.0` (npm, a git tag).
203
- `diff/version-bump` then warns when the release's bump is smaller (`0.0.x` promises
204
- nothing and `0.x` may break in a minor), and `diff/version-backwards` when the version
205
- goes down. Calendar versions and prereleases aren't judged. The version in the snapshot is what the
284
+ `diff/version-bump` then warns when the release's bump is smaller. Under 1.0.0 it
285
+ reads versions as npm's caret does (`^0.2.3` accepts any 0.2.x): breaking changes need
286
+ a minor bump, new features a patch, and under 0.1.0 anything goes.
287
+ `diff/version-backwards` warns when the version goes down. Calendar versions and prereleases aren't judged. The version in the snapshot is what the
206
288
  server reports (`serverInfo.version`), which is often not the release: the filesystem
207
289
  server has said `0.2.0` for 19 releases (FINDINGS F5). It's shown, and only checked
208
290
  with `--server-version-is-release`. `diff` also enforces an optional `tokenBudget`
@@ -228,10 +310,28 @@ names, then searches for every operation it finds or sees mentioned, until 20 qu
228
310
  in a row turn up nothing new (at most 200). On Atlassian's hosted server that finds all
229
311
  154 operations behind its 21 tools; on Sentry's, 65 behind 9, the same on every run.
230
312
  Queries are paced and a rate limit is waited out. An operation not found this time is a
231
- notice ("not returned by the same queries"), never "removed". To steer it, set
313
+ notice (`diff/catalog-missing`: "not returned by the same queries"), never "removed";
314
+ one found only now is `diff/catalog-added`, and catalogs read with different queries,
315
+ or partly (a query failed), are `diff/catalog-queries`. To steer it, set
232
316
  `"catalog": { "queries": [...], "maxQueries": 200, "crawl": true }` in
233
317
  `toolmenu.config.json` (FINDINGS F15).
234
318
 
319
+ A crawl through search is a lower bound, not a proof: an operation that nothing in the
320
+ catalog names, and whose words no query uses, stays unfound. On `@sentry/mcp-server`
321
+ 0.42 (stdio) it finds 63 of the 64 operations in the package's catalog; `whoami` comes
322
+ back only when asked for by name. Add queries like that to `catalog.queries`.
323
+
324
+ **Command routers.** Some servers put their operations behind router tools instead:
325
+ one tool per area that takes a `command` and its `parameters`, and lists its commands
326
+ when called with `learn: true` (Azure's MCP server serves 71 such tools in front of
327
+ ~410 operations). `--catalog` detects them (a `command` parameter, an arguments
328
+ object, a boolean listing flag, and a description that says "router" or "sub
329
+ commands") and calls each once in its listing mode, never with a command, so nothing
330
+ runs. Routers whose command is required aren't called. Detection missed one? Name
331
+ them: `"catalog": { "routers": ["keyvault", "storage"] }`. A router lists everything
332
+ it has, so unlike a search the catalog is complete except for routers that failed
333
+ (`catalog/failed` says which, and why).
334
+
235
335
  ## `history`: a package's releases, researched
236
336
 
237
337
  ```sh
@@ -248,6 +348,19 @@ It's best effort: packages that need env vars, arguments or auth are recorded as
248
348
  failed, with a reason, never skipped. **It installs and runs third-party code. Run it
249
349
  in a container.**
250
350
 
351
+ ## Where the tokens go
352
+
353
+ `snapshot` also shows what the menu's tokens are spent on: the biggest tools (split
354
+ into description and schema), large enums, and waste you can remove:
355
+
356
+ - **Repeated blocks**: the same piece of schema at several places, inside one tool (a
357
+ `$defs` entry could hold it once: "one ~230-token block ×5 in
358
+ content_generateMessageHtml") or across tools (paid once per tool: mongodb-mcp-server
359
+ 3.0.4 repeats a ~2,200-token `pipeline` schema in 3 tools). Only the largest repeated
360
+ block is named, not the pieces inside it.
361
+ - **Unused `$defs`**: definitions nothing in the tool refers to. Notion's server 2.5.2
362
+ ships the same 9 in all 24 tools, 72% of its menu.
363
+
251
364
  ## `snapshot` rules
252
365
 
253
366
  Every rule comes from a failure I actually hit building 115 tools for one MCP
@@ -259,12 +372,15 @@ server, or from the spec itself. If a rule can't point to one, it doesn't ship.
259
372
  | `menu/process-variance` | error | A second server process, started the same way, serves a different menu, so every restart misses the cache. Seen on mcp-atlassian: a default built from a Python set, in a new order every start (FINDINGS F12). toolmenu runs the main process with `PYTHONHASHSEED=0`, so a Python server's saved menu is reproducible, and the second with another seed. `--processes 1` turns it off |
260
373
  | `menu/connection-variance` | error (warn on 2025 protocols) | The same over HTTP: a second connection with the same credentials gets a different menu. Retried once before reporting |
261
374
  | `spec/schema` | error | A `tools/list` result that fails the official schema for its protocol version |
375
+ | `spec/tools-capability` | error | The server doesn't declare the `tools` capability, so clients (the official SDK included) never ask for its tools |
376
+ | `menu/duplicate-name` | error | Two tools with the same name: only one can be called, and some clients reject the list |
262
377
  | `naming/route` | error | A `routes.yml` expectation broke: a keyword now matches the wrong tool at least as well as the right one |
263
378
  | `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions: 2,048 characters in Claude Code, or your `descriptionLimit`. From a real failure: a client cut at 280 characters, and the line that decided routing was at 1,222 |
264
- | `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary. The model gets a prefix that can read as complete |
265
- | `naming/vague-id` | warn | A parameter called just `id` that doesn't say *which* thing |
379
+ | `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary that shows where each one is cut. The model gets a prefix that can read as complete |
380
+ | `description/late-instruction` | info | With the default cut: instructions after character 280, which a client that cuts descriptions short would hide, as one summary. Set `descriptionLimit` and `description/buried` checks your real cut instead. (Before 0.13 this was part of `description/cut`; that id's setting still applies.) |
381
+ | `naming/vague-id` | warn | Parameters called just `id` that don't say *which* thing, as one summary with a suggested name |
266
382
  | `ids/authored` | info | An ID the agent must supply that no tool appears to return, so the agent may invent it, one finding per kind of ID (heuristic: IDs from a URL or another tool's text output look the same) |
267
- | `write/unannotated` | warn | A tool named like a write (`delete_`, `send_`…) with no annotations |
383
+ | `write/unannotated` | warn | Tools named like writes (`delete_`, `send_`…) with no annotations, as one summary grouped by the annotation each name suggests |
268
384
  | `naming/shared-word` | info | Nouns that name different things across tools (`list_team_audits` vs `get_audit_trail`), as one summary |
269
385
  | `write/no-dry-run` | info | Destructive tools with no dry-run parameter or preview tool, as one summary |
270
386
  | `spec/discover` | info | The server only speaks the 2025 protocol, so 2026-07-28 rules are skipped |
@@ -342,11 +458,12 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
342
458
  --env <K=V> env var for a stdio server, repeatable (it only gets a
343
459
  minimal environment otherwise)
344
460
  --timeout <ms> per-request timeout (default: 30000)
345
- --processes <n> server processes or connections to compare (default: 2)
461
+ --processes <n> server processes or connections to compare (default: 2);
462
+ 1 opens no second one, session's scope check included
346
463
  ```
347
464
 
348
465
  Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
349
- or bad usage. `toolmenu --help` lists the `session` and `history` options.
466
+ or bad usage. `toolmenu <command> --help` lists that command's own options.
350
467
 
351
468
  ## Limitations
352
469
 
@@ -356,7 +473,10 @@ or bad usage. `toolmenu --help` lists the `session` and `history` options.
356
473
  - **Token counts are estimates** (`o200k_base`) of what the model reads: each tool's
357
474
  name, description and input schema. Output schemas, annotations, icons and `_meta`
358
475
  stay with the client and aren't counted. Vendors tokenize, cache and bill
359
- differently.
476
+ differently. `diff` recounts both menus with the toolmenu that runs it, so a
477
+ baseline from an older version still compares fairly; a number quoted elsewhere (a
478
+ commit message, a dashboard) should say which toolmenu counted it (0.7.1 changed the
479
+ count for menus with icons by up to 5×, FINDINGS F7).
360
480
  - **Heuristic rules say so** in their messages.
361
481
  - Not a security scanner or a full conformance suite. Other tools do those well.
362
482
 
package/dist/args.d.ts CHANGED
@@ -7,15 +7,41 @@ export type Synthesized = {
7
7
  ok: false;
8
8
  };
9
9
  /**
10
- * A value the schema itself vouches for: `const`, `default`, `examples`, `enum`,
11
- * then by type and format. Never a guess at an ID or anything with a `pattern`:
12
- * a tool that needs one is skipped, not called with junk.
10
+ * Does a value fit the schema's type, enum and const? Only the checks that catch a
11
+ * default the server itself wouldn't accept (Microsoft Learn: `default: null` on a
12
+ * `type: "string"`); patterns and formats are the server's business.
13
+ */
14
+ export declare function fits(value: unknown, schema: JsonSchema | undefined): boolean;
15
+ /**
16
+ * A value the schema itself vouches for: `const`, a `default` or `examples` value
17
+ * that fits the type, `enum`, then by type and format. Never a guess at an ID or
18
+ * anything with a `pattern`, and never an empty list for a list it has to fill: a
19
+ * tool that needs one is skipped, not called with junk.
13
20
  */
14
21
  export declare function synthesize(schema: JsonSchema | undefined, name?: string): Synthesized;
15
- /** Arguments for a tool's required parameters, or the ones it can't fill. */
16
- export declare function synthesizeArgs(schema: JsonSchema | undefined): {
22
+ /**
23
+ * A value the user gave (--value), made to fit the parameter: `5` is the number 5
24
+ * for an integer and the string "5" for a string; JSON that doesn't fit the type
25
+ * falls back to the text as typed.
26
+ */
27
+ export declare function fitValue(given: {
28
+ raw?: string;
29
+ value: unknown;
30
+ }, schema: JsonSchema | undefined): unknown;
31
+ /** Where a tool's argument came from: the user (--value) or the schema. */
32
+ export type ArgSource = 'value' | 'schema';
33
+ /**
34
+ * Arguments for a tool's required parameters, or the ones it can't fill. `given`:
35
+ * values the user supplied for this tool, by parameter name; they come first, and
36
+ * one for an optional parameter is passed too.
37
+ */
38
+ export declare function synthesizeArgs(schema: JsonSchema | undefined, given?: Record<string, {
39
+ raw?: string;
40
+ value: unknown;
41
+ }>): {
17
42
  ok: true;
18
43
  args: Record<string, unknown>;
44
+ sources: Record<string, ArgSource>;
19
45
  } | {
20
46
  ok: false;
21
47
  missing: string[];
package/dist/args.js CHANGED
@@ -3,19 +3,60 @@ const DATE_TIME = '2026-01-01T00:00:00Z';
3
3
  const DATE = '2026-01-01';
4
4
  const SEARCH_WORDS = /^(q|query|search|search_?query|text|keywords?|terms?|prompt|question)$/i;
5
5
  /**
6
- * A value the schema itself vouches for: `const`, `default`, `examples`, `enum`,
7
- * then by type and format. Never a guess at an ID or anything with a `pattern`:
8
- * a tool that needs one is skipped, not called with junk.
6
+ * Does a value fit the schema's type, enum and const? Only the checks that catch a
7
+ * default the server itself wouldn't accept (Microsoft Learn: `default: null` on a
8
+ * `type: "string"`); patterns and formats are the server's business.
9
+ */
10
+ export function fits(value, schema) {
11
+ if (!schema || typeof schema !== 'object')
12
+ return true;
13
+ if ('const' in schema && JSON.stringify(schema.const) !== JSON.stringify(value))
14
+ return false;
15
+ if (schema.enum?.length && !schema.enum.some((e) => JSON.stringify(e) === JSON.stringify(value)))
16
+ return false;
17
+ const branches = (schema.anyOf ?? schema.oneOf);
18
+ if (Array.isArray(branches) && branches.length && !branches.some((b) => fits(value, b)))
19
+ return false;
20
+ if (schema.type === undefined)
21
+ return true;
22
+ const types = Array.isArray(schema.type) ? schema.type : [schema.type];
23
+ return types.some((t) => {
24
+ switch (t) {
25
+ case 'null':
26
+ return value === null;
27
+ case 'string':
28
+ return typeof value === 'string';
29
+ case 'boolean':
30
+ return typeof value === 'boolean';
31
+ case 'integer':
32
+ return Number.isInteger(value);
33
+ case 'number':
34
+ return typeof value === 'number' && Number.isFinite(value);
35
+ case 'array':
36
+ return Array.isArray(value) && (!schema.items || value.every((v) => fits(v, schema.items)));
37
+ case 'object':
38
+ return !!value && typeof value === 'object' && !Array.isArray(value);
39
+ default:
40
+ return true;
41
+ }
42
+ });
43
+ }
44
+ /**
45
+ * A value the schema itself vouches for: `const`, a `default` or `examples` value
46
+ * that fits the type, `enum`, then by type and format. Never a guess at an ID or
47
+ * anything with a `pattern`, and never an empty list for a list it has to fill: a
48
+ * tool that needs one is skipped, not called with junk.
9
49
  */
10
50
  export function synthesize(schema, name = '') {
11
51
  if (!schema || typeof schema !== 'object')
12
52
  return { ok: false };
13
53
  if ('const' in schema)
14
54
  return { ok: true, value: schema.const };
15
- if ('default' in schema)
55
+ if ('default' in schema && fits(schema.default, schema))
16
56
  return { ok: true, value: schema.default };
17
- if (Array.isArray(schema.examples) && schema.examples.length)
18
- return { ok: true, value: schema.examples[0] };
57
+ const example = Array.isArray(schema.examples) ? schema.examples.find((e) => fits(e, schema)) : undefined;
58
+ if (example !== undefined)
59
+ return { ok: true, value: example };
19
60
  if (schema.enum?.length)
20
61
  return { ok: true, value: schema.enum[0] };
21
62
  const branches = (schema.anyOf ?? schema.oneOf);
@@ -53,10 +94,10 @@ export function synthesize(schema, name = '') {
53
94
  return { ok: false };
54
95
  }
55
96
  case 'array': {
97
+ // A required list is there to be filled: `{"urls": []}` is refused by the
98
+ // server (Exa's web_fetch_exa), or does nothing. No item, no call.
56
99
  const item = synthesize(schema.items, name);
57
- if (item.ok)
58
- return { ok: true, value: [item.value] };
59
- return typeof schema.minItems === 'number' && schema.minItems > 0 ? { ok: false } : { ok: true, value: [] };
100
+ return item.ok ? { ok: true, value: [item.value] } : { ok: false };
60
101
  }
61
102
  case 'object': {
62
103
  const args = synthesizeArgs(schema);
@@ -66,16 +107,52 @@ export function synthesize(schema, name = '') {
66
107
  return { ok: false };
67
108
  }
68
109
  }
69
- /** Arguments for a tool's required parameters, or the ones it can't fill. */
70
- export function synthesizeArgs(schema) {
110
+ /**
111
+ * A value the user gave (--value), made to fit the parameter: `5` is the number 5
112
+ * for an integer and the string "5" for a string; JSON that doesn't fit the type
113
+ * falls back to the text as typed.
114
+ */
115
+ export function fitValue(given, schema) {
116
+ if (fits(given.value, schema))
117
+ return given.value;
118
+ if (given.raw !== undefined && fits(given.raw, schema))
119
+ return given.raw;
120
+ // A single value for a list: --value urls=https://… means ["https://…"].
121
+ if (schema?.type === 'array' && fits([given.value], schema))
122
+ return [given.value];
123
+ return given.value;
124
+ }
125
+ /**
126
+ * Arguments for a tool's required parameters, or the ones it can't fill. `given`:
127
+ * values the user supplied for this tool, by parameter name; they come first, and
128
+ * one for an optional parameter is passed too.
129
+ */
130
+ export function synthesizeArgs(schema, given = {}) {
71
131
  const args = {};
132
+ const sources = {};
72
133
  const missing = [];
73
- for (const param of schema?.required ?? []) {
134
+ const required = schema?.required ?? [];
135
+ for (const param of required) {
136
+ if (param in given) {
137
+ args[param] = fitValue(given[param], schema?.properties?.[param]);
138
+ sources[param] = 'value';
139
+ continue;
140
+ }
74
141
  const v = synthesize(schema?.properties?.[param], param);
75
- if (v.ok)
142
+ if (v.ok) {
76
143
  args[param] = v.value;
144
+ sources[param] = 'schema';
145
+ }
77
146
  else
78
147
  missing.push(param);
79
148
  }
80
- return missing.length ? { ok: false, missing } : { ok: true, args };
149
+ if (missing.length)
150
+ return { ok: false, missing };
151
+ for (const [param, g] of Object.entries(given)) {
152
+ if (param in args || !schema?.properties || !(param in schema.properties))
153
+ continue;
154
+ args[param] = fitValue(g, schema.properties[param]);
155
+ sources[param] = 'value';
156
+ }
157
+ return { ok: true, args, sources };
81
158
  }
package/dist/auth.d.ts CHANGED
@@ -6,7 +6,7 @@ import type { OAuthClientInformationContext, OAuthClientMetadata, OAuthClientPro
6
6
  export declare const DEFAULT_PORT = 33418;
7
7
  /** Where logins live: $TOOLMENU_AUTH_DIR, else $XDG_CONFIG_HOME/toolmenu/auth, else ~/.config/toolmenu/auth. */
8
8
  export declare function authDir(): string;
9
- /** Is there a stored login for this server? */
9
+ /** Is there a stored file for this server (a login, or what's left of one)? Why a login is needed reads its contents. */
10
10
  export declare function hasLogin(serverUrl: string): boolean;
11
11
  export interface ProviderOptions {
12
12
  /** A browser login may start (auth login). Otherwise a needed login is an error. */
@@ -28,6 +28,8 @@ export declare class StoredOAuthProvider implements OAuthClientProvider {
28
28
  private readonly options;
29
29
  private data;
30
30
  private expectedState;
31
+ /** invalidateCredentials('all') deleted a file that held tokens, in this run. */
32
+ private removedLogin;
31
33
  constructor(serverUrl: string, options?: ProviderOptions);
32
34
  private stored;
33
35
  private update;
@@ -50,7 +52,7 @@ export declare class StoredOAuthProvider implements OAuthClientProvider {
50
52
  invalidateCredentials(scope: 'all' | 'client' | 'tokens' | 'verifier' | 'discovery'): Promise<void>;
51
53
  }
52
54
  export declare class LoginNeededError extends Error {
53
- constructor(serverUrl: string);
55
+ constructor(serverUrl: string, why?: string);
54
56
  }
55
57
  export interface LoginOptions extends Omit<ProviderOptions, 'interactive' | 'onRedirect'> {
56
58
  /** How to show the authorization URL. Default: print it and try to open a browser. */