toolmenu 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,507 @@
1
+ {
2
+ "auth/invalid-accepted": {
3
+ "command": "snapshot",
4
+ "severity": "warn",
5
+ "text": "HTTP with credentials, `--processes` 2 or more. toolmenu lists the menu once more with an invalid credential in the same scheme (Bearer, Basic, the API-key header, or the stored login). A server that answers it with a menu, not 401 or 403, gives a client whose token expired or was revoked no reason to log in again: the agent just sees fewer tools (Atlassian Rovo served 4 tools to any `Authorization` header)"
6
+ },
7
+ "catalog/failed": {
8
+ "command": "snapshot",
9
+ "severity": "info",
10
+ "text": "`--catalog`: searches or router listings that failed, grouped by cause (rate limit, authentication, unreadable output). Their operations aren't in the catalog, so `diff` can't see them"
11
+ },
12
+ "catalog/read": {
13
+ "command": "snapshot",
14
+ "severity": "info",
15
+ "text": "`--catalog`: what was read behind a search tool or command router, with how many searches or which listing call. A search returns its top matches, so the catalog is what the searches reached, not a guaranteed full list"
16
+ },
17
+ "catalog/skipped": {
18
+ "command": "snapshot",
19
+ "severity": "info",
20
+ "text": "`--catalog`: tools that look like command routers but weren't called, because no listing mode needs no command. toolmenu never sends a command to list one"
21
+ },
22
+ "description/buried": {
23
+ "command": "snapshot",
24
+ "severity": "warn",
25
+ "text": "A sentence that reads as an instruction to the agent sits past the point where the client cuts descriptions, so the model never sees it. Read from the words, so marked unsure"
26
+ },
27
+ "description/cut": {
28
+ "command": "snapshot",
29
+ "severity": "info",
30
+ "text": "Descriptions longer than the client sends, with where the cut falls. Nothing past it reads as an instruction, so it's a check, not a bug"
31
+ },
32
+ "description/late-instruction": {
33
+ "command": "snapshot",
34
+ "severity": "info",
35
+ "text": "Only with the default cut: an instruction past 280 characters, where one real client cut, but inside the default cut. Split out of `description/cut` in 0.13; a setting for `description/cut` still applies to it unless it's set itself (`RULE_ALIASES`)"
36
+ },
37
+ "ids/authored": {
38
+ "command": "snapshot",
39
+ "severity": "info",
40
+ "text": "An input that looks like an opaque ID where no tool appears to return that kind of ID, so the agent would have to invent it. Heuristic, and worded as one. Stronger when `outputSchema` is present"
41
+ },
42
+ "ids/handle-lifetime": {
43
+ "command": "snapshot",
44
+ "severity": "info",
45
+ "text": "2026-07-28. A tool that creates a handle (`create`/`start`/`open`/`new`/`begin`/`acquire` + session, basket, cart, checkout, handle, lease or lock) that another tool takes back (`session_id`, `sessionToken`, `session`), whose description and output schema don't say how long it lasts. The spec asks for retention in the creation tool's description. Unsure"
46
+ },
47
+ "lists/schema": {
48
+ "command": "snapshot",
49
+ "severity": "error",
50
+ "text": "Each list's raw result pages against the official schema of the protocol version (`ListPromptsResult`, `ListResourcesResult`, `ListResourceTemplatesResult`); 2026-07-28 requires `ttlMs` and `cacheScope`. A declared list that fails to answer is a warning; Method not found (-32601) is info, and a setup failure (credentials, this machine) is info that says it wasn't checked. A list asked for whose capability the server doesn't declare isn't read, and the snapshot says so in a note, not a finding"
51
+ },
52
+ "lists/variance": {
53
+ "command": "snapshot",
54
+ "severity": "warn",
55
+ "text": "`snapshot --lists prompts,resources` (or `lists` in the config) reads `prompts/list`, `resources/list` and `resources/templates/list` (only those the server declares a capability for, up to 50 pages each, bypassing the SDK's list cache) twice on the main connection and once from a second process or connection (`--processes 1` turns that off). A difference by item (name, uri, uriTemplate), in content, key order or order is the finding"
56
+ },
57
+ "menu/auth-variance": {
58
+ "command": "snapshot",
59
+ "severity": "info",
60
+ "text": "`snapshot --as name:credential` (repeatable) or `identities` in the config lists the menu once per identity: a header set (HTTP) or env vars (stdio) over `--header`/`--env`. The first identity's menu is saved and checked; each other one, from its own process or connection, is compared with it: tools it adds, lacks, or defines differently, and its tokens. The tool set **MAY** vary by authorization, so this is info. An identity that can't be listed is reported here, and the others are still compared"
61
+ },
62
+ "menu/connection-variance": {
63
+ "command": "snapshot",
64
+ "severity": "error",
65
+ "text": "The same over HTTP: a second connection with the same credentials gets a different menu. Retried once before reporting"
66
+ },
67
+ "menu/duplicate-name": {
68
+ "command": "snapshot",
69
+ "severity": "error",
70
+ "text": "Two tools with the same name: only one can be called, and some clients reject the list"
71
+ },
72
+ "menu/gated": {
73
+ "command": "snapshot",
74
+ "severity": "warn",
75
+ "text": "The server declares `listChanged: true` and has a tool that unlocks more (an enum of toolsets or domains), and the unlock's values aren't already listed. The snapshot may be only the starting set, so a baseline made from it misses every tool behind the unlock (a real server: 18 of 115 tools). Run `session --auto` to walk the unlocks"
76
+ },
77
+ "menu/host-specific": {
78
+ "command": "snapshot",
79
+ "severity": "info",
80
+ "text": "A description holds a value that differs per host (a container, host or instance ID in hex), so the menu differs on every install and every machine misses the prompt cache. Heuristic"
81
+ },
82
+ "menu/nondeterministic": {
83
+ "command": "snapshot",
84
+ "severity": "error",
85
+ "text": "Two identical `tools/list` calls return different menus. Reports what differed: order, description, schema, annotations, tool added, tool removed. Order and content changes are both cache-relevant. Spec basis for order: servers **SHOULD** return \"the same ordering across requests when the underlying set of tools has not changed\" (§12). Content changes are toolmenu's own rule (lesson 05), not a spec violation"
86
+ },
87
+ "menu/process-variance": {
88
+ "command": "snapshot",
89
+ "severity": "error",
90
+ "text": "A second server process, started the same way, serves a different menu, so every restart misses the cache. Seen on mcp-atlassian: a default built from a Python set, in a new order every start (FINDINGS F12). toolmenu runs the main process with `PYTHONHASHSEED=0`, so a Python server's saved menu is reproducible, and the second with another seed. `--processes 1` turns it off"
91
+ },
92
+ "naming/client-name": {
93
+ "command": "snapshot",
94
+ "severity": "warn",
95
+ "text": "`--client` only. Tool names outside the client's API name format: the Claude API's `^[a-zA-Z0-9_-]{1,128}$` (platform.claude.com define-tools) for `claude-code` and `claude-api-search`, OpenAI's `FunctionToolParam.name` (`^[a-zA-Z0-9_-]+$`, maxLength 128, openai/openai-openapi) for `openai`. The detail names the source and read date"
96
+ },
97
+ "naming/route": {
98
+ "command": "snapshot",
99
+ "severity": "error",
100
+ "text": "A `routes.yml` expectation broke"
101
+ },
102
+ "naming/shared-word": {
103
+ "command": "snapshot",
104
+ "severity": "info",
105
+ "text": "The same noun is a primary term in tools from different areas (the \"audit\" collision). Lists groups for review, never fails the build"
106
+ },
107
+ "naming/vague-id": {
108
+ "command": "snapshot",
109
+ "severity": "warn",
110
+ "text": "A parameter named `id`, `ids` or `key` that doesn't say which kind of thing it identifies (the ~13-wasted-calls bug)"
111
+ },
112
+ "search/findable": {
113
+ "command": "snapshot",
114
+ "severity": "info",
115
+ "text": "For a menu a client would defer (10+ tools and 10K+ tokens, or 30+ tools): a tool that a BM25 search for its own name words doesn't put in the top 5, the number a tool search returns by default. With `routes.yml`, a `must_match` tool past the top 5 for its keyword is a warning. Ranked over the name, description, argument names and argument descriptions, as Claude's tool search reads them; the client's tokenizer and parameters aren't published, so marked unsure"
116
+ },
117
+ "spec/cache-hints": {
118
+ "command": "snapshot",
119
+ "severity": "info",
120
+ "text": "Cache metadata recommendation, not a correctness error (`since: 2026-07-28`). `ttlMs: 0` on `tools/list`: valid, but tells clients the list is \"immediately stale\" and they may re-fetch every time. `cacheScope: \"public\"` on a server that required auth: valid only if the list has no user-specific data, because shared caches \"MAY … serve it across authorization contexts\". Worth a look if the tool set depends on the caller's scopes"
121
+ },
122
+ "spec/cache-scope-public": {
123
+ "command": "snapshot",
124
+ "severity": "warn",
125
+ "text": "`since: 2026-07-28`. The tool set differs by identity and `tools/list` says `cacheScope: \"public\"` (shared caches **MAY** serve it across authorization contexts). With identities to compare, it replaces `spec/cache-hints`' public-scope hint, which has no evidence either way"
126
+ },
127
+ "spec/deprecated": {
128
+ "command": "snapshot",
129
+ "severity": "info",
130
+ "text": "Advertises Sampling, Roots or Logging, or uses HTTP+SSE (`since: 2026-07-28`)"
131
+ },
132
+ "spec/discover": {
133
+ "command": "snapshot",
134
+ "severity": "info",
135
+ "text": "The server only speaks the 2025 protocol: no working `server/discover`, which 2026-07-28 servers **MUST** implement. Info, not error: see §13"
136
+ },
137
+ "spec/ref": {
138
+ "command": "snapshot",
139
+ "severity": "error",
140
+ "text": "`since: 2026-07-28`. A `$ref` in a tool's input or output schema that doesn't resolve inside the schema: a network URI (clients **MUST NOT** fetch it, SEP-2106) or a pointer, `#anchor` or `$id` that names nothing. In an `outputSchema`, error: the official SDK client fails every call to the tool (\"has an invalid outputSchema\"). In an `inputSchema`, error for a ref to nothing (the schema isn't valid, and the SDK's server won't register it), warn for a network URI (clients **SHOULD** reject the schema). `#/$defs`, `#anchor`, `$dynamicAnchor`, `$id` (pointers resolve inside the resource an `$id` names), draft-07 `#name` ids, `\"\"` and `#` refs resolve. The token breakdown's unused-`$defs` count uses the same resolver. Composition-keyword bounds are left to each implementation by the spec, so not checked. Checked against the SDK client's behaviour in the tests"
141
+ },
142
+ "spec/schema": {
143
+ "command": "snapshot",
144
+ "severity": "error",
145
+ "text": "`tools/list` result fails the official `schema.json` for its version. Version-aware: a 2026-07-28 result missing `ttlMs`/`cacheScope` fails, because that version requires them on cacheable results (SEP-2549); a 2025 result is checked against its own schema, which doesn't have them"
146
+ },
147
+ "spec/server-info": {
148
+ "command": "snapshot",
149
+ "severity": "warn",
150
+ "text": "The handshake's `serverInfo` has no `name` or no `version`, so a client (and `diff --server-version-is-release`) can't tell which server or release answered"
151
+ },
152
+ "spec/tools-capability": {
153
+ "command": "snapshot",
154
+ "severity": "error",
155
+ "text": "The server doesn't declare the `tools` capability, so clients (the official SDK included) never ask for its tools"
156
+ },
157
+ "spec/x-mcp-header": {
158
+ "command": "snapshot",
159
+ "severity": "error",
160
+ "text": "`since: 2026-07-28`. An `x-mcp-header` annotation the spec doesn't allow (SEP-2243): empty; not an RFC 9110 token; the same header twice ignoring case; on a parameter that isn't `string`, `integer` or `boolean`, or has no single `type`; anywhere but a parameter reached through `properties` alone. Streamable HTTP clients **MUST** leave such a tool out of `tools/list`, and the official SDK client does, with only a console warning. Error on an HTTP server, warn on stdio (clients there MAY ignore the annotation). `number` is warn on both: the spec forbids it, the SDK client accepts it for now. Every broken annotation in a tool is listed (the SDK stops at the first). Checked against the SDK client's behaviour in the tests"
161
+ },
162
+ "write/no-dry-run": {
163
+ "command": "snapshot",
164
+ "severity": "info",
165
+ "text": "A `destructiveHint: true` or `readOnlyHint: false` tool with no `dry_run`/`preview`-style parameter and no matching preview tool"
166
+ },
167
+ "write/read-only-mismatch": {
168
+ "command": "snapshot",
169
+ "severity": "warn",
170
+ "text": "`readOnlyHint: true` on a tool whose name starts with a write verb (create, delete, send, upload, set…). Unsure: a tool can write only local state. `session --auto` skips such a tool, saying why, rather than call it on the hint alone"
171
+ },
172
+ "write/unannotated": {
173
+ "command": "snapshot",
174
+ "severity": "warn",
175
+ "text": "A tool whose name suggests writing (`delete_`, `send_`, `update_`…) has no annotations"
176
+ },
177
+ "diff/annotations": {
178
+ "command": "diff",
179
+ "severity": "info (notice)",
180
+ "text": "Other annotations changed (`title`, `idempotentHint`, `openWorldHint`, a hint removed). Grouped when the same change is in several tools"
181
+ },
182
+ "diff/catalog-added": {
183
+ "command": "diff",
184
+ "severity": "info",
185
+ "text": "Operations behind a search tool that are new, or newly found by the same queries"
186
+ },
187
+ "diff/catalog-missing": {
188
+ "command": "diff",
189
+ "severity": "info",
190
+ "text": "Operations behind a search tool (`--catalog`) that the same queries didn't return this time: removed, renamed, or ranked lower. A notice, no bump"
191
+ },
192
+ "diff/catalog-queries": {
193
+ "command": "diff",
194
+ "severity": "info",
195
+ "text": "Only one snapshot has a catalog, the two were read with different queries, or one is partial: what's on one side only says less"
196
+ },
197
+ "diff/description": {
198
+ "command": "diff",
199
+ "severity": "info (notice)",
200
+ "text": "A description changed, with a text diff. It doesn't break the protocol, but it changes what the agent does"
201
+ },
202
+ "diff/enum-narrowed": {
203
+ "command": "diff",
204
+ "severity": "error (breaking)",
205
+ "text": "A parameter now accepts fewer values, or became limited to a fixed set. Calls that send any other value fail validation"
206
+ },
207
+ "diff/enum-widened": {
208
+ "command": "diff",
209
+ "severity": "info (minor)",
210
+ "text": "A parameter accepts more values, or is no longer limited to a fixed set"
211
+ },
212
+ "diff/lists-unread": {
213
+ "command": "diff",
214
+ "severity": "info",
215
+ "text": "Only one menu was snapshotted with `--lists`, so prompts or resources weren't compared"
216
+ },
217
+ "diff/order": {
218
+ "command": "diff",
219
+ "severity": "info (notice)",
220
+ "text": "The tool order changed. It matters within a session (the prompt cache), not between releases"
221
+ },
222
+ "diff/other": {
223
+ "command": "diff",
224
+ "severity": "info (notice)",
225
+ "text": "Other tool fields changed (`title`, `icons`, `_meta`). Clients may show these; the agent may read them"
226
+ },
227
+ "diff/param-added": {
228
+ "command": "diff",
229
+ "severity": "info (minor)",
230
+ "text": "A new optional parameter"
231
+ },
232
+ "diff/param-default": {
233
+ "command": "diff",
234
+ "severity": "info (notice)",
235
+ "text": "A parameter's default changed: calls that leave it out now behave differently"
236
+ },
237
+ "diff/param-dropped": {
238
+ "command": "diff",
239
+ "severity": "info (notice)",
240
+ "text": "An optional parameter removed from an object that accepts properties it doesn't list: calls that still send it stay valid, but the server may ignore it"
241
+ },
242
+ "diff/param-relaxed": {
243
+ "command": "diff",
244
+ "severity": "info (minor)",
245
+ "text": "A required parameter became optional"
246
+ },
247
+ "diff/param-removed": {
248
+ "command": "diff",
249
+ "severity": "error (breaking)",
250
+ "text": "A required parameter removed (every existing call sends it, and the server no longer reads it), or an optional one removed from an object that rejects properties it doesn't list (calls that still send it fail validation)"
251
+ },
252
+ "diff/param-renamed": {
253
+ "command": "diff",
254
+ "severity": "warn",
255
+ "text": "A tool lost a parameter and gained a required one with a close name: likely a rename. A hint next to the two breaking findings, not a change of its own"
256
+ },
257
+ "diff/param-required": {
258
+ "command": "diff",
259
+ "severity": "error (breaking)",
260
+ "text": "A parameter that's new and required, or was optional and is now required. Calls that don't send it fail"
261
+ },
262
+ "diff/param-type": {
263
+ "command": "diff",
264
+ "severity": "error (breaking)",
265
+ "text": "A parameter's type changed to one that doesn't accept every old value (`string` → `integer`), or a `oneOf`/`anyOf` option was dropped"
266
+ },
267
+ "diff/prompt-added": {
268
+ "command": "diff",
269
+ "severity": "info (minor)",
270
+ "text": "`--lists`: a new prompt"
271
+ },
272
+ "diff/prompt-argument-required": {
273
+ "command": "diff",
274
+ "severity": "error (breaking)",
275
+ "text": "`--lists`: a prompt argument that's new and required, or now required. Invocations that don't pass it fail"
276
+ },
277
+ "diff/prompt-changed": {
278
+ "command": "diff",
279
+ "severity": "info (notice)",
280
+ "text": "`--lists`: a prompt's description or arguments changed, and nothing it requires"
281
+ },
282
+ "diff/prompt-removed": {
283
+ "command": "diff",
284
+ "severity": "error (breaking)",
285
+ "text": "`--lists`: a prompt removed. Users and clients that invoke it by name (a slash command, a saved workflow) get an error"
286
+ },
287
+ "diff/properties-closed": {
288
+ "command": "diff",
289
+ "severity": "error (breaking)",
290
+ "text": "`additionalProperties: false` added: calls that send extra arguments now fail validation"
291
+ },
292
+ "diff/properties-opened": {
293
+ "command": "diff",
294
+ "severity": "info (minor)",
295
+ "text": "`additionalProperties: false` removed: the tool accepts extra arguments"
296
+ },
297
+ "diff/resource-template-added": {
298
+ "command": "diff",
299
+ "severity": "info (minor)",
300
+ "text": "`--lists`: a new resource template"
301
+ },
302
+ "diff/resource-template-removed": {
303
+ "command": "diff",
304
+ "severity": "error (breaking)",
305
+ "text": "`--lists`: a resource template removed. Clients that read resources through it get an error"
306
+ },
307
+ "diff/resources": {
308
+ "command": "diff",
309
+ "severity": "info (notice)",
310
+ "text": "`--lists`: concrete resources that came or went. Resources are data (files come and go): listed, no bump"
311
+ },
312
+ "diff/safety-hint": {
313
+ "command": "diff",
314
+ "severity": "error (breaking)",
315
+ "text": "`readOnlyHint` true → false, or `destructiveHint` added. An agent or client that ran the tool without asking now has a write it didn't approve"
316
+ },
317
+ "diff/schema-dialect": {
318
+ "command": "diff",
319
+ "severity": "info (notice)",
320
+ "text": "Only the declared `$schema` dialect changed. One finding for the menu"
321
+ },
322
+ "diff/schema-equivalent": {
323
+ "command": "diff",
324
+ "severity": "info (notice)",
325
+ "text": "The schema was restructured or respelled (`$ref`, `$defs`, key order) but accepts the same input: bytes and tokens changed, nothing else"
326
+ },
327
+ "diff/schema-other": {
328
+ "command": "diff",
329
+ "severity": "info (notice)",
330
+ "text": "The input schema changed in keywords toolmenu doesn't classify, a schema that's empty or invalid, or one too large to expand through its `$ref`s. Review it by hand"
331
+ },
332
+ "diff/token-budget": {
333
+ "command": "diff",
334
+ "severity": "error",
335
+ "text": "The new menu is over `tokenBudget` from the config. Every conversation that loads it pays that before its first message"
336
+ },
337
+ "diff/tool-added": {
338
+ "command": "diff",
339
+ "severity": "info (minor)",
340
+ "text": "A new tool. It costs tokens in every conversation that loads the menu (see the token change)"
341
+ },
342
+ "diff/tool-removed": {
343
+ "command": "diff",
344
+ "severity": "error (breaking)",
345
+ "text": "A tool in the old menu isn't in the new one. Agents and saved workflows that call it get an error"
346
+ },
347
+ "diff/tool-renamed": {
348
+ "command": "diff",
349
+ "severity": "error (breaking)",
350
+ "text": "One tool removed and one added with a near-identical schema: reported as a rename, still breaking for anything that calls the old name"
351
+ },
352
+ "diff/type-widened": {
353
+ "command": "diff",
354
+ "severity": "info (minor)",
355
+ "text": "A parameter's type now accepts more (`integer` → `number`, a type added to a union)"
356
+ },
357
+ "diff/version-backwards": {
358
+ "command": "diff",
359
+ "severity": "warn",
360
+ "text": "The release version went backwards, so the bump can't be checked"
361
+ },
362
+ "diff/version-bump": {
363
+ "command": "diff",
364
+ "severity": "warn",
365
+ "text": "`--release old..new` (or `--server-version-is-release`): the bump is smaller than the changes need, a breaking change released as a minor or patch"
366
+ },
367
+ "diff/x-mcp-header": {
368
+ "command": "diff",
369
+ "severity": "info (notice)",
370
+ "text": "2026-07-28: a parameter mirrored into an `Mcp-Param-*` HTTP header was added, removed or renamed. Callers don't change; proxies and WAFs that route on the header do"
371
+ },
372
+ "diff/x-mcp-header-invalid": {
373
+ "command": "diff",
374
+ "severity": "warn (notice)",
375
+ "text": "The new release has an `x-mcp-header` annotation that breaks the spec's rules, so Streamable HTTP clients must leave the tool out of their menu. A warning, not breaking: a menu file doesn't say whether the server is served over HTTP"
376
+ },
377
+ "session/append": {
378
+ "command": "session",
379
+ "severity": "warn",
380
+ "text": "Tools added at the end of the list. Still a cache miss for the conversation when your client sends tools at the start of the prompt (Claude's Messages API does); cache-safe only when the client adds new tools after the cached content, as tool search (deferred loading) does"
381
+ },
382
+ "session/assumed-read-only": {
383
+ "command": "session",
384
+ "severity": "info",
385
+ "text": "Tools called on your word (`--assume-read-only`, `assume_read_only`) although the server doesn't mark them readOnlyHint"
386
+ },
387
+ "session/connection-local": {
388
+ "command": "session",
389
+ "severity": "error on 2026-07-28 (HTTP)",
390
+ "text": "A change only this connection sees. 2026-07-28: the tool set MUST NOT vary \"per-connection or as a side effect of other requests on the connection\""
391
+ },
392
+ "session/decline-fails": {
393
+ "command": "session",
394
+ "severity": "warn",
395
+ "text": "The retry carrying the decline fails with a JSON-RPC error, not a result the model can read. A tool error (`isError`) after a decline is the right answer and isn't reported"
396
+ },
397
+ "session/edit": {
398
+ "command": "session",
399
+ "severity": "error",
400
+ "text": "A tool's description, schema or annotations changed mid-session, with which value changed. The prompt cache is invalidated from that tool on"
401
+ },
402
+ "session/handle-error": {
403
+ "command": "session",
404
+ "severity": "warn",
405
+ "text": "A handle that can't exist answered with a JSON-RPC error, not a tool error (`isError`) the model can read and recover from by making a new one. A tool error, or an answer, isn't reported. One finding per tool; unsure"
406
+ },
407
+ "session/handle-roundtrip": {
408
+ "command": "session",
409
+ "severity": "warn, info when no handle was found",
410
+ "text": "`--create-handles`: the handle a creating tool just returned (`start_session` → `session_id`) refused by the tools that take it back (warn), or a creating tool whose result has no handle to find (info), so the made-up one was used instead"
411
+ },
412
+ "session/input-loop": {
413
+ "command": "session",
414
+ "severity": "warn",
415
+ "text": "After a decline the tool asks the same question again (the same kind and schema, whatever the request key or wording), or keeps asking something else, until the round limit; or it returns `requestState`-only rounds without end. An agent whose user says no can't finish the call"
416
+ },
417
+ "session/input-required": {
418
+ "command": "session",
419
+ "severity": "info",
420
+ "text": "Which tools asked for input, for what, how many rounds, and what the decline led to. One finding for the run"
421
+ },
422
+ "session/known-variance": {
423
+ "command": "session",
424
+ "severity": "info",
425
+ "text": "A mid-session change only in values that already vary from one tools/list to the next: same cause as the finding it names, not counted again"
426
+ },
427
+ "session/mid-insert": {
428
+ "command": "session",
429
+ "severity": "error",
430
+ "text": "Tools inserted into the menu mid-session anywhere but the end. Any change to the tool list invalidates the cached prompt: the whole tool list and the conversation after it are processed again"
431
+ },
432
+ "session/nothing-called": {
433
+ "command": "session",
434
+ "severity": "warn",
435
+ "text": "`--auto` called no tools (all open-world, missing values, or not read-only): the run only listed the menu"
436
+ },
437
+ "session/rate-limited": {
438
+ "command": "session",
439
+ "severity": "error",
440
+ "text": "The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it"
441
+ },
442
+ "session/refused": {
443
+ "command": "session",
444
+ "severity": "error",
445
+ "text": "A call the scenario didn't allow: a tool not marked `readOnlyHint` without `allow_writes`, or one marked `destructiveHint`. Nothing was sent"
446
+ },
447
+ "session/remove": {
448
+ "command": "session",
449
+ "severity": "error",
450
+ "text": "A tool gone from the menu mid-session. A model that already planned to call it gets an error, and the prompt cache is invalidated"
451
+ },
452
+ "session/reorder": {
453
+ "command": "session",
454
+ "severity": "error",
455
+ "text": "The same tools mid-session in a different order: a cache miss for everything after the first moved tool"
456
+ },
457
+ "session/result-type": {
458
+ "command": "session",
459
+ "severity": "error",
460
+ "text": "A 2026-07-28 `tools/call` result without `resultType` (**MUST**), or an `input_required` result whose `requestState` isn't a string (it can't be echoed as it came; toolmenu stops the call). The official SDK client rejects a result without `resultType`"
461
+ },
462
+ "session/scope-unchecked": {
463
+ "command": "session",
464
+ "severity": "info",
465
+ "text": "A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run"
466
+ },
467
+ "session/session-lost": {
468
+ "command": "session",
469
+ "severity": "error",
470
+ "text": "The server ended the session after toolmenu opened a second connection with the same credentials (servers with one session per client; 2025-era servers only). The run stops there; rerun with `--processes 1`"
471
+ },
472
+ "session/side-effect": {
473
+ "command": "session",
474
+ "severity": "warn on 2026-07-28 (stdio)",
475
+ "text": "The same, where stdio can't tell per-connection from global"
476
+ },
477
+ "session/step-failed": {
478
+ "command": "session",
479
+ "severity": "error",
480
+ "text": "A step that failed for a reason that isn't setup (authentication, network, missing software): a JSON-RPC error, a timeout, or a server that exited. A server that stops answering (`tools/list` times out after a step) ends the run there, with one finding"
481
+ },
482
+ "session/tool-error": {
483
+ "command": "session",
484
+ "severity": "warn",
485
+ "text": "A tool that answered with an error for another reason (`isError`): the run tested less than it looks. The same error at several steps is one finding that names them"
486
+ },
487
+ "session/unannounced": {
488
+ "command": "session",
489
+ "severity": "warn",
490
+ "text": "The menu changed, the server declared `listChanged: true`, and no `notifications/tools/list_changed` arrived. The spec says servers **SHOULD** send it"
491
+ },
492
+ "session/undeclared-input": {
493
+ "command": "session",
494
+ "severity": "warn",
495
+ "text": "The server asked for something the request didn't declare (roots, sampling) instead of returning `-32021`"
496
+ },
497
+ "session/unlock-coverage": {
498
+ "command": "session",
499
+ "severity": "warn",
500
+ "text": "An unlock whose enum lists its values, called with only some of them: the tools behind the rest were never seen, so a baseline from the session misses them. With `--union-out` or `--auto`, an unlock never called at all too, with why `--auto` skipped it"
501
+ },
502
+ "session/untested": {
503
+ "command": "session",
504
+ "severity": "warn, error if no call got through",
505
+ "text": "Calls that failed before reaching the tool: authentication, something missing on this machine (no Chrome), the network; or an answer, not marked `isError`, that is the API's 401 or 403 passed on as JSON (`{\"status\":401,\"code\":\"unauthorized\"}`, Notion's server). One finding for the run, not one per step, so an expired CI secret doesn't pass a run that tested nothing"
506
+ }
507
+ }
@@ -0,0 +1,18 @@
1
+ import type { MenuTool } from '../types.js';
2
+ import { type Rule } from './rule.js';
3
+ /** What an HTTP server answered to the same listing with an invalid credential. */
4
+ export interface InvalidCredential {
5
+ /** It served a menu anyway. */
6
+ tools?: MenuTool[];
7
+ /** It refused with this HTTP status (401 or 403 is the answer a client can act on). */
8
+ status?: number;
9
+ /** It failed some other way. */
10
+ error?: string;
11
+ }
12
+ /**
13
+ * A credential that can't be valid, in place of the one this run used. A server that
14
+ * answers it with a menu, not 401, gives a client whose token expired or was revoked
15
+ * no reason to log in again: the agent just has fewer tools (Atlassian Rovo served 4
16
+ * of its tools to any Authorization header, 2026-11-03).
17
+ */
18
+ export declare const invalidCredential: Rule;
@@ -0,0 +1,29 @@
1
+ import { count, names, num } from './rule.js';
2
+ /**
3
+ * A credential that can't be valid, in place of the one this run used. A server that
4
+ * answers it with a menu, not 401, gives a client whose token expired or was revoked
5
+ * no reason to log in again: the agent just has fewer tools (Atlassian Rovo served 4
6
+ * of its tools to any Authorization header, 2026-11-03).
7
+ */
8
+ export const invalidCredential = {
9
+ id: 'auth/invalid-accepted',
10
+ severity: 'warn',
11
+ summary: 'An invalid credential gets an auth error, not a menu',
12
+ run(ctx) {
13
+ const probe = ctx.invalidCredential;
14
+ if (!probe?.tools)
15
+ return [];
16
+ const mine = ctx.menu.tools.map((t) => t.name);
17
+ const theirs = probe.tools.map((t) => t.name);
18
+ const same = theirs.length === mine.length && theirs.every((n) => mine.includes(n));
19
+ return [
20
+ {
21
+ message: same
22
+ ? `With an invalid credential in place of this run's, the server served the same ${count(theirs.length, 'tool')} instead of refusing (401). Listing doesn't check credentials, so a client learns its token expired or was revoked only when a call fails.`
23
+ : `With an invalid credential in place of this run's, the server served ${count(theirs.length, 'tool')} (of ${num(mine.length)} with it) instead of refusing (401). A client whose token expired or was revoked gets a smaller menu, not an auth error, so it has no reason to ask the user to log in again; the agent just has fewer tools.`,
24
+ detail: same ? [] : [`served to the invalid credential: ${names(theirs, 10)}`],
25
+ fix: "Answer a request whose credential doesn't verify with 401 and a WWW-Authenticate header, even for tools that need no login: a client re-authenticates on 401. If some tools are meant to work without a login, serve them to requests with no credential, not to ones with a bad credential.",
26
+ },
27
+ ];
28
+ },
29
+ };
@@ -4,7 +4,20 @@ import { type Rule } from './rule.js';
4
4
  export interface HandleProbe {
5
5
  param: string;
6
6
  kind: string;
7
+ /**
8
+ * --create-handles: a creating tool runs first, and this call gets the handle it
9
+ * returned instead of a made-up one (the made-up one when it returned none).
10
+ */
11
+ fromCreated?: string;
7
12
  }
13
+ /** Kinds --create-handles may create: scratch state that expires. Not a lock, lease or checkout. */
14
+ export declare const CREATABLE_KINDS: Set<string>;
15
+ /** The tool that creates handles of this kind, if the menu has one. */
16
+ export declare function creatorOf(tools: MenuTool[], kind: string): MenuTool | undefined;
17
+ export declare function handleFromResult(result: {
18
+ structuredContent?: unknown;
19
+ content?: unknown;
20
+ }, kind: string): string | undefined;
8
21
  /** The handle kinds some tool in the menu creates. */
9
22
  export declare function createdKinds(tools: MenuTool[]): Set<string>;
10
23
  /** Parameters of a tool that take a handle of this kind: session_id, cartToken, `session`. */