toolmenu 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -10
- package/dist/cli.js +10 -3
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/diff.js +39 -6
- package/dist/difference.d.ts +17 -0
- package/dist/difference.js +70 -0
- package/dist/failures.d.ts +14 -0
- package/dist/failures.js +47 -0
- package/dist/history.js +3 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/menu.d.ts +7 -0
- package/dist/menu.js +14 -5
- package/dist/probe.d.ts +27 -0
- package/dist/probe.js +58 -0
- package/dist/report.js +7 -1
- package/dist/rules/determinism.d.ts +23 -1
- package/dist/rules/determinism.js +72 -2
- package/dist/rules/ids.d.ts +2 -0
- package/dist/rules/ids.js +34 -12
- package/dist/rules/index.js +3 -1
- package/dist/rules/naming.js +2 -1
- package/dist/rules/rule.d.ts +6 -0
- package/dist/session.d.ts +6 -0
- package/dist/session.js +74 -43
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +28 -18
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
|
|
|
29
29
|
(`history` installs old versions with today's dependencies, so it records what they
|
|
30
30
|
serve now, which can differ from what they shipped with: FINDINGS F4.)
|
|
31
31
|
|
|
32
|
-
> **Status: 0.
|
|
32
|
+
> **Status: 0.8, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
|
|
33
33
|
> real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
34
34
|
|
|
35
35
|
## Start here
|
|
@@ -67,7 +67,7 @@ jobs:
|
|
|
67
67
|
- uses: actions/setup-node@v7
|
|
68
68
|
with: { node-version: 22 }
|
|
69
69
|
- run: npm ci && npm run build
|
|
70
|
-
- uses: niksa90/toolmenu@v0.
|
|
70
|
+
- uses: niksa90/toolmenu@v0.8.0
|
|
71
71
|
with:
|
|
72
72
|
command: node dist/server.js
|
|
73
73
|
baseline: menu.json # your committed snapshot
|
|
@@ -93,6 +93,7 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
93
93
|
| `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
|
|
94
94
|
|
|
95
95
|
Server behind auth, or PRs from forks and Dependabot? See [docs/github-action.md](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md).
|
|
96
|
+
Server that unlocks tools on request? Snapshot the full menu for the baseline and run the session on the default one: [servers that unlock tools](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#servers-that-unlock-tools).
|
|
96
97
|
PRs without secrets are skipped, and a skipped check still passes; the `skipped` output tells you.
|
|
97
98
|
|
|
98
99
|
## `session`: the menu changing while the agent works
|
|
@@ -121,14 +122,15 @@ as its steps, so the starter is a floor, not a ceiling.
|
|
|
121
122
|
|
|
122
123
|
| Rule | Default | Catches |
|
|
123
124
|
|---|---|---|
|
|
124
|
-
| `session/mid-insert`, `session/reorder`, `session/remove`, `session/edit` | error | The menu changing mid-session anywhere but the end, with the
|
|
125
|
+
| `session/mid-insert`, `session/reorder`, `session/remove`, `session/edit` | error | The menu changing mid-session anywhere but the end, with where the change starts and which value changed. Any change to the tool list invalidates the cached prompt: the whole tool list (its size, estimated) and the conversation after it are processed again |
|
|
125
126
|
| `session/append` | warn | Tools added at the end of the list. Still a cache miss for the conversation when your client sends tools at the start of the prompt (Claude's Messages API does); cache-safe only when the client adds new tools after the cached content, as tool search (deferred loading) does |
|
|
126
127
|
| `session/connection-local` | error on 2026-07-28 (HTTP) | A change only this connection sees. 2026-07-28: the tool set MUST NOT vary "per-connection or as a side effect of other requests on the connection" |
|
|
127
128
|
| `session/side-effect` | warn on 2026-07-28 (stdio) | The same, where stdio can't tell per-connection from global |
|
|
128
|
-
| `
|
|
129
|
+
| `menu/process-variance`, `menu/connection-variance` | error | Checked before the first step, as in `snapshot` (below) |
|
|
129
130
|
| `session/unannounced` | warn | The menu changed without `notifications/tools/list_changed`, although the server declared `listChanged` |
|
|
130
|
-
| `session/
|
|
131
|
-
| `session/
|
|
131
|
+
| `session/untested` | warn, error if no call got through | Calls that failed before reaching the tool: authentication, something missing on this machine (no Chrome), the network. One finding for the run, not one per step, so an expired CI secret doesn't pass a run that tested nothing |
|
|
132
|
+
| `session/refused`, `session/step-failed` | error | A write the scenario didn't allow, or a call that failed for another reason |
|
|
133
|
+
| `session/tool-error` | warn | A tool that answered with an error for another reason (`isError`): the run tested less than it looks |
|
|
132
134
|
|
|
133
135
|
**`session` calls tools for real.** Without `allow_writes: true` it refuses any tool
|
|
134
136
|
not marked `readOnlyHint: true`. `--plan` prints the steps without connecting.
|
|
@@ -142,7 +144,7 @@ npx toolmenu diff menu.json new-menu.json
|
|
|
142
144
|
```
|
|
143
145
|
$ npx toolmenu diff --release 2026.1.14..2026.8.31 fs-2026.1.14.json fs-2026.8.31.json
|
|
144
146
|
toolmenu diff secure-filesystem-server 2026.1.14 → 2026.8.31
|
|
145
|
-
14 → 14 tools · ~
|
|
147
|
+
14 → 14 tools · ~1,640 → ~1,664 tokens (+24, estimate): this release adds ~24 tokens to every conversation that loads the menu
|
|
146
148
|
1 breaking · 0 minor · 15 notice · suggested bump: major · actual: not checked (calendar version)
|
|
147
149
|
|
|
148
150
|
ERROR diff/safety-hint
|
|
@@ -195,13 +197,15 @@ server, or from the spec itself. If a rule can't point to one, it doesn't ship.
|
|
|
195
197
|
|
|
196
198
|
| Rule | Default | Catches |
|
|
197
199
|
|---|---|---|
|
|
198
|
-
| `menu/nondeterministic` | error | Two identical `tools/list` calls returning different menus (order, descriptions, schemas, annotations) |
|
|
200
|
+
| `menu/nondeterministic` | error | Two identical `tools/list` calls returning different menus (order, descriptions, schemas, annotations), with the value that differs |
|
|
201
|
+
| `menu/process-variance` | error | A second server process, started the same way, serves a different menu, so every restart misses the cache. Seen on mcp-atlassian: a default built from a Python set, in a new order every start (FINDINGS F12). toolmenu runs the main process with `PYTHONHASHSEED=0`, so a Python server's saved menu is reproducible, and the second with another seed. `--processes 1` turns it off |
|
|
202
|
+
| `menu/connection-variance` | error (warn on 2025 protocols) | The same over HTTP: a second connection with the same credentials gets a different menu. Retried once before reporting |
|
|
199
203
|
| `spec/schema` | error | A `tools/list` result that fails the official schema for its protocol version |
|
|
200
204
|
| `naming/route` | error | A `routes.yml` expectation broke: a keyword now matches the wrong tool at least as well as the right one |
|
|
201
205
|
| `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions: 2,048 characters in Claude Code, or your `descriptionLimit`. From a real failure: a client cut at 280 characters, and the line that decided routing was at 1,222 |
|
|
202
206
|
| `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary. The model gets a prefix that can read as complete |
|
|
203
207
|
| `naming/vague-id` | warn | A parameter called just `id` that doesn't say *which* thing |
|
|
204
|
-
| `ids/authored` |
|
|
208
|
+
| `ids/authored` | info | An ID the agent must supply that no tool appears to return, so the agent may invent it, one finding per kind of ID (heuristic: IDs from a URL or another tool's text output look the same) |
|
|
205
209
|
| `write/unannotated` | warn | A tool named like a write (`delete_`, `send_`…) with no annotations |
|
|
206
210
|
| `naming/shared-word` | info | Nouns that name different things across tools (`list_team_audits` vs `get_audit_trail`), as one summary |
|
|
207
211
|
| `write/no-dry-run` | info | Destructive tools with no dry-run parameter or preview tool, as one summary |
|
|
@@ -280,6 +284,7 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
280
284
|
--env <K=V> env var for a stdio server, repeatable (it only gets a
|
|
281
285
|
minimal environment otherwise)
|
|
282
286
|
--timeout <ms> per-request timeout (default: 30000)
|
|
287
|
+
--processes <n> server processes or connections to compare (default: 2)
|
|
283
288
|
```
|
|
284
289
|
|
|
285
290
|
Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
|
|
@@ -290,7 +295,9 @@ or bad usage. `toolmenu --help` lists the `session` and `history` options.
|
|
|
290
295
|
- **Server order isn't final client order.** Clients merge tools from several servers
|
|
291
296
|
and may sort or filter them. A stable server menu is necessary for cache hits, not
|
|
292
297
|
sufficient. toolmenu measures what the server controls.
|
|
293
|
-
- **Token counts are estimates** (`o200k_base`)
|
|
298
|
+
- **Token counts are estimates** (`o200k_base`) of what the model reads: each tool's
|
|
299
|
+
name, description and input schema. Output schemas, annotations, icons and `_meta`
|
|
300
|
+
stay with the client and aren't counted. Vendors tokenize, cache and bill
|
|
294
301
|
differently.
|
|
295
302
|
- **Heuristic rules say so** in their messages.
|
|
296
303
|
- Not a security scanner or a full conformance suite. Other tools do those well.
|
package/dist/cli.js
CHANGED
|
@@ -13,6 +13,7 @@ import { buildMenu } from './menu.js';
|
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { loadRoutes } from './routes.js';
|
|
15
15
|
import { snapshot } from './snapshot.js';
|
|
16
|
+
import { MAIN_SEED, seeded } from './probe.js';
|
|
16
17
|
import { SEVERITY_RANK } from './types.js';
|
|
17
18
|
import { VERSION } from './version.js';
|
|
18
19
|
const HELP = `toolmenu ${VERSION}
|
|
@@ -44,6 +45,8 @@ Options:
|
|
|
44
45
|
--env <K=V> environment variable for a stdio server, repeatable. The server
|
|
45
46
|
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
46
47
|
--timeout <ms> per-request timeout (default: 30000)
|
|
48
|
+
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
49
|
+
the main one included (default: 2; 1 turns the check off)
|
|
47
50
|
-h, --help show this help
|
|
48
51
|
-v, --version show the version
|
|
49
52
|
|
|
@@ -93,6 +96,7 @@ export async function main(argv) {
|
|
|
93
96
|
header: { type: 'string', multiple: true },
|
|
94
97
|
env: { type: 'string', multiple: true },
|
|
95
98
|
timeout: { type: 'string' },
|
|
99
|
+
processes: { type: 'string' },
|
|
96
100
|
versions: { type: 'string' },
|
|
97
101
|
release: { type: 'string' },
|
|
98
102
|
'server-version-is-release': { type: 'boolean' },
|
|
@@ -132,6 +136,9 @@ export async function main(argv) {
|
|
|
132
136
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
133
137
|
throw new UsageError(`--timeout must be a positive number of milliseconds`);
|
|
134
138
|
const config = await loadConfig(values.config);
|
|
139
|
+
const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
|
|
140
|
+
if (!Number.isInteger(processes) || processes < 1)
|
|
141
|
+
throw new UsageError('--processes must be a whole number, 1 or more');
|
|
135
142
|
if (sub === 'diff') {
|
|
136
143
|
if (command)
|
|
137
144
|
throw new UsageError('diff compares two menu files; it doesn\'t start a server. Run snapshot first.');
|
|
@@ -187,7 +194,7 @@ export async function main(argv) {
|
|
|
187
194
|
if (existsSync(path))
|
|
188
195
|
throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
|
|
189
196
|
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
190
|
-
const conn = await connect(initTarget, { timeoutMs });
|
|
197
|
+
const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
|
|
191
198
|
try {
|
|
192
199
|
const list = await listTools(conn, { timeoutMs });
|
|
193
200
|
const menu = buildMenu(list.tools, { name: conn.server.name, version: conn.server.version, protocolVersion: conn.protocolVersion });
|
|
@@ -214,7 +221,7 @@ export async function main(argv) {
|
|
|
214
221
|
return 0;
|
|
215
222
|
}
|
|
216
223
|
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
217
|
-
const result = await session(sessionTarget, scenario, { timeoutMs, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
224
|
+
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
218
225
|
const output = formatSession(result, format);
|
|
219
226
|
if (output)
|
|
220
227
|
process.stdout.write(output + '\n');
|
|
@@ -223,7 +230,7 @@ export async function main(argv) {
|
|
|
223
230
|
const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
224
231
|
const routesPath = values.routes ?? config.routes;
|
|
225
232
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
226
|
-
const { menu, findings } = await snapshot(target, { routes, timeoutMs, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
233
|
+
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
227
234
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
228
235
|
if (outPath)
|
|
229
236
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
package/dist/config.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ export interface Config {
|
|
|
14
14
|
descriptionLimit?: number | string;
|
|
15
15
|
/** Tools (names or globs) your client sends with the full description. */
|
|
16
16
|
fullDescriptions?: string[];
|
|
17
|
+
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
18
|
+
processes?: number;
|
|
17
19
|
}
|
|
18
20
|
/** Optional: no config file is fine. */
|
|
19
21
|
export declare function loadConfig(path?: string): Promise<Config>;
|
package/dist/config.js
CHANGED
|
@@ -23,5 +23,8 @@ export async function loadConfig(path) {
|
|
|
23
23
|
if (limit !== undefined && !(typeof limit === 'number' && limit > 0) && !(typeof limit === 'string' && limit in CLIENT_LIMITS)) {
|
|
24
24
|
throw new Error(`${file}: descriptionLimit must be a positive number or one of ${Object.keys(CLIENT_LIMITS).join(', ')}`);
|
|
25
25
|
}
|
|
26
|
+
if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
|
|
27
|
+
throw new Error(`${file}: processes must be a whole number, 1 or more`);
|
|
28
|
+
}
|
|
26
29
|
return config;
|
|
27
30
|
}
|
package/dist/diff.js
CHANGED
|
@@ -147,12 +147,12 @@ function compareTool(old, t) {
|
|
|
147
147
|
const before = oldProps[p];
|
|
148
148
|
if (!before) {
|
|
149
149
|
out.push(newReq.has(p)
|
|
150
|
-
? { rule: 'diff/param-required', tool: name, message: `${name}.${p} is new and required. Existing calls don't send it.` }
|
|
150
|
+
? { rule: 'diff/param-required', tool: name, message: `${name}.${p} is new and required. Existing calls don't send it.${hasDefault(schema) ? ` It has a default (${show(schema.default)}), so the server may still accept calls without it, but clients that validate arguments won't.` : ''}` }
|
|
151
151
|
: { rule: 'diff/param-added', tool: name, message: `${name}.${p} is a new optional parameter.` });
|
|
152
152
|
continue;
|
|
153
153
|
}
|
|
154
154
|
if (!oldReq.has(p) && newReq.has(p)) {
|
|
155
|
-
out.push({ rule: 'diff/param-required', tool: name, message: `${name}.${p} was optional and is now required.` });
|
|
155
|
+
out.push({ rule: 'diff/param-required', tool: name, message: `${name}.${p} was optional and is now required.${hasDefault(schema) ? ` It has a default (${show(schema.default)}), so the server may still accept calls without it, but clients that validate arguments won't.` : ''}` });
|
|
156
156
|
}
|
|
157
157
|
else if (oldReq.has(p) && !newReq.has(p)) {
|
|
158
158
|
out.push({ rule: 'diff/param-relaxed', tool: name, message: `${name}.${p} was required and is now optional.` });
|
|
@@ -172,7 +172,7 @@ function compareTool(old, t) {
|
|
|
172
172
|
}
|
|
173
173
|
// Array parameters: the element type and allowed values count like the
|
|
174
174
|
// parameter's own (no items schema = any element).
|
|
175
|
-
const isArray = (s) => (
|
|
175
|
+
const isArray = (s) => typesOf(s)?.includes('array') ?? false;
|
|
176
176
|
if (isArray(before) && isArray(schema)) {
|
|
177
177
|
const oldItems = before.items ?? {};
|
|
178
178
|
const newItems = schema.items ?? {};
|
|
@@ -247,6 +247,9 @@ function readOnly(a) {
|
|
|
247
247
|
function destructive(a) {
|
|
248
248
|
return !readOnly(a) && a.destructiveHint !== false;
|
|
249
249
|
}
|
|
250
|
+
function hasDefault(schema) {
|
|
251
|
+
return 'default' in schema;
|
|
252
|
+
}
|
|
250
253
|
function show(value) {
|
|
251
254
|
return value === undefined ? '(unset)' : JSON.stringify(value);
|
|
252
255
|
}
|
|
@@ -277,7 +280,10 @@ function paramKey(t) {
|
|
|
277
280
|
}
|
|
278
281
|
/** Does `next` accept every value type that `prev` accepted? */
|
|
279
282
|
function accepts(next, prev) {
|
|
280
|
-
const types = (s) =>
|
|
283
|
+
const types = (s) => {
|
|
284
|
+
const t = typesOf(s);
|
|
285
|
+
return t && new Set(t);
|
|
286
|
+
};
|
|
281
287
|
const n = types(next);
|
|
282
288
|
const p = types(prev);
|
|
283
289
|
if (!n)
|
|
@@ -289,6 +295,9 @@ function accepts(next, prev) {
|
|
|
289
295
|
/** A parameter's schema without the parts diff classifies itself. */
|
|
290
296
|
function residual(schema) {
|
|
291
297
|
const { type: _t, enum: _e, description: _d, items, ...rest } = schema;
|
|
298
|
+
// A type written as anyOf/oneOf alternatives is compared as the type.
|
|
299
|
+
if (typeAlternatives(schema))
|
|
300
|
+
delete rest[schema.anyOf ? 'anyOf' : 'oneOf'];
|
|
292
301
|
if (items && typeof items === 'object') {
|
|
293
302
|
const { enum: _ie, type: _it, ...itemRest } = items;
|
|
294
303
|
return Object.keys(itemRest).length ? { ...rest, items: itemRest } : rest;
|
|
@@ -296,8 +305,32 @@ function residual(schema) {
|
|
|
296
305
|
return rest;
|
|
297
306
|
}
|
|
298
307
|
function typeOf(schema) {
|
|
299
|
-
|
|
300
|
-
|
|
308
|
+
return [...new Set(typesOf(schema) ?? [])].sort().join('|');
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* The types a schema accepts, undefined for any. `anyOf: [{type: string}, {type:
|
|
312
|
+
* null}]` is the same as `type: [string, null]`: schema generators switch between
|
|
313
|
+
* the two (zod did), and that isn't a change.
|
|
314
|
+
*/
|
|
315
|
+
function typesOf(schema) {
|
|
316
|
+
if (schema.type !== undefined)
|
|
317
|
+
return Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
318
|
+
return typeAlternatives(schema);
|
|
319
|
+
}
|
|
320
|
+
/** anyOf/oneOf whose branches only name a type (a description or title aside). */
|
|
321
|
+
function typeAlternatives(schema) {
|
|
322
|
+
const branches = (schema.anyOf ?? schema.oneOf);
|
|
323
|
+
if (!Array.isArray(branches) || branches.length === 0)
|
|
324
|
+
return undefined;
|
|
325
|
+
const types = [];
|
|
326
|
+
for (const b of branches) {
|
|
327
|
+
if (!b || typeof b !== 'object' || b.type === undefined)
|
|
328
|
+
return undefined;
|
|
329
|
+
if (Object.keys(b).some((k) => !['type', 'description', 'title'].includes(k)))
|
|
330
|
+
return undefined;
|
|
331
|
+
types.push(...(Array.isArray(b.type) ? b.type : [b.type]));
|
|
332
|
+
}
|
|
333
|
+
return types;
|
|
301
334
|
}
|
|
302
335
|
function enumChange(before, after) {
|
|
303
336
|
const a = before.enum;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { MenuTool } from './types.js';
|
|
2
|
+
export interface Difference {
|
|
3
|
+
/** JSON path of the first value that differs, from the tool (`inputSchema.properties.fields.default`). */
|
|
4
|
+
path: string;
|
|
5
|
+
before: unknown;
|
|
6
|
+
after: unknown;
|
|
7
|
+
/** Same items in a different order: a comma-separated string or an array. */
|
|
8
|
+
reordered: boolean;
|
|
9
|
+
}
|
|
10
|
+
/** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
|
|
11
|
+
export declare function firstDifference(a: unknown, b: unknown, path?: string): Difference | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* One line saying what differs in a tool between two menus, for a finding's
|
|
14
|
+
* detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
|
|
15
|
+
* different order ("assignee,…" vs "issuetype,…")`.
|
|
16
|
+
*/
|
|
17
|
+
export declare function describeToolDifference(before: MenuTool, after: MenuTool): string;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { canonical } from './compare.js';
|
|
2
|
+
import { toolDefinition } from './menu.js';
|
|
3
|
+
/** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
|
|
4
|
+
export function firstDifference(a, b, path = '') {
|
|
5
|
+
if (canonical(a) === canonical(b))
|
|
6
|
+
return undefined;
|
|
7
|
+
if (isObject(a) && isObject(b)) {
|
|
8
|
+
for (const key of [...new Set([...Object.keys(a), ...Object.keys(b)])].sort()) {
|
|
9
|
+
const found = firstDifference(a[key], b[key], join(path, key));
|
|
10
|
+
if (found)
|
|
11
|
+
return found;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
if (Array.isArray(a) && Array.isArray(b) && a.length === b.length && !samePermutation(a, b)) {
|
|
15
|
+
for (let i = 0; i < a.length; i++) {
|
|
16
|
+
const found = firstDifference(a[i], b[i], `${path}[${i}]`);
|
|
17
|
+
if (found)
|
|
18
|
+
return found;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return { path, before: a, after: b, reordered: samePermutation(a, b) };
|
|
22
|
+
}
|
|
23
|
+
/** Different order, same items: arrays, or strings that are comma-separated lists. */
|
|
24
|
+
function samePermutation(a, b) {
|
|
25
|
+
const items = (v) => {
|
|
26
|
+
if (Array.isArray(v))
|
|
27
|
+
return v.map((x) => canonical(x));
|
|
28
|
+
if (typeof v === 'string' && v.includes(','))
|
|
29
|
+
return v.split(',').map((s) => s.trim());
|
|
30
|
+
return undefined;
|
|
31
|
+
};
|
|
32
|
+
const x = items(a);
|
|
33
|
+
const y = items(b);
|
|
34
|
+
if (!x || !y || x.length !== y.length || x.length < 2)
|
|
35
|
+
return false;
|
|
36
|
+
return canonical([...x].sort()) === canonical([...y].sort()) && canonical(x) !== canonical(y);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* One line saying what differs in a tool between two menus, for a finding's
|
|
40
|
+
* detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
|
|
41
|
+
* different order ("assignee,…" vs "issuetype,…")`.
|
|
42
|
+
*/
|
|
43
|
+
export function describeToolDifference(before, after) {
|
|
44
|
+
const a = toolDefinition(before);
|
|
45
|
+
const b = toolDefinition(after);
|
|
46
|
+
const d = firstDifference(a, b);
|
|
47
|
+
if (!d) {
|
|
48
|
+
return JSON.stringify(a) === JSON.stringify(b)
|
|
49
|
+
? `${after.name}: identical`
|
|
50
|
+
: `${after.name}: same content, keys in a different order (the bytes differ)`;
|
|
51
|
+
}
|
|
52
|
+
const where = d.path || '(the whole tool)';
|
|
53
|
+
if (d.reordered) {
|
|
54
|
+
const n = Array.isArray(d.before) ? d.before.length : String(d.before).split(',').length;
|
|
55
|
+
return `${after.name}: ${where}: same ${n} items, different order (${show(d.before)} vs ${show(d.after)}). Sort them.`;
|
|
56
|
+
}
|
|
57
|
+
return `${after.name}: ${where}: ${show(d.before)} vs ${show(d.after)}`;
|
|
58
|
+
}
|
|
59
|
+
function show(value) {
|
|
60
|
+
if (value === undefined)
|
|
61
|
+
return '(absent)';
|
|
62
|
+
const text = JSON.stringify(value);
|
|
63
|
+
return text.length > 60 ? text.slice(0, 57) + '…' : text;
|
|
64
|
+
}
|
|
65
|
+
function isObject(v) {
|
|
66
|
+
return !!v && typeof v === 'object' && !Array.isArray(v);
|
|
67
|
+
}
|
|
68
|
+
function join(path, key) {
|
|
69
|
+
return /^[A-Za-z_$][\w$]*$/.test(key) ? (path ? `${path}.${key}` : key) : `${path}[${JSON.stringify(key)}]`;
|
|
70
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'invalid-arguments' | 'other';
|
|
2
|
+
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
3
|
+
export declare const SETUP_FAILURES: Set<FailureClass>;
|
|
4
|
+
/**
|
|
5
|
+
* Classify a failed call. `code` is the JSON-RPC error code when the call threw.
|
|
6
|
+
* `hadArguments`: a 404 on a call that named nothing (no arguments) means the
|
|
7
|
+
* account or site behind the credentials isn't there (HubSpot answers a bad token
|
|
8
|
+
* with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
|
|
9
|
+
*/
|
|
10
|
+
export declare function classifyFailure(text: string, options?: {
|
|
11
|
+
code?: number;
|
|
12
|
+
hadArguments?: boolean;
|
|
13
|
+
}): FailureClass;
|
|
14
|
+
export declare const FAILURE_LABELS: Record<FailureClass, string>;
|
package/dist/failures.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Why a tool call failed, from its error text. The patterns come from real
|
|
3
|
+
* servers run with dummy credentials (FINDINGS F12): a failure that stops a call
|
|
4
|
+
* before the tool's own logic runs says nothing about the menu, and shouldn't read
|
|
5
|
+
* as a finding.
|
|
6
|
+
*/
|
|
7
|
+
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
8
|
+
export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found']);
|
|
9
|
+
// Something this machine lacks: "Could not find Google Chrome executable"
|
|
10
|
+
// (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright).
|
|
11
|
+
const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b/i;
|
|
12
|
+
// "401 Bad credentials" (GitHub), "Authorization Expired" (Sentry), "Invalid
|
|
13
|
+
// token" (Firecrawl), "authentication token is not valid" (Apify), "Please provide
|
|
14
|
+
// a valid access token" (Supabase). Not a bare "token": plenty of errors mention one.
|
|
15
|
+
const AUTH = /\b(401|403)\b|unauthori[sz]ed|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|valid (access )?token/i;
|
|
16
|
+
const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
|
|
17
|
+
// "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
|
|
18
|
+
// "**Input Error**" (Sentry), JSON-RPC's invalid params.
|
|
19
|
+
const INVALID_ARGUMENTS = /invalid (params|parameters|arguments|input)|input error|validation (error|failed)|must be provided|is required\b|\beither\b[^.]{0,80}\bor\b[^.]{0,80}\b(must|required)/i;
|
|
20
|
+
const NOT_FOUND = /\b404\b|not found/i;
|
|
21
|
+
/**
|
|
22
|
+
* Classify a failed call. `code` is the JSON-RPC error code when the call threw.
|
|
23
|
+
* `hadArguments`: a 404 on a call that named nothing (no arguments) means the
|
|
24
|
+
* account or site behind the credentials isn't there (HubSpot answers a bad token
|
|
25
|
+
* with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
|
|
26
|
+
*/
|
|
27
|
+
export function classifyFailure(text, options = {}) {
|
|
28
|
+
if (ENVIRONMENT.test(text))
|
|
29
|
+
return 'environment';
|
|
30
|
+
if (AUTH.test(text))
|
|
31
|
+
return 'auth';
|
|
32
|
+
if (NETWORK.test(text))
|
|
33
|
+
return 'network';
|
|
34
|
+
if (options.code === -32602 || INVALID_ARGUMENTS.test(text))
|
|
35
|
+
return 'invalid-arguments';
|
|
36
|
+
if (NOT_FOUND.test(text) && !options.hadArguments)
|
|
37
|
+
return 'not-found';
|
|
38
|
+
return 'other';
|
|
39
|
+
}
|
|
40
|
+
export const FAILURE_LABELS = {
|
|
41
|
+
auth: 'authentication',
|
|
42
|
+
environment: 'something missing on this machine',
|
|
43
|
+
network: 'the network',
|
|
44
|
+
'not-found': 'a 404 on a call that named nothing (usually the account or site behind the credentials)',
|
|
45
|
+
'invalid-arguments': 'the arguments',
|
|
46
|
+
other: 'other errors',
|
|
47
|
+
};
|
package/dist/history.js
CHANGED
|
@@ -80,7 +80,9 @@ async function inspectVersion(pkg, v, options, source) {
|
|
|
80
80
|
return { row: fail(row, 'no-bin', new Error('The package has no bin to start, and no --cmd was given.')) };
|
|
81
81
|
let result;
|
|
82
82
|
try {
|
|
83
|
-
result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir },
|
|
83
|
+
result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir },
|
|
84
|
+
// One process per version: history already starts one for each, and runs long.
|
|
85
|
+
{ timeoutMs: options.timeoutMs ?? 30_000, processes: 1 });
|
|
84
86
|
}
|
|
85
87
|
catch (error) {
|
|
86
88
|
return { row: fail(row, classifyFailure(error), error) };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { snapshot, type SnapshotOptions, type SnapshotResult } from './snapshot.js';
|
|
2
2
|
export { connect, listTools, type Target, type Connection } from './connect.js';
|
|
3
|
-
export { buildMenu, countTokens, loadMenu } from './menu.js';
|
|
3
|
+
export { buildMenu, countTokens, loadMenu, toolTokens } from './menu.js';
|
|
4
4
|
export { diffMenus, versionBump, DIFF_RULES, type DiffResult, type DiffOptions, type DiffFinding, type ChangeClass, type Bump, type TokenChange } from './diff.js';
|
|
5
5
|
export { history, historyCsv, npmSource, classifyFailure, WATCHED_DEPENDENCIES, type HistoryResult, type HistoryRow, type HistoryOptions, type PackageSource, type PublishedVersion, type FailureReason } from './history.js';
|
|
6
6
|
export { session, loadScenario, parseScenario, changeFindings, starterScenario, type Scenario, type Step, type SessionResult, type SessionOptions, type StepRecord, type Scope } from './session.js';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { snapshot } from './snapshot.js';
|
|
2
2
|
export { connect, listTools } from './connect.js';
|
|
3
|
-
export { buildMenu, countTokens, loadMenu } from './menu.js';
|
|
3
|
+
export { buildMenu, countTokens, loadMenu, toolTokens } from './menu.js';
|
|
4
4
|
export { diffMenus, versionBump, DIFF_RULES } from './diff.js';
|
|
5
5
|
export { history, historyCsv, npmSource, classifyFailure, WATCHED_DEPENDENCIES } from './history.js';
|
|
6
6
|
export { session, loadScenario, parseScenario, changeFindings, starterScenario } from './session.js';
|
package/dist/menu.d.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import type { Era, Menu, MenuTool } from './types.js';
|
|
2
2
|
/** Estimated tokens for a piece of text. An estimate: vendors tokenize differently. */
|
|
3
3
|
export declare function countTokens(text: string): number;
|
|
4
|
+
/**
|
|
5
|
+
* Estimated tokens the model reads for a tool: its name, description and input
|
|
6
|
+
* schema, which is what clients send (Claude's Messages API takes exactly those).
|
|
7
|
+
* outputSchema, annotations, icons, title and _meta stay with the client: GitHub's
|
|
8
|
+
* server embeds icons that would otherwise count five times its real menu.
|
|
9
|
+
*/
|
|
10
|
+
export declare function toolTokens(tool: Record<string, unknown>): number;
|
|
4
11
|
/** The tool definition without toolmenu's own `tokens` field. */
|
|
5
12
|
export declare function toolDefinition(tool: MenuTool): Record<string, unknown>;
|
|
6
13
|
export declare function buildMenu(rawTools: Record<string, unknown>[], server: Menu['server'], listMeta?: Menu['listMeta']): Menu;
|
package/dist/menu.js
CHANGED
|
@@ -6,6 +6,15 @@ export function countTokens(text) {
|
|
|
6
6
|
encoder ??= new Tiktoken(o200k);
|
|
7
7
|
return encoder.encode(text).length;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Estimated tokens the model reads for a tool: its name, description and input
|
|
11
|
+
* schema, which is what clients send (Claude's Messages API takes exactly those).
|
|
12
|
+
* outputSchema, annotations, icons, title and _meta stay with the client: GitHub's
|
|
13
|
+
* server embeds icons that would otherwise count five times its real menu.
|
|
14
|
+
*/
|
|
15
|
+
export function toolTokens(tool) {
|
|
16
|
+
return countTokens(JSON.stringify({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
|
|
17
|
+
}
|
|
9
18
|
/** The tool definition without toolmenu's own `tokens` field. */
|
|
10
19
|
export function toolDefinition(tool) {
|
|
11
20
|
const { tokens: _tokens, ...definition } = tool;
|
|
@@ -14,7 +23,7 @@ export function toolDefinition(tool) {
|
|
|
14
23
|
export function buildMenu(rawTools, server, listMeta) {
|
|
15
24
|
const tools = rawTools.map((raw) => ({
|
|
16
25
|
...raw,
|
|
17
|
-
tokens:
|
|
26
|
+
tokens: toolTokens(raw),
|
|
18
27
|
}));
|
|
19
28
|
return {
|
|
20
29
|
toolmenu: 1,
|
|
@@ -44,14 +53,14 @@ export async function loadMenu(path) {
|
|
|
44
53
|
if (menu?.toolmenu !== 1 || !Array.isArray(menu.tools)) {
|
|
45
54
|
throw new Error(`${path} isn't a toolmenu menu file (expected "toolmenu": 1 and a "tools" list). Write one with \`toolmenu snapshot\`.`);
|
|
46
55
|
}
|
|
56
|
+
// Always recounted, never read from the file: a baseline written by an older
|
|
57
|
+
// toolmenu (which counted every field) still compares fairly with a new snapshot.
|
|
47
58
|
for (const t of menu.tools) {
|
|
48
59
|
if (typeof t?.name !== 'string')
|
|
49
60
|
throw new Error(`${path}: every tool needs a name`);
|
|
50
|
-
|
|
51
|
-
t.tokens = countTokens(JSON.stringify(t));
|
|
61
|
+
t.tokens = toolTokens(t);
|
|
52
62
|
}
|
|
53
|
-
|
|
54
|
-
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
63
|
+
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
55
64
|
menu.server ??= {};
|
|
56
65
|
return menu;
|
|
57
66
|
}
|
package/dist/probe.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type Target } from './connect.js';
|
|
2
|
+
import type { Probe } from './rules/determinism.js';
|
|
3
|
+
import type { Menu, MenuTool } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* The hash seed of the process whose menu toolmenu saves and compares. Python
|
|
6
|
+
* orders sets of strings by a per-process random seed; pinning it makes a Python
|
|
7
|
+
* server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAIN_SEED = "0";
|
|
10
|
+
/** The probe's seed: different from the main one, so a set-order bug shows every time. */
|
|
11
|
+
export declare const PROBE_SEED = "1";
|
|
12
|
+
/**
|
|
13
|
+
* The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
|
|
14
|
+
* a stdio server. Other runtimes ignore it.
|
|
15
|
+
*/
|
|
16
|
+
export declare function seeded(target: Target, seed: string): Target;
|
|
17
|
+
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
18
|
+
export declare function isContainerWrapper(target: Target): boolean;
|
|
19
|
+
/** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
|
|
20
|
+
export declare function probeMenu(target: Target, timeoutMs: number): Promise<Menu>;
|
|
21
|
+
/**
|
|
22
|
+
* Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
|
|
23
|
+
* compare with the main one. stdio probes run with a different hash seed each. An
|
|
24
|
+
* HTTP probe that differs is retried once: a rolling deploy can serve two
|
|
25
|
+
* versions for a moment.
|
|
26
|
+
*/
|
|
27
|
+
export declare function probeVariance(target: Target, main: MenuTool[], processes: number, timeoutMs: number): Promise<Probe[]>;
|
package/dist/probe.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { basename } from 'node:path';
|
|
2
|
+
import { connect, listTools } from './connect.js';
|
|
3
|
+
import { compareMenus } from './compare.js';
|
|
4
|
+
import { buildMenu } from './menu.js';
|
|
5
|
+
/**
|
|
6
|
+
* The hash seed of the process whose menu toolmenu saves and compares. Python
|
|
7
|
+
* orders sets of strings by a per-process random seed; pinning it makes a Python
|
|
8
|
+
* server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
|
|
9
|
+
*/
|
|
10
|
+
export const MAIN_SEED = '0';
|
|
11
|
+
/** The probe's seed: different from the main one, so a set-order bug shows every time. */
|
|
12
|
+
export const PROBE_SEED = '1';
|
|
13
|
+
/**
|
|
14
|
+
* The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
|
|
15
|
+
* a stdio server. Other runtimes ignore it.
|
|
16
|
+
*/
|
|
17
|
+
export function seeded(target, seed) {
|
|
18
|
+
if (target.kind !== 'stdio' || target.env?.PYTHONHASHSEED !== undefined)
|
|
19
|
+
return target;
|
|
20
|
+
return { ...target, env: { ...target.env, PYTHONHASHSEED: seed } };
|
|
21
|
+
}
|
|
22
|
+
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
23
|
+
export function isContainerWrapper(target) {
|
|
24
|
+
return target.kind === 'stdio' && /^(docker|podman|nerdctl)(\.exe)?$/.test(basename(target.command));
|
|
25
|
+
}
|
|
26
|
+
/** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
|
|
27
|
+
export async function probeMenu(target, timeoutMs) {
|
|
28
|
+
const c = await connect(target, { timeoutMs });
|
|
29
|
+
try {
|
|
30
|
+
const list = await listTools(c, { timeoutMs });
|
|
31
|
+
return buildMenu(list.tools, { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era });
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
await c.close().catch(() => { });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
|
|
39
|
+
* compare with the main one. stdio probes run with a different hash seed each. An
|
|
40
|
+
* HTTP probe that differs is retried once: a rolling deploy can serve two
|
|
41
|
+
* versions for a moment.
|
|
42
|
+
*/
|
|
43
|
+
export async function probeVariance(target, main, processes, timeoutMs) {
|
|
44
|
+
const probes = [];
|
|
45
|
+
for (let i = 0; i < processes - 1; i++) {
|
|
46
|
+
const seed = String(Number(PROBE_SEED) + i);
|
|
47
|
+
try {
|
|
48
|
+
let tools = (await probeMenu(seeded(target, seed), timeoutMs)).tools;
|
|
49
|
+
if (target.kind === 'http' && compareMenus(main, tools).length)
|
|
50
|
+
tools = (await probeMenu(target, timeoutMs)).tools;
|
|
51
|
+
probes.push({ tools });
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
probes.push({ error: error instanceof Error ? error.message : String(error) });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return probes;
|
|
58
|
+
}
|
package/dist/report.js
CHANGED
|
@@ -221,7 +221,7 @@ export function formatSession(s, format) {
|
|
|
221
221
|
`toolmenu session ${s.scenario}`,
|
|
222
222
|
` ${s.server.name ?? 'server'} ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport}`,
|
|
223
223
|
` baseline: ${plural(s.baseline.tools, 'tool')} · ${tok(s.baseline.tokens)} tokens (estimate) · listening for list_changed: ${s.listening ? 'yes' : 'no'}`,
|
|
224
|
-
` fresh
|
|
224
|
+
` fresh-${s.transport === 'stdio' ? 'process' : 'connection'} check: ${s.connectionCheck === undefined ? 'not checked' : s.connectionCheck === 'same' ? 'the same menu' : 'a DIFFERENT menu'}`,
|
|
225
225
|
'',
|
|
226
226
|
];
|
|
227
227
|
const at = (step) => s.findings.filter((f) => f.step === step);
|
|
@@ -247,6 +247,12 @@ export function formatSession(s, format) {
|
|
|
247
247
|
lines.push(`step ${step.index}: ${step.label} · ${facts.join(' · ')}`);
|
|
248
248
|
block(at(step.index));
|
|
249
249
|
}
|
|
250
|
+
// Findings about the whole run (session/untested), after the steps.
|
|
251
|
+
const run = s.findings.filter((f) => f.step === undefined);
|
|
252
|
+
if (run.length) {
|
|
253
|
+
lines.push('');
|
|
254
|
+
block(run);
|
|
255
|
+
}
|
|
250
256
|
lines.push('');
|
|
251
257
|
lines.push(`final: ${plural(s.final.tools, 'tool')} · ${tok(s.final.tokens)} tokens (estimate)`);
|
|
252
258
|
lines.push(summaryLine(s.findings));
|
|
@@ -1,2 +1,24 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { MenuTool, Severity } from '../types.js';
|
|
2
|
+
import type { Rule, RuleFinding } from './rule.js';
|
|
2
3
|
export declare const nondeterministic: Rule;
|
|
4
|
+
/** What a second process or connection served, for the variance rules. */
|
|
5
|
+
export interface Probe {
|
|
6
|
+
tools?: MenuTool[];
|
|
7
|
+
/** Why the probe couldn't list tools, when it couldn't. */
|
|
8
|
+
error?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The finding for a menu that differs between processes (stdio) or connections
|
|
12
|
+
* (HTTP), shared by snapshot and session so both say the same thing at the same
|
|
13
|
+
* severity. Undefined when the menus match.
|
|
14
|
+
*/
|
|
15
|
+
export declare function varianceFinding(main: MenuTool[], other: MenuTool[], opts: {
|
|
16
|
+
transport: 'stdio' | 'http';
|
|
17
|
+
modern: boolean;
|
|
18
|
+
wrapper: boolean;
|
|
19
|
+
}): (RuleFinding & {
|
|
20
|
+
rule: string;
|
|
21
|
+
severity: Severity;
|
|
22
|
+
}) | undefined;
|
|
23
|
+
export declare const processVariance: Rule;
|
|
24
|
+
export declare const connectionVariance: Rule;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { compareMenus } from '../compare.js';
|
|
2
|
+
import { describeToolDifference } from '../difference.js';
|
|
2
3
|
const LABELS = {
|
|
3
4
|
added: 'tool added',
|
|
4
5
|
removed: 'tool removed',
|
|
@@ -27,9 +28,78 @@ export const nondeterministic = {
|
|
|
27
28
|
message: onlyOrder
|
|
28
29
|
? 'Two identical tools/list calls returned the tools in a different order. Every conversation can miss the prompt cache. The spec says servers SHOULD return a deterministic order.'
|
|
29
30
|
: 'Two identical tools/list calls returned different menus. Every conversation can miss the prompt cache.',
|
|
30
|
-
detail:
|
|
31
|
-
.concat(changes.length > 10 ? [`…and ${changes.length - 10} more`] : []),
|
|
31
|
+
detail: detailFor(ctx.menu.tools, ctx.secondList, changes),
|
|
32
32
|
},
|
|
33
33
|
];
|
|
34
34
|
},
|
|
35
35
|
};
|
|
36
|
+
/** One line per change; edits say which value differs, down to the leaf. */
|
|
37
|
+
function detailFor(before, after, changes) {
|
|
38
|
+
const byName = new Map(before.map((t) => [t.name, t]));
|
|
39
|
+
const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
|
|
40
|
+
const lines = [
|
|
41
|
+
...changes.filter((c) => !EDITS.has(c.kind)).map((c) => `${LABELS[c.kind]}: ${c.tool} (position ${c.position})`),
|
|
42
|
+
...after.filter((t) => edited.has(t.name)).map((t) => describeToolDifference(byName.get(t.name), t)),
|
|
43
|
+
];
|
|
44
|
+
return lines.slice(0, 10).concat(lines.length > 10 ? [`…and ${lines.length - 10} more`] : []);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The finding for a menu that differs between processes (stdio) or connections
|
|
48
|
+
* (HTTP), shared by snapshot and session so both say the same thing at the same
|
|
49
|
+
* severity. Undefined when the menus match.
|
|
50
|
+
*/
|
|
51
|
+
export function varianceFinding(main, other, opts) {
|
|
52
|
+
const changes = compareMenus(main, other);
|
|
53
|
+
if (changes.length === 0)
|
|
54
|
+
return undefined;
|
|
55
|
+
const detail = detailFor(main, other, changes);
|
|
56
|
+
const reordered = detail.some((d) => d.includes('same ') && d.includes('different order'));
|
|
57
|
+
const stdio = opts.transport === 'stdio';
|
|
58
|
+
const message = [
|
|
59
|
+
stdio
|
|
60
|
+
? 'A second server process, started the same way, served a different menu. Every restart, and every client that starts its own copy, gets a tool list that can\'t reuse a cached prompt.'
|
|
61
|
+
: `A second connection with the same credentials got a different menu.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
|
|
62
|
+
reordered
|
|
63
|
+
? 'The same items came out in a different order: likely a set or map iterated in hash order.' +
|
|
64
|
+
(stdio ? ' (toolmenu runs the first process with PYTHONHASHSEED=0 and the second with 1, so a Python set shows up every time; Go and Rust maps vary per process on their own.)' : '')
|
|
65
|
+
: '',
|
|
66
|
+
opts.wrapper
|
|
67
|
+
? 'The server runs in a container: PYTHONHASHSEED and --env only reach it if passed through (docker run -e).'
|
|
68
|
+
: '',
|
|
69
|
+
].filter(Boolean).join(' ');
|
|
70
|
+
return {
|
|
71
|
+
rule: stdio ? 'menu/process-variance' : 'menu/connection-variance',
|
|
72
|
+
severity: stdio || opts.modern ? 'error' : 'warn',
|
|
73
|
+
message,
|
|
74
|
+
detail,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
const EDITS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
78
|
+
function varianceRule(id, transport) {
|
|
79
|
+
return {
|
|
80
|
+
id,
|
|
81
|
+
severity: 'error',
|
|
82
|
+
lesson: '05',
|
|
83
|
+
summary: transport === 'stdio' ? 'A second server process serves the same menu' : 'A second connection gets the same menu',
|
|
84
|
+
run(ctx) {
|
|
85
|
+
if (ctx.transport !== transport || !ctx.probes?.length)
|
|
86
|
+
return [];
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const probe of ctx.probes) {
|
|
89
|
+
if (probe.error) {
|
|
90
|
+
out.push({ severity: 'info', message: `Couldn't ${transport === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const f = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper });
|
|
94
|
+
if (f) {
|
|
95
|
+
const { rule: _r, ...rest } = f;
|
|
96
|
+
out.push(rest);
|
|
97
|
+
break; // one finding: a second differing probe says nothing new
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
export const processVariance = varianceRule('menu/process-variance', 'stdio');
|
|
105
|
+
export const connectionVariance = varianceRule('menu/connection-variance', 'http');
|
package/dist/rules/ids.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ interface IdParam {
|
|
|
5
5
|
/** Which kind of thing it identifies (singular noun), when we can tell. */
|
|
6
6
|
kind?: string;
|
|
7
7
|
}
|
|
8
|
+
/** One spelling per kind word. */
|
|
9
|
+
export declare const canonicalWord: (w: string) => string;
|
|
8
10
|
/** Inputs that look like opaque IDs the agent has to supply. */
|
|
9
11
|
export declare function idParams(tool: MenuTool): IdParam[];
|
|
10
12
|
export declare const authoredIds: Rule;
|
package/dist/rules/ids.js
CHANGED
|
@@ -12,6 +12,14 @@ const QUALIFIERS = new Set([
|
|
|
12
12
|
'creator', 'new', 'old', 'target', 'source', 'parent', 'child', 'from', 'to', 'destination', 'current',
|
|
13
13
|
'previous', 'next', 'owner', 'assigned', 'assignee', 'primary', 'default', 'other', 'related', 'base', 'root',
|
|
14
14
|
]);
|
|
15
|
+
// Short and long names for the same kind: `orgId` is returned by
|
|
16
|
+
// `get-organization` (Miro), `repoId` by `list_repositories`.
|
|
17
|
+
const ABBREVIATIONS = {
|
|
18
|
+
org: 'organization', repo: 'repository', db: 'database', env: 'environment', dir: 'directory',
|
|
19
|
+
msg: 'message', doc: 'document', proj: 'project', ws: 'workspace', cfg: 'config', conf: 'config',
|
|
20
|
+
};
|
|
21
|
+
/** One spelling per kind word. */
|
|
22
|
+
export const canonicalWord = (w) => ABBREVIATIONS[w] ?? w;
|
|
15
23
|
// A description that tells the agent not to make the value up.
|
|
16
24
|
const DONT_INVENT = /\b(do not|don't|never|avoid|must not)\s+(invent|make up|guess|fabricate|generate)\b/i;
|
|
17
25
|
// "Omit this in the normal case": optional, and the agent is told to leave it out.
|
|
@@ -45,9 +53,12 @@ export function idParams(tool) {
|
|
|
45
53
|
const opaqueFormat = schema.format === 'uuid' || (typeof schema.pattern === 'string' && /\[0-9a-f|\[a-f0-9/i.test(schema.pattern));
|
|
46
54
|
if (!match && !bare && !opaqueFormat)
|
|
47
55
|
continue;
|
|
48
|
-
const kindWords = match?.[1] ? words(match[1]).map(singular).filter((w) => !JOINERS.has(w)) : [];
|
|
56
|
+
const kindWords = match?.[1] ? words(match[1]).map(singular).filter((w) => !JOINERS.has(w)).map(canonicalWord) : [];
|
|
49
57
|
while (kindWords.length > 1 && QUALIFIERS.has(kindWords[0]))
|
|
50
58
|
kindWords.shift();
|
|
59
|
+
// A qualifier alone (`parentId`, `targetId`) says how it relates, not what it is.
|
|
60
|
+
if (kindWords.length === 1 && QUALIFIERS.has(kindWords[0]))
|
|
61
|
+
kindWords.length = 0;
|
|
51
62
|
const prefix = kindWords.join('_');
|
|
52
63
|
// A bare `id` doesn't say what it identifies (naming/vague-id covers that), so no kind is guessed.
|
|
53
64
|
params.push({ param, kind: prefix || undefined });
|
|
@@ -65,7 +76,7 @@ function isReturnedBySomeTool(kind, self, tools, common) {
|
|
|
65
76
|
return tools.some((other) => {
|
|
66
77
|
if (other.name === self.name)
|
|
67
78
|
return false;
|
|
68
|
-
const w = words(other.name).map(singular).filter((x) => !common.has(x));
|
|
79
|
+
const w = words(other.name).map(singular).map(canonicalWord).filter((x) => !common.has(x));
|
|
69
80
|
const verbs = w.filter((x) => VERBS.has(x));
|
|
70
81
|
const subject = nouns(other.name).filter((x) => !common.has(x));
|
|
71
82
|
// The kind in its name, as a noun or a word run (data_source), or as the verb itself (searchId ← search).
|
|
@@ -90,8 +101,8 @@ function outputMentions(schema, kind, depth = 0, parent = '') {
|
|
|
90
101
|
if (!schema || depth > 6)
|
|
91
102
|
return false;
|
|
92
103
|
for (const [name, child] of Object.entries(schema.properties ?? {})) {
|
|
93
|
-
const n = words(name).map(singular).join('_');
|
|
94
|
-
if (n === `${kind}_id` || n === `${kind}_uuid` || (n === 'id' && singular(parent) === kind))
|
|
104
|
+
const n = words(name).map(singular).map(canonicalWord).join('_');
|
|
105
|
+
if (n === `${kind}_id` || n === `${kind}_uuid` || (n === 'id' && canonicalWord(singular(parent)) === kind))
|
|
95
106
|
return true;
|
|
96
107
|
if (outputMentions(child, kind, depth + 1, name))
|
|
97
108
|
return true;
|
|
@@ -102,11 +113,13 @@ function outputMentions(schema, kind, depth = 0, parent = '') {
|
|
|
102
113
|
}
|
|
103
114
|
export const authoredIds = {
|
|
104
115
|
id: 'ids/authored',
|
|
105
|
-
|
|
116
|
+
// A heuristic: IDs that only appear in another tool's text output look the same
|
|
117
|
+
// as invented ones from the schema alone (FINDINGS F12).
|
|
118
|
+
severity: 'info',
|
|
106
119
|
lesson: '01',
|
|
107
120
|
summary: 'IDs the agent must supply should come from some other tool',
|
|
108
121
|
run(ctx) {
|
|
109
|
-
const
|
|
122
|
+
const byKind = new Map();
|
|
110
123
|
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
111
124
|
for (const tool of ctx.menu.tools) {
|
|
112
125
|
for (const { param, kind } of idParams(tool)) {
|
|
@@ -119,16 +132,25 @@ export const authoredIds = {
|
|
|
119
132
|
const optional = !(tool.inputSchema?.required ?? []).includes(param);
|
|
120
133
|
if (optional && toldNotToInvent(tool, param, schema))
|
|
121
134
|
continue;
|
|
122
|
-
|
|
123
|
-
tool: tool.name,
|
|
124
|
-
message: `${tool.name}.${param}: looks like a ${kind} ID the agent must supply, but no tool appears to return one (no list/search/get tool for "${kind}", no output schema with a ${kind} ID). The agent may invent it. Heuristic.`,
|
|
125
|
-
});
|
|
135
|
+
byKind.set(kind, [...(byKind.get(kind) ?? []), { tool: tool.name, param }]);
|
|
126
136
|
}
|
|
127
137
|
}
|
|
128
|
-
|
|
138
|
+
// One finding per kind: 17 tools taking an orgId are one question, not 17.
|
|
139
|
+
return [...byKind].map(([kind, uses]) => {
|
|
140
|
+
const tools = [...new Set(uses.map((u) => u.tool))];
|
|
141
|
+
const where = uses.map((u) => `${u.tool}.${u.param}`);
|
|
142
|
+
return {
|
|
143
|
+
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
144
|
+
message: `${where.length === 1 ? `${where[0]} looks like` : `${where.length} parameters look like`} a ${kind} ID the agent must supply, but no tool appears to return one (no list/search/get tool for "${kind}", no output schema with a ${kind} ID). The agent may invent it, unless it comes from somewhere toolmenu can't see (a URL, another tool's text output). Heuristic.`,
|
|
145
|
+
...(where.length > 1 ? { detail: [where.slice(0, 8).join(', ') + (where.length > 8 ? `, and ${where.length - 8} more` : '')] } : {}),
|
|
146
|
+
};
|
|
147
|
+
});
|
|
129
148
|
},
|
|
130
149
|
};
|
|
131
|
-
|
|
150
|
+
// Where the value comes from: another tool, or a URL the user has ("often found as
|
|
151
|
+
// URL parameter node-id": Figma). A bare mention of a URL isn't enough: Sentry's
|
|
152
|
+
// "Required when not using a URL" names an alternative input, not a source.
|
|
153
|
+
const PROVENANCE = /\b(from|returned by|obtained|output of|result of|as (listed|shown|returned))\b|\bfound (in|as|on)\b[^.]{0,40}\b(url|link)\b|\b(url|query) param(eter)?\b/i;
|
|
132
154
|
/** The parameter's description says where the value comes from, or names the tool that gives it. */
|
|
133
155
|
function saysWhereItComesFrom(schema, tools, self) {
|
|
134
156
|
const text = typeof schema?.description === 'string' ? schema.description : '';
|
package/dist/rules/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { SEVERITY_RANK } from '../types.js';
|
|
2
2
|
import { buried, cut } from './description.js';
|
|
3
|
-
import { nondeterministic } from './determinism.js';
|
|
3
|
+
import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
5
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
6
6
|
import { cacheHints, deprecated, discover, schema } from './spec.js';
|
|
7
7
|
import { noDryRun, unannotated } from './write.js';
|
|
8
8
|
export const MENU_RULES = [
|
|
9
9
|
nondeterministic,
|
|
10
|
+
processVariance,
|
|
11
|
+
connectionVariance,
|
|
10
12
|
vagueId,
|
|
11
13
|
sharedWord,
|
|
12
14
|
route,
|
package/dist/rules/naming.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { commonWords, nouns, singular, words } from '../words.js';
|
|
2
2
|
import { missingWords, negatedMention, routeScores } from '../routes.js';
|
|
3
|
-
|
|
3
|
+
// Not `ref`: in real menus it's a git ref (GitHub ×4), a name with a meaning of its own.
|
|
4
|
+
const VAGUE_IDS = /^(id|ids|uuid|uuids|identifier|identifiers)$/i;
|
|
4
5
|
export const vagueId = {
|
|
5
6
|
id: 'naming/vague-id',
|
|
6
7
|
severity: 'warn',
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -13,6 +13,12 @@ export interface RuleContext {
|
|
|
13
13
|
capabilities: Record<string, unknown>;
|
|
14
14
|
usedAuth: boolean;
|
|
15
15
|
routes?: Routes;
|
|
16
|
+
/** stdio or HTTP: which variance rule applies. */
|
|
17
|
+
transport?: 'stdio' | 'http';
|
|
18
|
+
/** Menus from fresh processes (stdio) or connections (HTTP), for the variance rules. */
|
|
19
|
+
probes?: import('./determinism.js').Probe[];
|
|
20
|
+
/** The stdio command is a container wrapper (docker run …): env vars may not reach the server. */
|
|
21
|
+
wrapper?: boolean;
|
|
16
22
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
|
17
23
|
descriptionLimit?: number | string;
|
|
18
24
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
package/dist/session.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ToolChange } from './compare.js';
|
|
2
2
|
import { type Target } from './connect.js';
|
|
3
|
+
import { type FailureClass } from './failures.js';
|
|
3
4
|
import type { Era, Finding, Menu, MenuTool, Severity } from './types.js';
|
|
4
5
|
export type Step = {
|
|
5
6
|
kind: 'list';
|
|
@@ -28,6 +29,8 @@ export interface StepRecord {
|
|
|
28
29
|
/** Why no call was made, when none was: set where the status is set, never parsed from text. */
|
|
29
30
|
reason?: 'missing' | 'refused';
|
|
30
31
|
note?: string;
|
|
32
|
+
/** Why the call failed, when it did (a tool error or a failed request). */
|
|
33
|
+
failure?: FailureClass;
|
|
31
34
|
changed: boolean;
|
|
32
35
|
listChanged: number;
|
|
33
36
|
scope?: Scope;
|
|
@@ -47,6 +50,7 @@ export interface SessionResult {
|
|
|
47
50
|
tools: number;
|
|
48
51
|
tokens: number;
|
|
49
52
|
};
|
|
53
|
+
/** Undefined when not checked (processes: 1). */
|
|
50
54
|
connectionCheck?: 'same' | 'different';
|
|
51
55
|
steps: StepRecord[];
|
|
52
56
|
findings: Finding[];
|
|
@@ -58,6 +62,8 @@ export interface SessionOptions {
|
|
|
58
62
|
rules?: Record<string, Severity | 'off'>;
|
|
59
63
|
ignore?: string[];
|
|
60
64
|
scenarioName?: string;
|
|
65
|
+
/** Server processes (stdio) or connections (HTTP) to compare before the first step, the main one included (default 2). */
|
|
66
|
+
processes?: number;
|
|
61
67
|
}
|
|
62
68
|
type Raw = Omit<Finding, 'severity'> & {
|
|
63
69
|
severity: Severity;
|
package/dist/session.js
CHANGED
|
@@ -3,6 +3,9 @@ import { parse } from 'yaml';
|
|
|
3
3
|
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
4
|
import { connect, listTools } from './connect.js';
|
|
5
5
|
import { buildMenu } from './menu.js';
|
|
6
|
+
import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
|
+
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
+
import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
|
|
6
9
|
import { SEVERITY_RANK } from './types.js';
|
|
7
10
|
import { verbOf, WRITE_VERBS } from './words.js';
|
|
8
11
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
@@ -63,7 +66,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
63
66
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
64
67
|
const grace = options.noticeGraceMs ?? 500;
|
|
65
68
|
const raw = [];
|
|
66
|
-
|
|
69
|
+
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
70
|
+
// a probe with a different seed would count ordering variance as a change.
|
|
71
|
+
const mainTarget = seeded(target, MAIN_SEED);
|
|
72
|
+
const conn = await connect(mainTarget, { timeoutMs });
|
|
67
73
|
try {
|
|
68
74
|
const era = conn.era;
|
|
69
75
|
const modern = era === 'modern';
|
|
@@ -81,22 +87,18 @@ export async function session(target, scenario, options = {}) {
|
|
|
81
87
|
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
82
88
|
let current = await menuOf(conn);
|
|
83
89
|
const baseline = current;
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
raw.push({
|
|
91
|
-
rule: 'session/connection-variance',
|
|
92
|
-
severity: modern ? 'error' : 'warn',
|
|
93
|
-
step: 0,
|
|
94
|
-
message: `A second connection with the same credentials got a different menu.${modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
|
|
95
|
-
detail,
|
|
96
|
-
});
|
|
90
|
+
// Fresh processes or connections, same credentials, must see the same menu.
|
|
91
|
+
let connectionCheck;
|
|
92
|
+
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
|
|
93
|
+
if (probe.error) {
|
|
94
|
+
raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
|
|
95
|
+
continue;
|
|
97
96
|
}
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
98
|
+
connectionCheck = f ? 'different' : connectionCheck ?? 'same';
|
|
99
|
+
if (f) {
|
|
100
|
+
raw.push({ ...f, step: 0 });
|
|
101
|
+
break;
|
|
100
102
|
}
|
|
101
103
|
}
|
|
102
104
|
const steps = [];
|
|
@@ -136,19 +138,25 @@ export async function session(target, scenario, options = {}) {
|
|
|
136
138
|
.trim();
|
|
137
139
|
const short = clip(text, 120);
|
|
138
140
|
record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
141
|
+
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
142
|
+
// A setup failure is reported once, for the whole run (session/untested).
|
|
143
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
144
|
+
raw.push({
|
|
145
|
+
rule: 'session/tool-error',
|
|
146
|
+
severity: 'warn',
|
|
147
|
+
step: index,
|
|
148
|
+
tool: step.tool,
|
|
149
|
+
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
150
|
+
});
|
|
146
151
|
}
|
|
147
152
|
}
|
|
148
153
|
catch (error) {
|
|
149
154
|
record.status = 'failed';
|
|
150
155
|
record.note = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
151
|
-
|
|
156
|
+
const code = error.code;
|
|
157
|
+
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
158
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
159
|
+
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
|
|
152
160
|
}
|
|
153
161
|
}
|
|
154
162
|
else if (step.kind === 'wait_for') {
|
|
@@ -197,7 +205,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
197
205
|
});
|
|
198
206
|
}
|
|
199
207
|
if (step.kind !== 'list') {
|
|
200
|
-
const probe = await
|
|
208
|
+
const probe = await probeMenu(mainTarget, timeoutMs);
|
|
201
209
|
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind);
|
|
202
210
|
raw.push(...scopeFindings(record.scope, modern, index));
|
|
203
211
|
}
|
|
@@ -208,6 +216,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
208
216
|
record.tokens = current.totalTokens;
|
|
209
217
|
steps.push(record);
|
|
210
218
|
}
|
|
219
|
+
raw.push(...untested(steps, target));
|
|
211
220
|
return {
|
|
212
221
|
scenario: options.scenarioName ?? 'scenario',
|
|
213
222
|
server: baseline.server,
|
|
@@ -224,21 +233,34 @@ export async function session(target, scenario, options = {}) {
|
|
|
224
233
|
await conn.close().catch(() => { });
|
|
225
234
|
}
|
|
226
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* Calls that failed before reaching the tool's logic (credentials, this machine,
|
|
238
|
+
* the network) tested nothing but whether a failed call changes the menu. Said
|
|
239
|
+
* once. An error when no call got through at all: otherwise an expired CI secret
|
|
240
|
+
* turns the job green while testing nothing.
|
|
241
|
+
*/
|
|
242
|
+
function untested(steps, target) {
|
|
243
|
+
const calls = steps.filter((s) => s.label.startsWith('call ') && s.reason === undefined);
|
|
244
|
+
const setup = calls.filter((s) => s.failure && SETUP_FAILURES.has(s.failure));
|
|
245
|
+
if (setup.length === 0)
|
|
246
|
+
return [];
|
|
247
|
+
const byClass = new Map();
|
|
248
|
+
for (const s of setup)
|
|
249
|
+
byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
|
|
250
|
+
const all = setup.length === calls.length;
|
|
251
|
+
const how = target.kind === 'http' ? 'real credentials (--header "Authorization: …")' : 'real credentials (--env KEY=…)';
|
|
252
|
+
return [
|
|
253
|
+
{
|
|
254
|
+
rule: 'session/untested',
|
|
255
|
+
severity: all ? 'error' : 'warn',
|
|
256
|
+
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.${byClass.has('auth') || byClass.has('not-found') ? ` Run with ${how}.` : ''}`,
|
|
257
|
+
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
|
|
258
|
+
},
|
|
259
|
+
];
|
|
260
|
+
}
|
|
227
261
|
function serverOf(c) {
|
|
228
262
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
229
263
|
}
|
|
230
|
-
async function freshMenu(target, timeoutMs) {
|
|
231
|
-
const c = await connect(target, { timeoutMs });
|
|
232
|
-
try {
|
|
233
|
-
return buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
234
|
-
}
|
|
235
|
-
finally {
|
|
236
|
-
await c.close().catch(() => { });
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
function sameMenu(a, b) {
|
|
240
|
-
return compareMenus(a.tools, b.tools).length === 0;
|
|
241
|
-
}
|
|
242
264
|
function waitFor(done, ms) {
|
|
243
265
|
return new Promise((resolve) => {
|
|
244
266
|
if (done())
|
|
@@ -261,11 +283,15 @@ const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annot
|
|
|
261
283
|
export function changeFindings(before, after, changes, step, origin) {
|
|
262
284
|
const out = [];
|
|
263
285
|
const brk = cacheBreak(before, after);
|
|
286
|
+
// Claude's docs: adding, removing or reordering a tool invalidates the entire
|
|
287
|
+
// cache (SPEC §20), not just what follows the change. So the cost is the whole
|
|
288
|
+
// tool list and everything after it; toolmenu can measure this server's part.
|
|
289
|
+
const total = after.reduce((sum, t) => sum + t.tokens, 0);
|
|
264
290
|
const cost = !brk
|
|
265
291
|
? []
|
|
266
|
-
:
|
|
267
|
-
|
|
268
|
-
|
|
292
|
+
: [
|
|
293
|
+
`the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}; any change to the tool list invalidates the cached prompt, so the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it are processed again`,
|
|
294
|
+
];
|
|
269
295
|
const why = origin ? [origin] : [];
|
|
270
296
|
const added = changes.filter((c) => c.kind === 'added');
|
|
271
297
|
const others = changes.filter((c) => c.kind !== 'added');
|
|
@@ -289,7 +315,7 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
289
315
|
rule: 'session/mid-insert',
|
|
290
316
|
severity: 'error',
|
|
291
317
|
step,
|
|
292
|
-
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt
|
|
318
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
293
319
|
detail: [...cost, ...why],
|
|
294
320
|
});
|
|
295
321
|
}
|
|
@@ -389,7 +415,11 @@ function settle(raw, options) {
|
|
|
389
415
|
* back it up. Words like "scope" or "mode" alone say nothing: `get_my_scope_ids`
|
|
390
416
|
* changes no menu.
|
|
391
417
|
*/
|
|
392
|
-
|
|
418
|
+
// A parameter that names what to unlock. Strong names say so alone; weak ones
|
|
419
|
+
// (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
|
|
420
|
+
// description to back them up.
|
|
421
|
+
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?|namespaces?)$/i;
|
|
422
|
+
const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?)$/i;
|
|
393
423
|
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
394
424
|
const UNLOCK_DESC = /(?:unlock|enable|activate|load|expose|add)s?[^.]{0,60}tools?|more tools|toolsets?|capabilit(?:y|ies)|unlock/i;
|
|
395
425
|
/**
|
|
@@ -423,7 +453,8 @@ export function unlockers(tools) {
|
|
|
423
453
|
const found = [];
|
|
424
454
|
for (const tool of tools) {
|
|
425
455
|
const props = tool.inputSchema?.properties ?? {};
|
|
426
|
-
const
|
|
456
|
+
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(tool.description ?? '');
|
|
457
|
+
const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ?? (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined);
|
|
427
458
|
let score = 0;
|
|
428
459
|
if (param)
|
|
429
460
|
score += enumOf(props[param]).length ? 3 : 2;
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -7,6 +7,11 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
7
7
|
timeoutMs?: number;
|
|
8
8
|
descriptionLimit?: number | string;
|
|
9
9
|
fullDescriptions?: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Server processes (stdio) or connections (HTTP) to compare, the main one
|
|
12
|
+
* included. Default 2; 1 turns the check off.
|
|
13
|
+
*/
|
|
14
|
+
processes?: number;
|
|
10
15
|
}
|
|
11
16
|
export interface SnapshotResult {
|
|
12
17
|
menu: Menu;
|
package/dist/snapshot.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { connect, listTools } from './connect.js';
|
|
2
2
|
import { buildMenu } from './menu.js';
|
|
3
|
+
import { isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
3
4
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
4
5
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
5
6
|
export async function snapshot(target, options = {}) {
|
|
6
|
-
const connection = await connect(target, { timeoutMs: options.timeoutMs });
|
|
7
|
+
const connection = await connect(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
8
|
+
let menu;
|
|
9
|
+
let secondList;
|
|
10
|
+
let first;
|
|
7
11
|
try {
|
|
8
|
-
|
|
12
|
+
first = await listTools(connection, options);
|
|
9
13
|
const second = await listTools(connection, options);
|
|
10
14
|
const server = {
|
|
11
15
|
name: connection.server.name,
|
|
@@ -13,22 +17,8 @@ export async function snapshot(target, options = {}) {
|
|
|
13
17
|
protocolVersion: connection.protocolVersion,
|
|
14
18
|
era: connection.era,
|
|
15
19
|
};
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const findings = runRules(MENU_RULES, {
|
|
19
|
-
menu,
|
|
20
|
-
secondList: secondMenu.tools,
|
|
21
|
-
pages: first.pages,
|
|
22
|
-
clientError: first.clientError,
|
|
23
|
-
protocolVersion: connection.protocolVersion,
|
|
24
|
-
era: connection.era,
|
|
25
|
-
capabilities: connection.capabilities,
|
|
26
|
-
usedAuth: connection.usedAuth,
|
|
27
|
-
routes: options.routes,
|
|
28
|
-
descriptionLimit: options.descriptionLimit,
|
|
29
|
-
fullDescriptions: options.fullDescriptions,
|
|
30
|
-
}, options);
|
|
31
|
-
return { menu, findings };
|
|
20
|
+
menu = buildMenu(first.tools, server, first.listMeta);
|
|
21
|
+
secondList = buildMenu(second.tools, server, second.listMeta).tools;
|
|
32
22
|
}
|
|
33
23
|
catch (error) {
|
|
34
24
|
const stderr = connection.stderr().trim();
|
|
@@ -39,4 +29,24 @@ export async function snapshot(target, options = {}) {
|
|
|
39
29
|
finally {
|
|
40
30
|
await connection.close().catch(() => { });
|
|
41
31
|
}
|
|
32
|
+
// After the main process has exited: a server that holds a file or a port
|
|
33
|
+
// shouldn't have to share it with its own probe.
|
|
34
|
+
const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000);
|
|
35
|
+
const findings = runRules(MENU_RULES, {
|
|
36
|
+
menu,
|
|
37
|
+
secondList,
|
|
38
|
+
pages: first.pages,
|
|
39
|
+
clientError: first.clientError,
|
|
40
|
+
protocolVersion: connection.protocolVersion,
|
|
41
|
+
era: connection.era,
|
|
42
|
+
capabilities: connection.capabilities,
|
|
43
|
+
usedAuth: connection.usedAuth,
|
|
44
|
+
routes: options.routes,
|
|
45
|
+
descriptionLimit: options.descriptionLimit,
|
|
46
|
+
fullDescriptions: options.fullDescriptions,
|
|
47
|
+
transport: target.kind,
|
|
48
|
+
probes,
|
|
49
|
+
wrapper: isContainerWrapper(target),
|
|
50
|
+
}, options);
|
|
51
|
+
return { menu, findings };
|
|
42
52
|
}
|