toolmenu 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -9
- package/dist/args.d.ts +22 -0
- package/dist/args.js +81 -0
- package/dist/auth.d.ts +77 -0
- package/dist/auth.js +304 -0
- package/dist/auto.d.ts +28 -0
- package/dist/auto.js +81 -0
- package/dist/cli.js +105 -9
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/connect.d.ts +1 -0
- package/dist/connect.js +11 -1
- package/dist/difference.d.ts +17 -0
- package/dist/difference.js +70 -0
- package/dist/failures.d.ts +14 -0
- package/dist/failures.js +47 -0
- package/dist/history.js +3 -1
- package/dist/probe.d.ts +27 -0
- package/dist/probe.js +58 -0
- package/dist/report.js +29 -3
- package/dist/rules/determinism.d.ts +23 -1
- package/dist/rules/determinism.js +72 -2
- package/dist/rules/ids.d.ts +2 -0
- package/dist/rules/ids.js +34 -12
- package/dist/rules/index.js +3 -1
- package/dist/rules/naming.js +2 -1
- package/dist/rules/rule.d.ts +6 -0
- package/dist/session.d.ts +16 -0
- package/dist/session.js +81 -44
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +28 -18
- package/package.json +1 -1
package/dist/report.js
CHANGED
|
@@ -197,8 +197,11 @@ export function formatHistory(h, format, outDir) {
|
|
|
197
197
|
return lines.join('\n');
|
|
198
198
|
}
|
|
199
199
|
export function formatSession(s, format) {
|
|
200
|
-
if (format === 'json')
|
|
201
|
-
|
|
200
|
+
if (format === 'json') {
|
|
201
|
+
// The union menu itself goes to --union-out; the report says how big it is.
|
|
202
|
+
const { union, ...rest } = s;
|
|
203
|
+
return JSON.stringify({ command: 'session', ...rest, union: { tools: union.tools.length, totalTokens: union.totalTokens }, counts: counts(s.findings) }, null, 2);
|
|
204
|
+
}
|
|
202
205
|
if (format === 'github') {
|
|
203
206
|
return githubLines(s.findings.map((f) => ({ ...f, message: f.step ? `step ${f.step} (${s.steps[f.step - 1]?.label}): ${f.message}` : f.message }))).join('\n');
|
|
204
207
|
}
|
|
@@ -208,6 +211,7 @@ export function formatSession(s, format) {
|
|
|
208
211
|
'',
|
|
209
212
|
`\`${s.server.name ?? 'server'}\` ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport} · ${s.baseline.tools} → ${s.final.tools} tools · ~${s.baseline.tokens.toLocaleString('en-US')} → ~${s.final.tokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
210
213
|
'',
|
|
214
|
+
...(autoLine(s) ? [autoLine(s), ''] : []),
|
|
211
215
|
'| Step | Menu | list_changed | Scope |',
|
|
212
216
|
'|---|---|---|---|',
|
|
213
217
|
...s.steps.map((st) => `| ${st.index}. ${mdCell(st.label)} | ${st.status !== 'ok' ? st.status : st.changed ? `changed (${st.tools} tools)` : 'no change'} | ${st.changed ? (st.listChanged ? 'received' : '**missing**') : ''} | ${st.scope ?? ''} |`),
|
|
@@ -221,7 +225,8 @@ export function formatSession(s, format) {
|
|
|
221
225
|
`toolmenu session ${s.scenario}`,
|
|
222
226
|
` ${s.server.name ?? 'server'} ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport}`,
|
|
223
227
|
` baseline: ${plural(s.baseline.tools, 'tool')} · ${tok(s.baseline.tokens)} tokens (estimate) · listening for list_changed: ${s.listening ? 'yes' : 'no'}`,
|
|
224
|
-
`
|
|
228
|
+
...(autoLine(s) ? [` ${autoLine(s)}`] : []),
|
|
229
|
+
` fresh-${s.transport === 'stdio' ? 'process' : 'connection'} check: ${s.connectionCheck === undefined ? 'not checked' : s.connectionCheck === 'same' ? 'the same menu' : 'a DIFFERENT menu'}`,
|
|
225
230
|
'',
|
|
226
231
|
];
|
|
227
232
|
const at = (step) => s.findings.filter((f) => f.step === step);
|
|
@@ -247,6 +252,12 @@ export function formatSession(s, format) {
|
|
|
247
252
|
lines.push(`step ${step.index}: ${step.label} · ${facts.join(' · ')}`);
|
|
248
253
|
block(at(step.index));
|
|
249
254
|
}
|
|
255
|
+
// Findings about the whole run (session/untested), after the steps.
|
|
256
|
+
const run = s.findings.filter((f) => f.step === undefined);
|
|
257
|
+
if (run.length) {
|
|
258
|
+
lines.push('');
|
|
259
|
+
block(run);
|
|
260
|
+
}
|
|
250
261
|
lines.push('');
|
|
251
262
|
lines.push(`final: ${plural(s.final.tools, 'tool')} · ${tok(s.final.tokens)} tokens (estimate)`);
|
|
252
263
|
lines.push(summaryLine(s.findings));
|
|
@@ -258,6 +269,21 @@ export function formatPlan(scenario, name) {
|
|
|
258
269
|
lines.push('', 'After every step the menu is listed and compared with the one before. Nothing was run.');
|
|
259
270
|
return lines.join('\n');
|
|
260
271
|
}
|
|
272
|
+
/** One line on what --auto called and what it left out, and how to reach more. */
|
|
273
|
+
function autoLine(s) {
|
|
274
|
+
if (!s.auto)
|
|
275
|
+
return undefined;
|
|
276
|
+
const by = (r) => s.auto.skipped.filter((x) => x.reason === r);
|
|
277
|
+
const needs = by('needs values');
|
|
278
|
+
const params = [...new Set(needs.flatMap((x) => x.missing ?? []))];
|
|
279
|
+
const parts = [
|
|
280
|
+
by('not read-only').length && `${by('not read-only').length} not read-only`,
|
|
281
|
+
by('open world').length && `${by('open world').length} marked openWorldHint: true (--open-world to call them)`,
|
|
282
|
+
needs.length && `${needs.length} need values the schema doesn't give (${params.slice(0, 6).join(', ')}${params.length > 6 ? ', …' : ''}: write a scenario, or --save-scenario and fill them in)`,
|
|
283
|
+
by('over the call budget').length && `${by('over the call budget').length} over --max-calls`,
|
|
284
|
+
].filter(Boolean);
|
|
285
|
+
return `auto: called ${s.auto.called.length} tool${s.auto.called.length === 1 ? '' : 's'}${parts.length ? ` · skipped ${parts.join(', ')}` : ''}`;
|
|
286
|
+
}
|
|
261
287
|
function mdCell(text) {
|
|
262
288
|
return text.replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
263
289
|
}
|
|
@@ -1,2 +1,24 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { MenuTool, Severity } from '../types.js';
|
|
2
|
+
import type { Rule, RuleFinding } from './rule.js';
|
|
2
3
|
export declare const nondeterministic: Rule;
|
|
4
|
+
/** What a second process or connection served, for the variance rules. */
|
|
5
|
+
export interface Probe {
|
|
6
|
+
tools?: MenuTool[];
|
|
7
|
+
/** Why the probe couldn't list tools, when it couldn't. */
|
|
8
|
+
error?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The finding for a menu that differs between processes (stdio) or connections
|
|
12
|
+
* (HTTP), shared by snapshot and session so both say the same thing at the same
|
|
13
|
+
* severity. Undefined when the menus match.
|
|
14
|
+
*/
|
|
15
|
+
export declare function varianceFinding(main: MenuTool[], other: MenuTool[], opts: {
|
|
16
|
+
transport: 'stdio' | 'http';
|
|
17
|
+
modern: boolean;
|
|
18
|
+
wrapper: boolean;
|
|
19
|
+
}): (RuleFinding & {
|
|
20
|
+
rule: string;
|
|
21
|
+
severity: Severity;
|
|
22
|
+
}) | undefined;
|
|
23
|
+
export declare const processVariance: Rule;
|
|
24
|
+
export declare const connectionVariance: Rule;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { compareMenus } from '../compare.js';
|
|
2
|
+
import { describeToolDifference } from '../difference.js';
|
|
2
3
|
const LABELS = {
|
|
3
4
|
added: 'tool added',
|
|
4
5
|
removed: 'tool removed',
|
|
@@ -27,9 +28,78 @@ export const nondeterministic = {
|
|
|
27
28
|
message: onlyOrder
|
|
28
29
|
? 'Two identical tools/list calls returned the tools in a different order. Every conversation can miss the prompt cache. The spec says servers SHOULD return a deterministic order.'
|
|
29
30
|
: 'Two identical tools/list calls returned different menus. Every conversation can miss the prompt cache.',
|
|
30
|
-
detail:
|
|
31
|
-
.concat(changes.length > 10 ? [`…and ${changes.length - 10} more`] : []),
|
|
31
|
+
detail: detailFor(ctx.menu.tools, ctx.secondList, changes),
|
|
32
32
|
},
|
|
33
33
|
];
|
|
34
34
|
},
|
|
35
35
|
};
|
|
36
|
+
/** One line per change; edits say which value differs, down to the leaf. */
|
|
37
|
+
function detailFor(before, after, changes) {
|
|
38
|
+
const byName = new Map(before.map((t) => [t.name, t]));
|
|
39
|
+
const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
|
|
40
|
+
const lines = [
|
|
41
|
+
...changes.filter((c) => !EDITS.has(c.kind)).map((c) => `${LABELS[c.kind]}: ${c.tool} (position ${c.position})`),
|
|
42
|
+
...after.filter((t) => edited.has(t.name)).map((t) => describeToolDifference(byName.get(t.name), t)),
|
|
43
|
+
];
|
|
44
|
+
return lines.slice(0, 10).concat(lines.length > 10 ? [`…and ${lines.length - 10} more`] : []);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The finding for a menu that differs between processes (stdio) or connections
|
|
48
|
+
* (HTTP), shared by snapshot and session so both say the same thing at the same
|
|
49
|
+
* severity. Undefined when the menus match.
|
|
50
|
+
*/
|
|
51
|
+
export function varianceFinding(main, other, opts) {
|
|
52
|
+
const changes = compareMenus(main, other);
|
|
53
|
+
if (changes.length === 0)
|
|
54
|
+
return undefined;
|
|
55
|
+
const detail = detailFor(main, other, changes);
|
|
56
|
+
const reordered = detail.some((d) => d.includes('same ') && d.includes('different order'));
|
|
57
|
+
const stdio = opts.transport === 'stdio';
|
|
58
|
+
const message = [
|
|
59
|
+
stdio
|
|
60
|
+
? 'A second server process, started the same way, served a different menu. Every restart, and every client that starts its own copy, gets a tool list that can\'t reuse a cached prompt.'
|
|
61
|
+
: `A second connection with the same credentials got a different menu.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
|
|
62
|
+
reordered
|
|
63
|
+
? 'The same items came out in a different order: likely a set or map iterated in hash order.' +
|
|
64
|
+
(stdio ? ' (toolmenu runs the first process with PYTHONHASHSEED=0 and the second with 1, so a Python set shows up every time; Go and Rust maps vary per process on their own.)' : '')
|
|
65
|
+
: '',
|
|
66
|
+
opts.wrapper
|
|
67
|
+
? 'The server runs in a container: PYTHONHASHSEED and --env only reach it if passed through (docker run -e).'
|
|
68
|
+
: '',
|
|
69
|
+
].filter(Boolean).join(' ');
|
|
70
|
+
return {
|
|
71
|
+
rule: stdio ? 'menu/process-variance' : 'menu/connection-variance',
|
|
72
|
+
severity: stdio || opts.modern ? 'error' : 'warn',
|
|
73
|
+
message,
|
|
74
|
+
detail,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
const EDITS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
78
|
+
function varianceRule(id, transport) {
|
|
79
|
+
return {
|
|
80
|
+
id,
|
|
81
|
+
severity: 'error',
|
|
82
|
+
lesson: '05',
|
|
83
|
+
summary: transport === 'stdio' ? 'A second server process serves the same menu' : 'A second connection gets the same menu',
|
|
84
|
+
run(ctx) {
|
|
85
|
+
if (ctx.transport !== transport || !ctx.probes?.length)
|
|
86
|
+
return [];
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const probe of ctx.probes) {
|
|
89
|
+
if (probe.error) {
|
|
90
|
+
out.push({ severity: 'info', message: `Couldn't ${transport === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const f = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper });
|
|
94
|
+
if (f) {
|
|
95
|
+
const { rule: _r, ...rest } = f;
|
|
96
|
+
out.push(rest);
|
|
97
|
+
break; // one finding: a second differing probe says nothing new
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
export const processVariance = varianceRule('menu/process-variance', 'stdio');
|
|
105
|
+
export const connectionVariance = varianceRule('menu/connection-variance', 'http');
|
package/dist/rules/ids.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ interface IdParam {
|
|
|
5
5
|
/** Which kind of thing it identifies (singular noun), when we can tell. */
|
|
6
6
|
kind?: string;
|
|
7
7
|
}
|
|
8
|
+
/** One spelling per kind word. */
|
|
9
|
+
export declare const canonicalWord: (w: string) => string;
|
|
8
10
|
/** Inputs that look like opaque IDs the agent has to supply. */
|
|
9
11
|
export declare function idParams(tool: MenuTool): IdParam[];
|
|
10
12
|
export declare const authoredIds: Rule;
|
package/dist/rules/ids.js
CHANGED
|
@@ -12,6 +12,14 @@ const QUALIFIERS = new Set([
|
|
|
12
12
|
'creator', 'new', 'old', 'target', 'source', 'parent', 'child', 'from', 'to', 'destination', 'current',
|
|
13
13
|
'previous', 'next', 'owner', 'assigned', 'assignee', 'primary', 'default', 'other', 'related', 'base', 'root',
|
|
14
14
|
]);
|
|
15
|
+
// Short and long names for the same kind: `orgId` is returned by
|
|
16
|
+
// `get-organization` (Miro), `repoId` by `list_repositories`.
|
|
17
|
+
const ABBREVIATIONS = {
|
|
18
|
+
org: 'organization', repo: 'repository', db: 'database', env: 'environment', dir: 'directory',
|
|
19
|
+
msg: 'message', doc: 'document', proj: 'project', ws: 'workspace', cfg: 'config', conf: 'config',
|
|
20
|
+
};
|
|
21
|
+
/** One spelling per kind word. */
|
|
22
|
+
export const canonicalWord = (w) => ABBREVIATIONS[w] ?? w;
|
|
15
23
|
// A description that tells the agent not to make the value up.
|
|
16
24
|
const DONT_INVENT = /\b(do not|don't|never|avoid|must not)\s+(invent|make up|guess|fabricate|generate)\b/i;
|
|
17
25
|
// "Omit this in the normal case": optional, and the agent is told to leave it out.
|
|
@@ -45,9 +53,12 @@ export function idParams(tool) {
|
|
|
45
53
|
const opaqueFormat = schema.format === 'uuid' || (typeof schema.pattern === 'string' && /\[0-9a-f|\[a-f0-9/i.test(schema.pattern));
|
|
46
54
|
if (!match && !bare && !opaqueFormat)
|
|
47
55
|
continue;
|
|
48
|
-
const kindWords = match?.[1] ? words(match[1]).map(singular).filter((w) => !JOINERS.has(w)) : [];
|
|
56
|
+
const kindWords = match?.[1] ? words(match[1]).map(singular).filter((w) => !JOINERS.has(w)).map(canonicalWord) : [];
|
|
49
57
|
while (kindWords.length > 1 && QUALIFIERS.has(kindWords[0]))
|
|
50
58
|
kindWords.shift();
|
|
59
|
+
// A qualifier alone (`parentId`, `targetId`) says how it relates, not what it is.
|
|
60
|
+
if (kindWords.length === 1 && QUALIFIERS.has(kindWords[0]))
|
|
61
|
+
kindWords.length = 0;
|
|
51
62
|
const prefix = kindWords.join('_');
|
|
52
63
|
// A bare `id` doesn't say what it identifies (naming/vague-id covers that), so no kind is guessed.
|
|
53
64
|
params.push({ param, kind: prefix || undefined });
|
|
@@ -65,7 +76,7 @@ function isReturnedBySomeTool(kind, self, tools, common) {
|
|
|
65
76
|
return tools.some((other) => {
|
|
66
77
|
if (other.name === self.name)
|
|
67
78
|
return false;
|
|
68
|
-
const w = words(other.name).map(singular).filter((x) => !common.has(x));
|
|
79
|
+
const w = words(other.name).map(singular).map(canonicalWord).filter((x) => !common.has(x));
|
|
69
80
|
const verbs = w.filter((x) => VERBS.has(x));
|
|
70
81
|
const subject = nouns(other.name).filter((x) => !common.has(x));
|
|
71
82
|
// The kind in its name, as a noun or a word run (data_source), or as the verb itself (searchId ← search).
|
|
@@ -90,8 +101,8 @@ function outputMentions(schema, kind, depth = 0, parent = '') {
|
|
|
90
101
|
if (!schema || depth > 6)
|
|
91
102
|
return false;
|
|
92
103
|
for (const [name, child] of Object.entries(schema.properties ?? {})) {
|
|
93
|
-
const n = words(name).map(singular).join('_');
|
|
94
|
-
if (n === `${kind}_id` || n === `${kind}_uuid` || (n === 'id' && singular(parent) === kind))
|
|
104
|
+
const n = words(name).map(singular).map(canonicalWord).join('_');
|
|
105
|
+
if (n === `${kind}_id` || n === `${kind}_uuid` || (n === 'id' && canonicalWord(singular(parent)) === kind))
|
|
95
106
|
return true;
|
|
96
107
|
if (outputMentions(child, kind, depth + 1, name))
|
|
97
108
|
return true;
|
|
@@ -102,11 +113,13 @@ function outputMentions(schema, kind, depth = 0, parent = '') {
|
|
|
102
113
|
}
|
|
103
114
|
export const authoredIds = {
|
|
104
115
|
id: 'ids/authored',
|
|
105
|
-
|
|
116
|
+
// A heuristic: IDs that only appear in another tool's text output look the same
|
|
117
|
+
// as invented ones from the schema alone (FINDINGS F12).
|
|
118
|
+
severity: 'info',
|
|
106
119
|
lesson: '01',
|
|
107
120
|
summary: 'IDs the agent must supply should come from some other tool',
|
|
108
121
|
run(ctx) {
|
|
109
|
-
const
|
|
122
|
+
const byKind = new Map();
|
|
110
123
|
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
111
124
|
for (const tool of ctx.menu.tools) {
|
|
112
125
|
for (const { param, kind } of idParams(tool)) {
|
|
@@ -119,16 +132,25 @@ export const authoredIds = {
|
|
|
119
132
|
const optional = !(tool.inputSchema?.required ?? []).includes(param);
|
|
120
133
|
if (optional && toldNotToInvent(tool, param, schema))
|
|
121
134
|
continue;
|
|
122
|
-
|
|
123
|
-
tool: tool.name,
|
|
124
|
-
message: `${tool.name}.${param}: looks like a ${kind} ID the agent must supply, but no tool appears to return one (no list/search/get tool for "${kind}", no output schema with a ${kind} ID). The agent may invent it. Heuristic.`,
|
|
125
|
-
});
|
|
135
|
+
byKind.set(kind, [...(byKind.get(kind) ?? []), { tool: tool.name, param }]);
|
|
126
136
|
}
|
|
127
137
|
}
|
|
128
|
-
|
|
138
|
+
// One finding per kind: 17 tools taking an orgId are one question, not 17.
|
|
139
|
+
return [...byKind].map(([kind, uses]) => {
|
|
140
|
+
const tools = [...new Set(uses.map((u) => u.tool))];
|
|
141
|
+
const where = uses.map((u) => `${u.tool}.${u.param}`);
|
|
142
|
+
return {
|
|
143
|
+
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
144
|
+
message: `${where.length === 1 ? `${where[0]} looks like` : `${where.length} parameters look like`} a ${kind} ID the agent must supply, but no tool appears to return one (no list/search/get tool for "${kind}", no output schema with a ${kind} ID). The agent may invent it, unless it comes from somewhere toolmenu can't see (a URL, another tool's text output). Heuristic.`,
|
|
145
|
+
...(where.length > 1 ? { detail: [where.slice(0, 8).join(', ') + (where.length > 8 ? `, and ${where.length - 8} more` : '')] } : {}),
|
|
146
|
+
};
|
|
147
|
+
});
|
|
129
148
|
},
|
|
130
149
|
};
|
|
131
|
-
|
|
150
|
+
// Where the value comes from: another tool, or a URL the user has ("often found as
|
|
151
|
+
// URL parameter node-id": Figma). A bare mention of a URL isn't enough: Sentry's
|
|
152
|
+
// "Required when not using a URL" names an alternative input, not a source.
|
|
153
|
+
const PROVENANCE = /\b(from|returned by|obtained|output of|result of|as (listed|shown|returned))\b|\bfound (in|as|on)\b[^.]{0,40}\b(url|link)\b|\b(url|query) param(eter)?\b/i;
|
|
132
154
|
/** The parameter's description says where the value comes from, or names the tool that gives it. */
|
|
133
155
|
function saysWhereItComesFrom(schema, tools, self) {
|
|
134
156
|
const text = typeof schema?.description === 'string' ? schema.description : '';
|
package/dist/rules/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { SEVERITY_RANK } from '../types.js';
|
|
2
2
|
import { buried, cut } from './description.js';
|
|
3
|
-
import { nondeterministic } from './determinism.js';
|
|
3
|
+
import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
5
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
6
6
|
import { cacheHints, deprecated, discover, schema } from './spec.js';
|
|
7
7
|
import { noDryRun, unannotated } from './write.js';
|
|
8
8
|
export const MENU_RULES = [
|
|
9
9
|
nondeterministic,
|
|
10
|
+
processVariance,
|
|
11
|
+
connectionVariance,
|
|
10
12
|
vagueId,
|
|
11
13
|
sharedWord,
|
|
12
14
|
route,
|
package/dist/rules/naming.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { commonWords, nouns, singular, words } from '../words.js';
|
|
2
2
|
import { missingWords, negatedMention, routeScores } from '../routes.js';
|
|
3
|
-
|
|
3
|
+
// Not `ref`: in real menus it's a git ref (GitHub ×4), a name with a meaning of its own.
|
|
4
|
+
const VAGUE_IDS = /^(id|ids|uuid|uuids|identifier|identifiers)$/i;
|
|
4
5
|
export const vagueId = {
|
|
5
6
|
id: 'naming/vague-id',
|
|
6
7
|
severity: 'warn',
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -13,6 +13,12 @@ export interface RuleContext {
|
|
|
13
13
|
capabilities: Record<string, unknown>;
|
|
14
14
|
usedAuth: boolean;
|
|
15
15
|
routes?: Routes;
|
|
16
|
+
/** stdio or HTTP: which variance rule applies. */
|
|
17
|
+
transport?: 'stdio' | 'http';
|
|
18
|
+
/** Menus from fresh processes (stdio) or connections (HTTP), for the variance rules. */
|
|
19
|
+
probes?: import('./determinism.js').Probe[];
|
|
20
|
+
/** The stdio command is a container wrapper (docker run …): env vars may not reach the server. */
|
|
21
|
+
wrapper?: boolean;
|
|
16
22
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
|
17
23
|
descriptionLimit?: number | string;
|
|
18
24
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
package/dist/session.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ToolChange } from './compare.js';
|
|
2
2
|
import { type Target } from './connect.js';
|
|
3
|
+
import { type FailureClass } from './failures.js';
|
|
3
4
|
import type { Era, Finding, Menu, MenuTool, Severity } from './types.js';
|
|
4
5
|
export type Step = {
|
|
5
6
|
kind: 'list';
|
|
@@ -28,6 +29,8 @@ export interface StepRecord {
|
|
|
28
29
|
/** Why no call was made, when none was: set where the status is set, never parsed from text. */
|
|
29
30
|
reason?: 'missing' | 'refused';
|
|
30
31
|
note?: string;
|
|
32
|
+
/** Why the call failed, when it did (a tool error or a failed request). */
|
|
33
|
+
failure?: FailureClass;
|
|
31
34
|
changed: boolean;
|
|
32
35
|
listChanged: number;
|
|
33
36
|
scope?: Scope;
|
|
@@ -47,9 +50,20 @@ export interface SessionResult {
|
|
|
47
50
|
tools: number;
|
|
48
51
|
tokens: number;
|
|
49
52
|
};
|
|
53
|
+
/** Undefined when not checked (processes: 1). */
|
|
50
54
|
connectionCheck?: 'same' | 'different';
|
|
51
55
|
steps: StepRecord[];
|
|
52
56
|
findings: Finding[];
|
|
57
|
+
/**
|
|
58
|
+
* Every tool the session saw, in the order first seen, each as last seen: the
|
|
59
|
+
* menu to diff between releases when tools only appear after an unlock.
|
|
60
|
+
*/
|
|
61
|
+
union: Menu;
|
|
62
|
+
/** With --auto: which tools were called, and which weren't and why. */
|
|
63
|
+
auto?: {
|
|
64
|
+
called: string[];
|
|
65
|
+
skipped: import('./auto.js').AutoPlan['skipped'];
|
|
66
|
+
};
|
|
53
67
|
}
|
|
54
68
|
export interface SessionOptions {
|
|
55
69
|
timeoutMs?: number;
|
|
@@ -58,6 +72,8 @@ export interface SessionOptions {
|
|
|
58
72
|
rules?: Record<string, Severity | 'off'>;
|
|
59
73
|
ignore?: string[];
|
|
60
74
|
scenarioName?: string;
|
|
75
|
+
/** Server processes (stdio) or connections (HTTP) to compare before the first step, the main one included (default 2). */
|
|
76
|
+
processes?: number;
|
|
61
77
|
}
|
|
62
78
|
type Raw = Omit<Finding, 'severity'> & {
|
|
63
79
|
severity: Severity;
|
package/dist/session.js
CHANGED
|
@@ -2,7 +2,10 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
3
|
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
4
|
import { connect, listTools } from './connect.js';
|
|
5
|
-
import { buildMenu } from './menu.js';
|
|
5
|
+
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
|
+
import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
|
+
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
+
import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
|
|
6
9
|
import { SEVERITY_RANK } from './types.js';
|
|
7
10
|
import { verbOf, WRITE_VERBS } from './words.js';
|
|
8
11
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
@@ -63,7 +66,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
63
66
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
64
67
|
const grace = options.noticeGraceMs ?? 500;
|
|
65
68
|
const raw = [];
|
|
66
|
-
|
|
69
|
+
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
70
|
+
// a probe with a different seed would count ordering variance as a change.
|
|
71
|
+
const mainTarget = seeded(target, MAIN_SEED);
|
|
72
|
+
const conn = await connect(mainTarget, { timeoutMs });
|
|
67
73
|
try {
|
|
68
74
|
const era = conn.era;
|
|
69
75
|
const modern = era === 'modern';
|
|
@@ -80,23 +86,23 @@ export async function session(target, scenario, options = {}) {
|
|
|
80
86
|
}
|
|
81
87
|
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
82
88
|
let current = await menuOf(conn);
|
|
89
|
+
// Map keeps first-insertion order; set() on a known name updates it in place.
|
|
90
|
+
const seen = new Map();
|
|
91
|
+
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
92
|
+
see(current);
|
|
83
93
|
const baseline = current;
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
raw.push({
|
|
91
|
-
rule: 'session/connection-variance',
|
|
92
|
-
severity: modern ? 'error' : 'warn',
|
|
93
|
-
step: 0,
|
|
94
|
-
message: `A second connection with the same credentials got a different menu.${modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
|
|
95
|
-
detail,
|
|
96
|
-
});
|
|
94
|
+
// Fresh processes or connections, same credentials, must see the same menu.
|
|
95
|
+
let connectionCheck;
|
|
96
|
+
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
|
|
97
|
+
if (probe.error) {
|
|
98
|
+
raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
|
|
99
|
+
continue;
|
|
97
100
|
}
|
|
98
|
-
|
|
99
|
-
|
|
101
|
+
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
102
|
+
connectionCheck = f ? 'different' : connectionCheck ?? 'same';
|
|
103
|
+
if (f) {
|
|
104
|
+
raw.push({ ...f, step: 0 });
|
|
105
|
+
break;
|
|
100
106
|
}
|
|
101
107
|
}
|
|
102
108
|
const steps = [];
|
|
@@ -136,19 +142,25 @@ export async function session(target, scenario, options = {}) {
|
|
|
136
142
|
.trim();
|
|
137
143
|
const short = clip(text, 120);
|
|
138
144
|
record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
145
|
+
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
146
|
+
// A setup failure is reported once, for the whole run (session/untested).
|
|
147
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
148
|
+
raw.push({
|
|
149
|
+
rule: 'session/tool-error',
|
|
150
|
+
severity: 'warn',
|
|
151
|
+
step: index,
|
|
152
|
+
tool: step.tool,
|
|
153
|
+
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
154
|
+
});
|
|
146
155
|
}
|
|
147
156
|
}
|
|
148
157
|
catch (error) {
|
|
149
158
|
record.status = 'failed';
|
|
150
159
|
record.note = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
151
|
-
|
|
160
|
+
const code = error.code;
|
|
161
|
+
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
162
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
163
|
+
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
|
|
152
164
|
}
|
|
153
165
|
}
|
|
154
166
|
else if (step.kind === 'wait_for') {
|
|
@@ -197,17 +209,19 @@ export async function session(target, scenario, options = {}) {
|
|
|
197
209
|
});
|
|
198
210
|
}
|
|
199
211
|
if (step.kind !== 'list') {
|
|
200
|
-
const probe = await
|
|
212
|
+
const probe = await probeMenu(mainTarget, timeoutMs);
|
|
201
213
|
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind);
|
|
202
214
|
raw.push(...scopeFindings(record.scope, modern, index));
|
|
203
215
|
}
|
|
204
216
|
current = next;
|
|
217
|
+
see(next);
|
|
205
218
|
}
|
|
206
219
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
207
220
|
record.tools = current.tools.length;
|
|
208
221
|
record.tokens = current.totalTokens;
|
|
209
222
|
steps.push(record);
|
|
210
223
|
}
|
|
224
|
+
raw.push(...untested(steps, target));
|
|
211
225
|
return {
|
|
212
226
|
scenario: options.scenarioName ?? 'scenario',
|
|
213
227
|
server: baseline.server,
|
|
@@ -218,27 +232,41 @@ export async function session(target, scenario, options = {}) {
|
|
|
218
232
|
connectionCheck,
|
|
219
233
|
steps,
|
|
220
234
|
findings: settle(mergeSideEffects(raw), options),
|
|
235
|
+
union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
|
|
221
236
|
};
|
|
222
237
|
}
|
|
223
238
|
finally {
|
|
224
239
|
await conn.close().catch(() => { });
|
|
225
240
|
}
|
|
226
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Calls that failed before reaching the tool's logic (credentials, this machine,
|
|
244
|
+
* the network) tested nothing but whether a failed call changes the menu. Said
|
|
245
|
+
* once. An error when no call got through at all: otherwise an expired CI secret
|
|
246
|
+
* turns the job green while testing nothing.
|
|
247
|
+
*/
|
|
248
|
+
function untested(steps, target) {
|
|
249
|
+
const calls = steps.filter((s) => s.label.startsWith('call ') && s.reason === undefined);
|
|
250
|
+
const setup = calls.filter((s) => s.failure && SETUP_FAILURES.has(s.failure));
|
|
251
|
+
if (setup.length === 0)
|
|
252
|
+
return [];
|
|
253
|
+
const byClass = new Map();
|
|
254
|
+
for (const s of setup)
|
|
255
|
+
byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
|
|
256
|
+
const all = setup.length === calls.length;
|
|
257
|
+
const how = target.kind === 'http' ? 'real credentials (--header "Authorization: …")' : 'real credentials (--env KEY=…)';
|
|
258
|
+
return [
|
|
259
|
+
{
|
|
260
|
+
rule: 'session/untested',
|
|
261
|
+
severity: all ? 'error' : 'warn',
|
|
262
|
+
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.${byClass.has('auth') || byClass.has('not-found') ? ` Run with ${how}.` : ''}`,
|
|
263
|
+
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
|
|
264
|
+
},
|
|
265
|
+
];
|
|
266
|
+
}
|
|
227
267
|
function serverOf(c) {
|
|
228
268
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
229
269
|
}
|
|
230
|
-
async function freshMenu(target, timeoutMs) {
|
|
231
|
-
const c = await connect(target, { timeoutMs });
|
|
232
|
-
try {
|
|
233
|
-
return buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
234
|
-
}
|
|
235
|
-
finally {
|
|
236
|
-
await c.close().catch(() => { });
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
function sameMenu(a, b) {
|
|
240
|
-
return compareMenus(a.tools, b.tools).length === 0;
|
|
241
|
-
}
|
|
242
270
|
function waitFor(done, ms) {
|
|
243
271
|
return new Promise((resolve) => {
|
|
244
272
|
if (done())
|
|
@@ -261,11 +289,15 @@ const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annot
|
|
|
261
289
|
export function changeFindings(before, after, changes, step, origin) {
|
|
262
290
|
const out = [];
|
|
263
291
|
const brk = cacheBreak(before, after);
|
|
292
|
+
// Claude's docs: adding, removing or reordering a tool invalidates the entire
|
|
293
|
+
// cache (SPEC §20), not just what follows the change. So the cost is the whole
|
|
294
|
+
// tool list and everything after it; toolmenu can measure this server's part.
|
|
295
|
+
const total = after.reduce((sum, t) => sum + t.tokens, 0);
|
|
264
296
|
const cost = !brk
|
|
265
297
|
? []
|
|
266
|
-
:
|
|
267
|
-
|
|
268
|
-
|
|
298
|
+
: [
|
|
299
|
+
`the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}; any change to the tool list invalidates the cached prompt, so the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it are processed again`,
|
|
300
|
+
];
|
|
269
301
|
const why = origin ? [origin] : [];
|
|
270
302
|
const added = changes.filter((c) => c.kind === 'added');
|
|
271
303
|
const others = changes.filter((c) => c.kind !== 'added');
|
|
@@ -289,7 +321,7 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
289
321
|
rule: 'session/mid-insert',
|
|
290
322
|
severity: 'error',
|
|
291
323
|
step,
|
|
292
|
-
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt
|
|
324
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
293
325
|
detail: [...cost, ...why],
|
|
294
326
|
});
|
|
295
327
|
}
|
|
@@ -389,7 +421,11 @@ function settle(raw, options) {
|
|
|
389
421
|
* back it up. Words like "scope" or "mode" alone say nothing: `get_my_scope_ids`
|
|
390
422
|
* changes no menu.
|
|
391
423
|
*/
|
|
392
|
-
|
|
424
|
+
// A parameter that names what to unlock. Strong names say so alone; weak ones
|
|
425
|
+
// (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
|
|
426
|
+
// description to back them up.
|
|
427
|
+
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?|namespaces?)$/i;
|
|
428
|
+
const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?)$/i;
|
|
393
429
|
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
394
430
|
const UNLOCK_DESC = /(?:unlock|enable|activate|load|expose|add)s?[^.]{0,60}tools?|more tools|toolsets?|capabilit(?:y|ies)|unlock/i;
|
|
395
431
|
/**
|
|
@@ -423,7 +459,8 @@ export function unlockers(tools) {
|
|
|
423
459
|
const found = [];
|
|
424
460
|
for (const tool of tools) {
|
|
425
461
|
const props = tool.inputSchema?.properties ?? {};
|
|
426
|
-
const
|
|
462
|
+
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(tool.description ?? '');
|
|
463
|
+
const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ?? (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined);
|
|
427
464
|
let score = 0;
|
|
428
465
|
if (param)
|
|
429
466
|
score += enumOf(props[param]).length ? 3 : 2;
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -7,6 +7,11 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
7
7
|
timeoutMs?: number;
|
|
8
8
|
descriptionLimit?: number | string;
|
|
9
9
|
fullDescriptions?: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Server processes (stdio) or connections (HTTP) to compare, the main one
|
|
12
|
+
* included. Default 2; 1 turns the check off.
|
|
13
|
+
*/
|
|
14
|
+
processes?: number;
|
|
10
15
|
}
|
|
11
16
|
export interface SnapshotResult {
|
|
12
17
|
menu: Menu;
|