toolmenu 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +311 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +283 -0
- package/dist/compare.d.ts +32 -0
- package/dist/compare.js +108 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.js +27 -0
- package/dist/connect.d.ts +66 -0
- package/dist/connect.js +122 -0
- package/dist/diff.d.ts +58 -0
- package/dist/diff.js +404 -0
- package/dist/history.d.ts +88 -0
- package/dist/history.js +219 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +9 -0
- package/dist/menu.d.ts +9 -0
- package/dist/menu.js +57 -0
- package/dist/report.d.ts +11 -0
- package/dist/report.js +278 -0
- package/dist/routes.d.ts +22 -0
- package/dist/routes.js +76 -0
- package/dist/rules/description.d.ts +28 -0
- package/dist/rules/description.js +158 -0
- package/dist/rules/determinism.d.ts +2 -0
- package/dist/rules/determinism.js +35 -0
- package/dist/rules/ids.d.ts +11 -0
- package/dist/rules/ids.js +138 -0
- package/dist/rules/index.d.ts +11 -0
- package/dist/rules/index.js +42 -0
- package/dist/rules/naming.d.ts +4 -0
- package/dist/rules/naming.js +145 -0
- package/dist/rules/rule.d.ts +33 -0
- package/dist/rules/rule.js +1 -0
- package/dist/rules/spec.d.ts +5 -0
- package/dist/rules/spec.js +102 -0
- package/dist/rules/write.d.ts +3 -0
- package/dist/rules/write.js +55 -0
- package/dist/semver.d.ts +11 -0
- package/dist/semver.js +41 -0
- package/dist/session.d.ts +90 -0
- package/dist/session.js +519 -0
- package/dist/snapshot.d.ts +16 -0
- package/dist/snapshot.js +42 -0
- package/dist/types.d.ts +49 -0
- package/dist/types.js +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/dist/words.d.ts +14 -0
- package/dist/words.js +63 -0
- package/package.json +57 -0
- package/schemas/2024-11-05.json +2077 -0
- package/schemas/2025-03-26.json +2139 -0
- package/schemas/2025-06-18.json +2517 -0
- package/schemas/2025-11-25.json +4058 -0
- package/schemas/2026-07-28.json +3963 -0
package/dist/session.js
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { parse } from 'yaml';
|
|
3
|
+
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
|
+
import { connect, listTools } from './connect.js';
|
|
5
|
+
import { buildMenu } from './menu.js';
|
|
6
|
+
import { SEVERITY_RANK } from './types.js';
|
|
7
|
+
import { verbOf, WRITE_VERBS } from './words.js';
|
|
8
|
+
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
9
|
+
/** Read and check a scenario file. Mistakes are reported before anything runs. */
|
|
10
|
+
export async function loadScenario(path) {
|
|
11
|
+
let data;
|
|
12
|
+
try {
|
|
13
|
+
data = parse(await readFile(path, 'utf8'));
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
17
|
+
}
|
|
18
|
+
return parseScenario(data, path);
|
|
19
|
+
}
|
|
20
|
+
export function parseScenario(data, where = 'scenario') {
|
|
21
|
+
const doc = data;
|
|
22
|
+
if (!doc || typeof doc !== 'object' || !Array.isArray(doc.steps) || doc.steps.length === 0) {
|
|
23
|
+
throw new Error(`${where}: expected "steps:" with at least one step`);
|
|
24
|
+
}
|
|
25
|
+
if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
|
|
26
|
+
throw new Error(`${where}: allow_writes must be true or false`);
|
|
27
|
+
}
|
|
28
|
+
const steps = doc.steps.map((raw, i) => {
|
|
29
|
+
const at = `${where}: step ${i + 1}`;
|
|
30
|
+
if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
|
|
31
|
+
return { kind: 'list' };
|
|
32
|
+
if (raw && typeof raw === 'object' && 'call' in raw) {
|
|
33
|
+
const r = raw;
|
|
34
|
+
if (typeof r.call !== 'string' || !r.call)
|
|
35
|
+
throw new Error(`${at}: "call" needs a tool name`);
|
|
36
|
+
if (r.args !== undefined && (typeof r.args !== 'object' || r.args === null || Array.isArray(r.args))) {
|
|
37
|
+
throw new Error(`${at}: "args" must be a map`);
|
|
38
|
+
}
|
|
39
|
+
return { kind: 'call', tool: r.call, args: r.args ?? {} };
|
|
40
|
+
}
|
|
41
|
+
if (raw && typeof raw === 'object' && 'wait_for' in raw) {
|
|
42
|
+
const r = raw;
|
|
43
|
+
if (r.wait_for !== 'tools_list_changed')
|
|
44
|
+
throw new Error(`${at}: wait_for supports "tools_list_changed"`);
|
|
45
|
+
if (r.timeout_ms !== undefined && (typeof r.timeout_ms !== 'number' || r.timeout_ms <= 0)) {
|
|
46
|
+
throw new Error(`${at}: timeout_ms must be a positive number`);
|
|
47
|
+
}
|
|
48
|
+
return { kind: 'wait_for', event: 'tools_list_changed', timeoutMs: r.timeout_ms ?? 5000 };
|
|
49
|
+
}
|
|
50
|
+
throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
|
|
51
|
+
});
|
|
52
|
+
return { allowWrites: doc.allow_writes === true, steps };
|
|
53
|
+
}
|
|
54
|
+
export function stepLabel(step) {
|
|
55
|
+
if (step.kind === 'list')
|
|
56
|
+
return 'list';
|
|
57
|
+
if (step.kind === 'call')
|
|
58
|
+
return `call ${step.tool}${Object.keys(step.args).length ? ' ' + JSON.stringify(step.args) : ''}`;
|
|
59
|
+
return `wait_for ${step.event}`;
|
|
60
|
+
}
|
|
61
|
+
/** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
|
|
62
|
+
export async function session(target, scenario, options = {}) {
|
|
63
|
+
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
64
|
+
const grace = options.noticeGraceMs ?? 500;
|
|
65
|
+
const raw = [];
|
|
66
|
+
const conn = await connect(target, { timeoutMs });
|
|
67
|
+
try {
|
|
68
|
+
const era = conn.era;
|
|
69
|
+
const modern = era === 'modern';
|
|
70
|
+
const declared = conn.capabilities.tools?.listChanged === true;
|
|
71
|
+
let listening = !modern; // 2025-era notifications arrive on the session itself
|
|
72
|
+
if (modern && declared) {
|
|
73
|
+
try {
|
|
74
|
+
await conn.client.listen({ toolsListChanged: true });
|
|
75
|
+
listening = true;
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
listening = false;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
82
|
+
let current = await menuOf(conn);
|
|
83
|
+
const baseline = current;
|
|
84
|
+
// Two fresh connections, same credentials, must see the same menu.
|
|
85
|
+
const fresh = await freshMenu(target, timeoutMs);
|
|
86
|
+
const connectionCheck = sameMenu(fresh, current) ? 'same' : 'different';
|
|
87
|
+
if (connectionCheck === 'different') {
|
|
88
|
+
const detail = describe(compareMenus(current.tools, fresh.tools));
|
|
89
|
+
if (target.kind === 'http') {
|
|
90
|
+
raw.push({
|
|
91
|
+
rule: 'session/connection-variance',
|
|
92
|
+
severity: modern ? 'error' : 'warn',
|
|
93
|
+
step: 0,
|
|
94
|
+
message: `A second connection with the same credentials got a different menu.${modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
|
|
95
|
+
detail,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
raw.push({ rule: 'session/connection-variance', severity: 'warn', step: 0, message: 'A second server process returned a different menu. Every client that starts this server gets its own menu, so none can share a cached prefix.', detail });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const steps = [];
|
|
103
|
+
for (const [i, step] of scenario.steps.entries()) {
|
|
104
|
+
const index = i + 1;
|
|
105
|
+
const label = stepLabel(step);
|
|
106
|
+
const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
|
|
107
|
+
const mark = conn.wire.notifications.length;
|
|
108
|
+
if (step.kind === 'call') {
|
|
109
|
+
const tool = current.tools.find((t) => t.name === step.tool);
|
|
110
|
+
if (!tool) {
|
|
111
|
+
record.status = 'failed';
|
|
112
|
+
record.reason = 'missing';
|
|
113
|
+
record.note = `${step.tool} isn't in the menu at this point`;
|
|
114
|
+
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `Step ${index} calls ${step.tool}, which isn't in the menu at this point.` });
|
|
115
|
+
}
|
|
116
|
+
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
|
|
117
|
+
record.status = 'refused';
|
|
118
|
+
record.reason = 'refused';
|
|
119
|
+
record.note = 'not marked readOnlyHint';
|
|
120
|
+
raw.push({
|
|
121
|
+
rule: 'session/refused',
|
|
122
|
+
severity: 'error',
|
|
123
|
+
step: index,
|
|
124
|
+
tool: step.tool,
|
|
125
|
+
message: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real. Set "allow_writes: true" in the scenario if that's intended.`,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (record.status === 'ok')
|
|
129
|
+
try {
|
|
130
|
+
const result = await conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs });
|
|
131
|
+
if (result.isError) {
|
|
132
|
+
const text = (Array.isArray(result.content) ? result.content : [])
|
|
133
|
+
.map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
|
|
134
|
+
.join(' ')
|
|
135
|
+
.replace(/\s+/g, ' ')
|
|
136
|
+
.trim();
|
|
137
|
+
const short = clip(text, 120);
|
|
138
|
+
record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
139
|
+
raw.push({
|
|
140
|
+
rule: 'session/tool-error',
|
|
141
|
+
severity: 'warn',
|
|
142
|
+
step: index,
|
|
143
|
+
tool: step.tool,
|
|
144
|
+
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
record.status = 'failed';
|
|
150
|
+
record.note = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
151
|
+
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
else if (step.kind === 'wait_for') {
|
|
155
|
+
const arrived = await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, step.timeoutMs);
|
|
156
|
+
if (!arrived) {
|
|
157
|
+
record.status = 'timed-out';
|
|
158
|
+
record.note = `no list_changed within ${step.timeoutMs} ms`;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// List even after a failed step: a call that errored or timed out may still
|
|
162
|
+
// have changed the menu, and the change belongs to this step, not the next.
|
|
163
|
+
let next;
|
|
164
|
+
try {
|
|
165
|
+
next = await menuOf(conn);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
const why = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
169
|
+
record.status = 'failed';
|
|
170
|
+
record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
|
|
171
|
+
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
|
|
172
|
+
steps.push(record);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
const changes = compareMenus(current.tools, next.tools);
|
|
176
|
+
if (changes.length) {
|
|
177
|
+
record.changed = true;
|
|
178
|
+
if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
179
|
+
await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
|
|
180
|
+
}
|
|
181
|
+
const origin = step.kind === 'list'
|
|
182
|
+
? 'no tool call in between'
|
|
183
|
+
: record.reason === 'missing'
|
|
184
|
+
? 'no tool call made (the tool wasn\'t in the menu)'
|
|
185
|
+
: record.reason === 'refused'
|
|
186
|
+
? 'no tool call made (refused)'
|
|
187
|
+
: record.status === 'failed'
|
|
188
|
+
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
189
|
+
: undefined;
|
|
190
|
+
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
191
|
+
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
192
|
+
raw.push({
|
|
193
|
+
rule: 'session/unannounced',
|
|
194
|
+
severity: 'warn',
|
|
195
|
+
step: index,
|
|
196
|
+
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
if (step.kind !== 'list') {
|
|
200
|
+
const probe = await freshMenu(target, timeoutMs);
|
|
201
|
+
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind);
|
|
202
|
+
raw.push(...scopeFindings(record.scope, modern, index));
|
|
203
|
+
}
|
|
204
|
+
current = next;
|
|
205
|
+
}
|
|
206
|
+
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
207
|
+
record.tools = current.tools.length;
|
|
208
|
+
record.tokens = current.totalTokens;
|
|
209
|
+
steps.push(record);
|
|
210
|
+
}
|
|
211
|
+
return {
|
|
212
|
+
scenario: options.scenarioName ?? 'scenario',
|
|
213
|
+
server: baseline.server,
|
|
214
|
+
transport: target.kind,
|
|
215
|
+
listening,
|
|
216
|
+
baseline: { tools: baseline.tools.length, tokens: baseline.totalTokens },
|
|
217
|
+
final: { tools: current.tools.length, tokens: current.totalTokens },
|
|
218
|
+
connectionCheck,
|
|
219
|
+
steps,
|
|
220
|
+
findings: settle(mergeSideEffects(raw), options),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
finally {
|
|
224
|
+
await conn.close().catch(() => { });
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
function serverOf(c) {
|
|
228
|
+
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
229
|
+
}
|
|
230
|
+
async function freshMenu(target, timeoutMs) {
|
|
231
|
+
const c = await connect(target, { timeoutMs });
|
|
232
|
+
try {
|
|
233
|
+
return buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
234
|
+
}
|
|
235
|
+
finally {
|
|
236
|
+
await c.close().catch(() => { });
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function sameMenu(a, b) {
|
|
240
|
+
return compareMenus(a.tools, b.tools).length === 0;
|
|
241
|
+
}
|
|
242
|
+
function waitFor(done, ms) {
|
|
243
|
+
return new Promise((resolve) => {
|
|
244
|
+
if (done())
|
|
245
|
+
return resolve(true);
|
|
246
|
+
const start = Date.now();
|
|
247
|
+
const timer = setInterval(() => {
|
|
248
|
+
if (done()) {
|
|
249
|
+
clearInterval(timer);
|
|
250
|
+
resolve(true);
|
|
251
|
+
}
|
|
252
|
+
else if (Date.now() - start >= ms) {
|
|
253
|
+
clearInterval(timer);
|
|
254
|
+
resolve(false);
|
|
255
|
+
}
|
|
256
|
+
}, 20);
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
260
|
+
/** Turn a menu change into findings, with the cost of each. */
|
|
261
|
+
export function changeFindings(before, after, changes, step, origin) {
|
|
262
|
+
const out = [];
|
|
263
|
+
const brk = cacheBreak(before, after);
|
|
264
|
+
const cost = !brk
|
|
265
|
+
? []
|
|
266
|
+
: brk.position >= after.length
|
|
267
|
+
? [`the tool list is unchanged up to position ${brk.position}, where tools were removed from the end; with the tool list at the start of the prompt, the conversation after it is processed again`]
|
|
268
|
+
: [`~${brk.tokensAffected.toLocaleString('en-US')} estimated tokens of the tool list from position ${brk.position} on (positions ${brk.position}–${after.length - 1}), a floor: with the tool list at the start of the prompt, the conversation after it is processed again too`];
|
|
269
|
+
const why = origin ? [origin] : [];
|
|
270
|
+
const added = changes.filter((c) => c.kind === 'added');
|
|
271
|
+
const others = changes.filter((c) => c.kind !== 'added');
|
|
272
|
+
const lastKept = Math.max(-1, ...after.map((t, i) => (added.some((a) => a.tool === t.name) ? -1 : i)));
|
|
273
|
+
const appended = added.length > 0 && added.every((a) => a.position > lastKept);
|
|
274
|
+
if (added.length) {
|
|
275
|
+
const names = added.map((a) => a.tool);
|
|
276
|
+
if (appended) {
|
|
277
|
+
const tokens = after.filter((t) => names.includes(t.name)).reduce((s, t) => s + t.tokens, 0);
|
|
278
|
+
out.push({
|
|
279
|
+
rule: 'session/append',
|
|
280
|
+
severity: 'warn',
|
|
281
|
+
step,
|
|
282
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}). The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.`,
|
|
283
|
+
detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
const first = Math.min(...added.map((a) => a.position));
|
|
288
|
+
out.push({
|
|
289
|
+
rule: 'session/mid-insert',
|
|
290
|
+
severity: 'error',
|
|
291
|
+
step,
|
|
292
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt from that tool on, and the conversation after the tool list.`,
|
|
293
|
+
detail: [...cost, ...why],
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const moved = changes.filter((c) => c.kind === 'moved');
|
|
298
|
+
if (moved.length) {
|
|
299
|
+
out.push({ rule: 'session/reorder', severity: 'error', step, message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}).`, detail: [...cost, ...why] });
|
|
300
|
+
}
|
|
301
|
+
const removed = changes.filter((c) => c.kind === 'removed');
|
|
302
|
+
if (removed.length) {
|
|
303
|
+
out.push({ rule: 'session/remove', severity: 'error', step, message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session.`, detail: [...cost, ...why] });
|
|
304
|
+
}
|
|
305
|
+
const edits = new Map();
|
|
306
|
+
for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
|
|
307
|
+
edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
|
|
308
|
+
for (const [tool, fields] of edits) {
|
|
309
|
+
out.push({ rule: 'session/edit', severity: 'error', step, tool, message: `${tool}: ${fields.join(', ')} changed mid-session.`, detail: [...cost, ...why] });
|
|
310
|
+
}
|
|
311
|
+
return out;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Does a fresh connection see this step's changes? All of them: the server's tool
|
|
315
|
+
* set changed. None: the change belongs to this connection (or, on stdio, this
|
|
316
|
+
* process). Reorders alone can't tell, and a mix is unclear.
|
|
317
|
+
*/
|
|
318
|
+
export function scopeOf(changes, next, probe, transport) {
|
|
319
|
+
const probeByName = new Map(probe.map((t) => [t.name, t]));
|
|
320
|
+
const nextByName = new Map(next.map((t) => [t.name, t]));
|
|
321
|
+
const verdicts = new Set();
|
|
322
|
+
for (const c of changes) {
|
|
323
|
+
if (c.kind === 'moved')
|
|
324
|
+
continue;
|
|
325
|
+
if (c.kind === 'added')
|
|
326
|
+
verdicts.add(probeByName.has(c.tool));
|
|
327
|
+
else if (c.kind === 'removed')
|
|
328
|
+
verdicts.add(!probeByName.has(c.tool));
|
|
329
|
+
else {
|
|
330
|
+
const seen = probeByName.get(c.tool);
|
|
331
|
+
verdicts.add(!!seen && compareMenus([seen], [nextByName.get(c.tool)]).length === 0);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (verdicts.size !== 1)
|
|
335
|
+
return 'unclear';
|
|
336
|
+
if (verdicts.has(true))
|
|
337
|
+
return 'global';
|
|
338
|
+
return transport === 'http' ? 'connection-local' : 'per-process';
|
|
339
|
+
}
|
|
340
|
+
function scopeFindings(scope, modern, step) {
|
|
341
|
+
if (scope === 'connection-local') {
|
|
342
|
+
return [
|
|
343
|
+
{
|
|
344
|
+
rule: 'session/connection-local',
|
|
345
|
+
severity: modern ? 'error' : 'info',
|
|
346
|
+
step,
|
|
347
|
+
message: modern
|
|
348
|
+
? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
|
|
349
|
+
: `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
|
|
350
|
+
},
|
|
351
|
+
];
|
|
352
|
+
}
|
|
353
|
+
if (scope === 'per-process' && modern) {
|
|
354
|
+
return [
|
|
355
|
+
{
|
|
356
|
+
rule: 'session/side-effect',
|
|
357
|
+
severity: 'warn',
|
|
358
|
+
step,
|
|
359
|
+
message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
|
|
360
|
+
},
|
|
361
|
+
];
|
|
362
|
+
}
|
|
363
|
+
return [];
|
|
364
|
+
}
|
|
365
|
+
/** stdio's side-effect warning is the same for every step: say it once, name the steps. */
|
|
366
|
+
function mergeSideEffects(raw) {
|
|
367
|
+
const side = raw.filter((f) => f.rule === 'session/side-effect');
|
|
368
|
+
if (side.length < 2)
|
|
369
|
+
return raw;
|
|
370
|
+
const steps = side.map((f) => f.step);
|
|
371
|
+
return raw
|
|
372
|
+
.filter((f) => f.rule !== 'session/side-effect' || f === side[0])
|
|
373
|
+
.map((f) => (f === side[0] ? { ...f, message: `Steps ${steps.join(', ')}: ${f.message}` } : f));
|
|
374
|
+
}
|
|
375
|
+
function describe(changes) {
|
|
376
|
+
return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
|
|
377
|
+
}
|
|
378
|
+
function settle(raw, options) {
|
|
379
|
+
const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
380
|
+
return raw
|
|
381
|
+
.filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
|
|
382
|
+
.map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
383
|
+
.sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
384
|
+
}
|
|
385
|
+
/*
|
|
386
|
+
* Tools that unlock more tools. A parameter that names what to unlock
|
|
387
|
+
* (`domains`, `toolset`…) is the strongest sign; "unlock"/"capabilities" in
|
|
388
|
+
* the name or description, and a description that talks about loading tools,
|
|
389
|
+
* back it up. Words like "scope" or "mode" alone say nothing: `get_my_scope_ids`
|
|
390
|
+
* changes no menu.
|
|
391
|
+
*/
|
|
392
|
+
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|categor(?:y|ies)|modules?|features?|groups?|packs?|bundles?|namespaces?)$/i;
|
|
393
|
+
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
394
|
+
const UNLOCK_DESC = /(?:unlock|enable|activate|load|expose|add)s?[^.]{0,60}tools?|more tools|toolsets?|capabilit(?:y|ies)|unlock/i;
|
|
395
|
+
/**
|
|
396
|
+
* At most `max` characters, cut by code point at a word boundary, never inside a
|
|
397
|
+
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
398
|
+
*/
|
|
399
|
+
export function clip(text, max) {
|
|
400
|
+
const chars = Array.from(text.replace(/\uFFFD+$/u, ''));
|
|
401
|
+
if (chars.length <= max)
|
|
402
|
+
return chars.join('');
|
|
403
|
+
const cut = chars.slice(0, max - 1).join('');
|
|
404
|
+
const space = cut.lastIndexOf(' ');
|
|
405
|
+
return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
|
|
406
|
+
}
|
|
407
|
+
function isWriteLike(t) {
|
|
408
|
+
if (t.annotations?.destructiveHint === true)
|
|
409
|
+
return true;
|
|
410
|
+
const verb = verbOf(t.name);
|
|
411
|
+
return verb !== undefined && WRITE_VERBS.has(verb) && !UNLOCK_NAME.test(t.name);
|
|
412
|
+
}
|
|
413
|
+
function enumOf(schema) {
|
|
414
|
+
if (!schema)
|
|
415
|
+
return [];
|
|
416
|
+
if (schema.enum?.length)
|
|
417
|
+
return schema.enum;
|
|
418
|
+
if (schema.items?.enum?.length)
|
|
419
|
+
return schema.items.enum;
|
|
420
|
+
return [];
|
|
421
|
+
}
|
|
422
|
+
export function unlockers(tools) {
|
|
423
|
+
const found = [];
|
|
424
|
+
for (const tool of tools) {
|
|
425
|
+
const props = tool.inputSchema?.properties ?? {};
|
|
426
|
+
const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p));
|
|
427
|
+
let score = 0;
|
|
428
|
+
if (param)
|
|
429
|
+
score += enumOf(props[param]).length ? 3 : 2;
|
|
430
|
+
if (UNLOCK_NAME.test(tool.name))
|
|
431
|
+
score += 2;
|
|
432
|
+
if (UNLOCK_DESC.test(tool.description ?? ''))
|
|
433
|
+
score += 1;
|
|
434
|
+
if (score >= 2)
|
|
435
|
+
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
|
|
436
|
+
}
|
|
437
|
+
return found.sort((a, b) => b.score - a.score);
|
|
438
|
+
}
|
|
439
|
+
/** A YAML value for an unlock parameter: an array if the schema takes one. */
|
|
440
|
+
function unlockValue(u, value) {
|
|
441
|
+
const schema = u.tool.inputSchema?.properties?.[u.param];
|
|
442
|
+
return schema?.type === 'array' || schema?.items ? `[${JSON.stringify(value)}]` : JSON.stringify(value);
|
|
443
|
+
}
|
|
444
|
+
export function starterScenario(menu) {
|
|
445
|
+
const readOnly = menu.tools.filter((t) => t.annotations?.readOnlyHint === true);
|
|
446
|
+
const required = (t) => t.inputSchema?.required ?? [];
|
|
447
|
+
const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)));
|
|
448
|
+
const unlocking = new Set(found.filter((u) => u.param).map((u) => u.tool));
|
|
449
|
+
const ready = readOnly.filter((t) => required(t).length === 0 && !unlocking.has(t)).slice(0, 12);
|
|
450
|
+
const needsArgs = readOnly.filter((t) => required(t).length > 0);
|
|
451
|
+
// The best candidate with a real value runs for real: two unlocks, then a repeat.
|
|
452
|
+
const live = found.find((u) => u.values.length > 0 && u.tool.annotations?.readOnlyHint === true && required(u.tool).every((p) => p === u.param));
|
|
453
|
+
const changers = found.filter((u) => u !== live).map((u) => u.tool);
|
|
454
|
+
const example = (t, p) => {
|
|
455
|
+
const schema = t.inputSchema?.properties?.[p];
|
|
456
|
+
const u = found.find((x) => x.tool === t && x.param === p);
|
|
457
|
+
if (u?.values.length)
|
|
458
|
+
return unlockValue(u, u.values[0]);
|
|
459
|
+
if (schema?.enum?.length)
|
|
460
|
+
return JSON.stringify(schema.enum[0]);
|
|
461
|
+
if (schema?.type === 'boolean')
|
|
462
|
+
return 'false';
|
|
463
|
+
if (schema?.type === 'integer' || schema?.type === 'number')
|
|
464
|
+
return '1';
|
|
465
|
+
return 'TODO';
|
|
466
|
+
};
|
|
467
|
+
const argNames = (t) => {
|
|
468
|
+
const u = found.find((x) => x.tool === t);
|
|
469
|
+
return u?.param && !required(t).includes(u.param) ? [...required(t), u.param] : required(t);
|
|
470
|
+
};
|
|
471
|
+
const placeholder = (t) => `{ ${argNames(t).map((p) => `${p}: ${example(t, p)}`).join(', ')} }`;
|
|
472
|
+
const lines = [
|
|
473
|
+
`# toolmenu session scenario for ${menu.server.name ?? 'the server'} ${menu.server.version ?? ''}`.trimEnd(),
|
|
474
|
+
`# Written by \`toolmenu session --init\`. Every step lists the menu afterwards; toolmenu`,
|
|
475
|
+
`# reports any change, where in the list it happened, and what it may cost the cache.`,
|
|
476
|
+
'#',
|
|
477
|
+
'# Only tools marked readOnlyHint are called. Replace each TODO with a real value',
|
|
478
|
+
'# and uncomment the steps that matter, above all anything that unlocks tools.',
|
|
479
|
+
'allow_writes: false',
|
|
480
|
+
'steps:',
|
|
481
|
+
' - list',
|
|
482
|
+
];
|
|
483
|
+
for (const t of ready)
|
|
484
|
+
lines.push(` - call: ${t.name}`);
|
|
485
|
+
if (live) {
|
|
486
|
+
const [first, second] = live.values;
|
|
487
|
+
lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock, list, and unlock again:`);
|
|
488
|
+
for (const v of second === undefined ? [first] : [first, second]) {
|
|
489
|
+
lines.push(` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, v)} }`, ' - list');
|
|
490
|
+
}
|
|
491
|
+
lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, first)} }`, ' - list');
|
|
492
|
+
}
|
|
493
|
+
if (changers.length) {
|
|
494
|
+
lines.push('', ` # ${live ? 'These also' : 'These'} look like they could change the menu. Try them:`);
|
|
495
|
+
for (const t of changers.slice(0, 8)) {
|
|
496
|
+
if (t.annotations?.readOnlyHint !== true)
|
|
497
|
+
lines.push(` # (not marked readOnlyHint: needs allow_writes: true)`);
|
|
498
|
+
lines.push(` # - call: ${t.name}`);
|
|
499
|
+
if (argNames(t).length)
|
|
500
|
+
lines.push(` # args: ${placeholder(t)}`);
|
|
501
|
+
lines.push(' # - list');
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
const rest = needsArgs.filter((t) => !changers.includes(t) && t !== live?.tool).slice(0, 10);
|
|
505
|
+
if (rest.length) {
|
|
506
|
+
lines.push('', ' # Read-only tools that need arguments:');
|
|
507
|
+
for (const t of rest)
|
|
508
|
+
lines.push(` # - call: ${t.name}`, ` # args: ${placeholder(t)}`);
|
|
509
|
+
}
|
|
510
|
+
// The unlock block already ends with a list.
|
|
511
|
+
if (!live)
|
|
512
|
+
lines.push('', ' - list');
|
|
513
|
+
if (ready[0])
|
|
514
|
+
lines.push(` # Repeat a call: the menu shouldn't change the second time either.`, ` - call: ${ready[0].name}`, ' - list');
|
|
515
|
+
if (readOnly.length === 0) {
|
|
516
|
+
lines.push('', '# No tool is marked readOnlyHint, so nothing can be called without allow_writes: true.');
|
|
517
|
+
}
|
|
518
|
+
return lines.join('\n') + '\n';
|
|
519
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type Target } from './connect.js';
|
|
2
|
+
import { type RuleSettings } from './rules/index.js';
|
|
3
|
+
import type { Routes } from './routes.js';
|
|
4
|
+
import type { Finding, Menu } from './types.js';
|
|
5
|
+
export interface SnapshotOptions extends RuleSettings {
|
|
6
|
+
routes?: Routes;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
descriptionLimit?: number | string;
|
|
9
|
+
fullDescriptions?: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface SnapshotResult {
|
|
12
|
+
menu: Menu;
|
|
13
|
+
findings: Finding[];
|
|
14
|
+
}
|
|
15
|
+
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
16
|
+
export declare function snapshot(target: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
package/dist/snapshot.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { connect, listTools } from './connect.js';
|
|
2
|
+
import { buildMenu } from './menu.js';
|
|
3
|
+
import { MENU_RULES, runRules } from './rules/index.js';
|
|
4
|
+
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
5
|
+
export async function snapshot(target, options = {}) {
|
|
6
|
+
const connection = await connect(target, { timeoutMs: options.timeoutMs });
|
|
7
|
+
try {
|
|
8
|
+
const first = await listTools(connection, options);
|
|
9
|
+
const second = await listTools(connection, options);
|
|
10
|
+
const server = {
|
|
11
|
+
name: connection.server.name,
|
|
12
|
+
version: connection.server.version,
|
|
13
|
+
protocolVersion: connection.protocolVersion,
|
|
14
|
+
era: connection.era,
|
|
15
|
+
};
|
|
16
|
+
const menu = buildMenu(first.tools, server, first.listMeta);
|
|
17
|
+
const secondMenu = buildMenu(second.tools, server, second.listMeta);
|
|
18
|
+
const findings = runRules(MENU_RULES, {
|
|
19
|
+
menu,
|
|
20
|
+
secondList: secondMenu.tools,
|
|
21
|
+
pages: first.pages,
|
|
22
|
+
clientError: first.clientError,
|
|
23
|
+
protocolVersion: connection.protocolVersion,
|
|
24
|
+
era: connection.era,
|
|
25
|
+
capabilities: connection.capabilities,
|
|
26
|
+
usedAuth: connection.usedAuth,
|
|
27
|
+
routes: options.routes,
|
|
28
|
+
descriptionLimit: options.descriptionLimit,
|
|
29
|
+
fullDescriptions: options.fullDescriptions,
|
|
30
|
+
}, options);
|
|
31
|
+
return { menu, findings };
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
const stderr = connection.stderr().trim();
|
|
35
|
+
if (stderr && error instanceof Error)
|
|
36
|
+
error.message += `\nserver stderr:\n${stderr}`;
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
await connection.close().catch(() => { });
|
|
41
|
+
}
|
|
42
|
+
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export type Severity = 'error' | 'warn' | 'info';
|
|
2
|
+
export declare const SEVERITY_RANK: Record<Severity, number>;
|
|
3
|
+
export interface Finding {
|
|
4
|
+
rule: string;
|
|
5
|
+
severity: Severity;
|
|
6
|
+
message: string;
|
|
7
|
+
/** Tool the finding is about, when there is one. */
|
|
8
|
+
tool?: string;
|
|
9
|
+
/** Extra lines shown under the message. */
|
|
10
|
+
detail?: string[];
|
|
11
|
+
/** For `session`: the scenario step that caused it (0 = before the first step). */
|
|
12
|
+
step?: number;
|
|
13
|
+
}
|
|
14
|
+
/** A tool exactly as the server returned it, plus toolmenu's token estimate. */
|
|
15
|
+
export interface MenuTool {
|
|
16
|
+
name: string;
|
|
17
|
+
description?: string;
|
|
18
|
+
inputSchema?: JsonSchema;
|
|
19
|
+
outputSchema?: JsonSchema;
|
|
20
|
+
annotations?: Record<string, unknown>;
|
|
21
|
+
tokens: number;
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
export interface JsonSchema {
|
|
25
|
+
type?: string | string[];
|
|
26
|
+
properties?: Record<string, JsonSchema>;
|
|
27
|
+
required?: string[];
|
|
28
|
+
enum?: unknown[];
|
|
29
|
+
items?: JsonSchema;
|
|
30
|
+
[key: string]: unknown;
|
|
31
|
+
}
|
|
32
|
+
export type Era = 'modern' | 'legacy';
|
|
33
|
+
export interface Menu {
|
|
34
|
+
toolmenu: 1;
|
|
35
|
+
server: {
|
|
36
|
+
name?: string;
|
|
37
|
+
version?: string;
|
|
38
|
+
protocolVersion?: string;
|
|
39
|
+
era?: Era;
|
|
40
|
+
};
|
|
41
|
+
capturedAt: string;
|
|
42
|
+
/** In server order. Order is the point. */
|
|
43
|
+
tools: MenuTool[];
|
|
44
|
+
totalTokens: number;
|
|
45
|
+
listMeta?: {
|
|
46
|
+
ttlMs?: number;
|
|
47
|
+
cacheScope?: string;
|
|
48
|
+
};
|
|
49
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const SEVERITY_RANK = { info: 0, warn: 1, error: 2 };
|
package/dist/version.js
ADDED
package/dist/words.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Lowercase words in a tool or parameter name: `listTeamAudits`, `list_team_audits` → list, team, audits. */
|
|
2
|
+
export declare function words(name: string): string[];
|
|
3
|
+
export declare function singular(word: string): string;
|
|
4
|
+
export declare const VERBS: Set<string>;
|
|
5
|
+
export declare const LOOKUP_VERBS: Set<string>;
|
|
6
|
+
/** Lookups that return collections: they hand out IDs even when they take one (get_block_children). */
|
|
7
|
+
export declare const COLLECTION_VERBS: Set<string>;
|
|
8
|
+
export declare const WRITE_VERBS: Set<string>;
|
|
9
|
+
/** Words most of a server's tool names share (`firecrawl_`, `API-`, `browser_`): no information. */
|
|
10
|
+
export declare function commonWords(names: string[]): Set<string>;
|
|
11
|
+
/** The tool's verb: the first word that is one, skipping prefixes like `API-`. */
|
|
12
|
+
export declare function verbOf(name: string): string | undefined;
|
|
13
|
+
/** The nouns in a tool name, singular: `list_team_audits` → team, audit. */
|
|
14
|
+
export declare function nouns(name: string): string[];
|