toolmenu 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -23
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +59 -21
- package/dist/auto.d.ts +80 -6
- package/dist/auto.js +255 -19
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +64 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +170 -185
- package/dist/close.d.ts +8 -0
- package/dist/close.js +18 -0
- package/dist/config.d.ts +7 -1
- package/dist/config.js +17 -0
- package/dist/connect.d.ts +14 -0
- package/dist/connect.js +150 -16
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +574 -120
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +113 -0
- package/dist/explain.js +461 -0
- package/dist/failures.d.ts +15 -0
- package/dist/failures.js +59 -2
- package/dist/help.d.ts +11 -0
- package/dist/help.js +205 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +93 -2
- package/dist/init.js +508 -21
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +141 -0
- package/dist/options.js +50 -0
- package/dist/report.d.ts +2 -2
- package/dist/report.js +305 -101
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +86 -42
- package/dist/rules/determinism.d.ts +4 -1
- package/dist/rules/determinism.js +175 -11
- package/dist/rules/gated.d.ts +7 -0
- package/dist/rules/gated.js +77 -0
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +18 -6
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +91 -21
- package/dist/rules/rule.d.ts +25 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.d.ts +6 -0
- package/dist/rules/spec.js +56 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +64 -14
- package/dist/session.d.ts +34 -9
- package/dist/session.js +525 -94
- package/dist/snapshot.d.ts +8 -2
- package/dist/snapshot.js +33 -2
- package/dist/types.d.ts +21 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +2 -2
package/dist/session.js
CHANGED
|
@@ -5,9 +5,11 @@ import { listTools } from './connect.js';
|
|
|
5
5
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
6
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
-
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently,
|
|
8
|
+
import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
9
9
|
import { SEVERITY_RANK } from './types.js';
|
|
10
10
|
import { leadingJson } from './catalog.js';
|
|
11
|
+
import { allDifferences, describeToolDifference } from './difference.js';
|
|
12
|
+
import { autoNextStep, neededValues, writeSign } from './auto.js';
|
|
11
13
|
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
12
14
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
13
15
|
/** A server saying the session is gone: "Session not found or expired" (toolception). */
|
|
@@ -31,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
31
33
|
if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
|
|
32
34
|
throw new Error(`${where}: allow_writes must be true or false`);
|
|
33
35
|
}
|
|
36
|
+
const assumed = doc.assume_read_only;
|
|
37
|
+
if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
|
|
38
|
+
throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
|
|
39
|
+
}
|
|
34
40
|
const steps = doc.steps.map((raw, i) => {
|
|
35
41
|
const at = `${where}: step ${i + 1}`;
|
|
36
42
|
if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
|
|
@@ -55,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
55
61
|
}
|
|
56
62
|
throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
|
|
57
63
|
});
|
|
58
|
-
return { allowWrites: doc.allow_writes === true, steps };
|
|
64
|
+
return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
|
|
59
65
|
}
|
|
60
66
|
export function stepLabel(step) {
|
|
61
67
|
if (step.kind === 'list')
|
|
@@ -104,9 +110,13 @@ export async function session(target, scenario, options = {}) {
|
|
|
104
110
|
rule: 'session/rate-limited',
|
|
105
111
|
severity: 'error',
|
|
106
112
|
step: index,
|
|
113
|
+
...(tool ? { tool } : {}),
|
|
107
114
|
message: tool
|
|
108
|
-
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}
|
|
109
|
-
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked}
|
|
115
|
+
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
|
|
116
|
+
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
|
|
117
|
+
fix: tool
|
|
118
|
+
? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
|
|
119
|
+
: 'Give the run its own server instance, or raise the limit for it.',
|
|
110
120
|
});
|
|
111
121
|
};
|
|
112
122
|
let current = await listPatiently();
|
|
@@ -115,17 +125,35 @@ export async function session(target, scenario, options = {}) {
|
|
|
115
125
|
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
116
126
|
see(current);
|
|
117
127
|
const baseline = current;
|
|
128
|
+
// Values that differ from one tools/list to the next on their own (a default
|
|
129
|
+
// computed from the clock: PayPal's list_transactions.end_date), by tool and
|
|
130
|
+
// path, with the finding that reported them first. A later change only there
|
|
131
|
+
// has the same cause and points to it (SPEC §25: one root cause, one finding).
|
|
132
|
+
const varying = new Map();
|
|
133
|
+
const learn = (before, after, changes, source) => {
|
|
134
|
+
for (const [key] of variedPaths(before, after, changes))
|
|
135
|
+
if (!varying.has(key))
|
|
136
|
+
varying.set(key, source);
|
|
137
|
+
};
|
|
118
138
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
119
139
|
let connectionCheck;
|
|
120
140
|
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
|
|
121
141
|
if (probe.error) {
|
|
122
|
-
raw.push({
|
|
142
|
+
raw.push({
|
|
143
|
+
rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
|
|
144
|
+
severity: 'info',
|
|
145
|
+
step: 0,
|
|
146
|
+
message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
|
|
147
|
+
fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
|
|
148
|
+
});
|
|
123
149
|
continue;
|
|
124
150
|
}
|
|
125
151
|
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
126
152
|
connectionCheck = f ? 'different' : connectionCheck ?? 'same';
|
|
127
153
|
if (f) {
|
|
128
|
-
|
|
154
|
+
const changes = compareMenus(current.tools, probe.tools ?? []);
|
|
155
|
+
raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
|
|
156
|
+
learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
|
|
129
157
|
break;
|
|
130
158
|
}
|
|
131
159
|
}
|
|
@@ -146,43 +174,102 @@ export async function session(target, scenario, options = {}) {
|
|
|
146
174
|
rule: 'session/session-lost',
|
|
147
175
|
severity: 'error',
|
|
148
176
|
step: index,
|
|
149
|
-
|
|
177
|
+
confidence: 'unsure',
|
|
178
|
+
message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
|
|
179
|
+
fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
|
|
150
180
|
});
|
|
151
181
|
return true;
|
|
152
182
|
};
|
|
153
183
|
const callWaited = { ms: 0 };
|
|
154
184
|
const listWaited = { ms: 0 };
|
|
185
|
+
// Read live: --auto vouches for tools that appear mid-session too.
|
|
186
|
+
const assumed = { has: (tool) => (scenario.assumeReadOnly ?? []).includes(tool) };
|
|
187
|
+
// Called on the user's word (assume_read_only), by step.
|
|
188
|
+
const onWord = [];
|
|
189
|
+
// The same failure from the same tool at several steps is one finding that lists them.
|
|
190
|
+
const failures = new Map();
|
|
191
|
+
const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
|
|
192
|
+
const key = `${rule}\0${step.tool}\0${text}`;
|
|
193
|
+
const before = failures.get(key);
|
|
194
|
+
if (before)
|
|
195
|
+
before.steps.push(index);
|
|
196
|
+
else {
|
|
197
|
+
const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
|
|
198
|
+
failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
// Steps whose change touched only values already known to vary on their own.
|
|
202
|
+
const explained = [];
|
|
203
|
+
// --auto's guessed unlocks: tried once, and those whose first call changed nothing.
|
|
204
|
+
const tried = new Set();
|
|
205
|
+
const quiet = new Set();
|
|
206
|
+
const stopped = new Map();
|
|
155
207
|
for (const [i, step] of scenario.steps.entries()) {
|
|
156
208
|
const index = i + 1;
|
|
209
|
+
if (step.kind === 'call' && step.tentative) {
|
|
210
|
+
if (quiet.has(step.tool)) {
|
|
211
|
+
stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
|
|
216
|
+
if (firstTry)
|
|
217
|
+
tried.add(step.tool);
|
|
157
218
|
const label = stepLabel(step);
|
|
158
219
|
const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
|
|
159
220
|
const mark = conn.wire.notifications.length;
|
|
221
|
+
let word = false;
|
|
222
|
+
const done = () => {
|
|
223
|
+
if (word)
|
|
224
|
+
record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
|
|
225
|
+
steps.push(record);
|
|
226
|
+
};
|
|
160
227
|
if (step.kind === 'call') {
|
|
161
228
|
const tool = current.tools.find((t) => t.name === step.tool);
|
|
229
|
+
const write = tool ? writeSign(tool) : undefined;
|
|
230
|
+
word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
|
|
162
231
|
if (!tool) {
|
|
163
232
|
record.status = 'failed';
|
|
164
233
|
record.reason = 'missing';
|
|
234
|
+
record.outcome = 'not-sent';
|
|
165
235
|
record.note = `${step.tool} isn't in the menu at this point`;
|
|
166
|
-
|
|
236
|
+
const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
|
|
237
|
+
const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
|
|
238
|
+
raw.push({
|
|
239
|
+
rule: 'session/step-failed',
|
|
240
|
+
severity: 'error',
|
|
241
|
+
step: index,
|
|
242
|
+
tool: step.tool,
|
|
243
|
+
message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
|
|
244
|
+
fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
|
|
245
|
+
});
|
|
167
246
|
}
|
|
168
|
-
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
|
|
247
|
+
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
|
|
169
248
|
record.status = 'refused';
|
|
170
249
|
record.reason = 'refused';
|
|
250
|
+
record.outcome = 'not-sent';
|
|
171
251
|
record.note = 'not marked readOnlyHint';
|
|
172
252
|
raw.push({
|
|
173
253
|
rule: 'session/refused',
|
|
174
254
|
severity: 'error',
|
|
175
255
|
step: index,
|
|
176
256
|
tool: step.tool,
|
|
177
|
-
message:
|
|
257
|
+
message: assumed.has(step.tool) && write
|
|
258
|
+
? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
|
|
259
|
+
: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
|
|
260
|
+
fix: write || tool.annotations?.readOnlyHint === false
|
|
261
|
+
? 'Set "allow_writes: true" in the scenario if calling it is intended.'
|
|
262
|
+
: `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
|
|
178
263
|
});
|
|
179
264
|
}
|
|
265
|
+
if (word)
|
|
266
|
+
onWord.push({ tool: step.tool, step: index });
|
|
180
267
|
if (record.status === 'ok')
|
|
181
268
|
try {
|
|
182
269
|
// Sent again after a refusal only when that repeats nothing: the transport's
|
|
183
270
|
// 429 means the server never ran it; a tool that says it was rate-limited,
|
|
184
271
|
// thrown or as its result, may have done part of the work first, so only a
|
|
185
|
-
// read-only one is asked again.
|
|
272
|
+
// read-only one is asked again. On the server's word, not the user's.
|
|
186
273
|
const readOnly = tool?.annotations?.readOnlyHint === true;
|
|
187
274
|
callWaited.ms = 0;
|
|
188
275
|
const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
|
|
@@ -191,13 +278,14 @@ export async function session(target, scenario, options = {}) {
|
|
|
191
278
|
error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
|
|
192
279
|
result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
|
|
193
280
|
});
|
|
281
|
+
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
194
282
|
if (result.isError) {
|
|
195
283
|
const text = errorText(result);
|
|
196
284
|
if (RATE_LIMITED.test(text)) {
|
|
197
285
|
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
198
286
|
record.status = 'failed';
|
|
199
287
|
record.note = 'rate-limited';
|
|
200
|
-
|
|
288
|
+
done();
|
|
201
289
|
break;
|
|
202
290
|
}
|
|
203
291
|
const short = clip(text, 120);
|
|
@@ -205,32 +293,29 @@ export async function session(target, scenario, options = {}) {
|
|
|
205
293
|
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
206
294
|
// A setup failure is reported once, for the whole run (session/untested).
|
|
207
295
|
if (!SETUP_FAILURES.has(record.failure))
|
|
208
|
-
|
|
209
|
-
rule: 'session/tool-error',
|
|
210
|
-
severity: 'warn',
|
|
211
|
-
step: index,
|
|
212
|
-
tool: step.tool,
|
|
213
|
-
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
214
|
-
});
|
|
296
|
+
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
215
297
|
}
|
|
216
298
|
}
|
|
217
299
|
catch (error) {
|
|
218
300
|
record.status = 'failed';
|
|
301
|
+
record.outcome = 'failed';
|
|
219
302
|
record.note = serverWords(error, 200);
|
|
220
303
|
if (lost(record.note, index)) {
|
|
221
|
-
|
|
304
|
+
done();
|
|
222
305
|
break;
|
|
223
306
|
}
|
|
224
307
|
if (tooMany(error)) {
|
|
225
308
|
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
226
309
|
record.note = 'rate-limited';
|
|
227
|
-
|
|
310
|
+
done();
|
|
228
311
|
break;
|
|
229
312
|
}
|
|
230
313
|
const code = error.code;
|
|
231
314
|
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
315
|
+
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
316
|
+
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
232
317
|
if (!SETUP_FAILURES.has(record.failure))
|
|
233
|
-
|
|
318
|
+
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
234
319
|
}
|
|
235
320
|
}
|
|
236
321
|
else if (step.kind === 'wait_for') {
|
|
@@ -252,72 +337,107 @@ export async function session(target, scenario, options = {}) {
|
|
|
252
337
|
record.status = 'failed';
|
|
253
338
|
record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
|
|
254
339
|
if (lost(why, index)) {
|
|
255
|
-
|
|
340
|
+
done();
|
|
256
341
|
break;
|
|
257
342
|
}
|
|
258
343
|
if (tooMany(error)) {
|
|
259
344
|
limited(index, why, listWaited.ms);
|
|
260
345
|
record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
|
|
261
|
-
|
|
346
|
+
done();
|
|
262
347
|
break;
|
|
263
348
|
}
|
|
264
|
-
raw.push({
|
|
265
|
-
|
|
349
|
+
raw.push({
|
|
350
|
+
rule: 'session/step-failed',
|
|
351
|
+
severity: 'error',
|
|
352
|
+
step: index,
|
|
353
|
+
message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
|
|
354
|
+
fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
|
|
355
|
+
});
|
|
356
|
+
done();
|
|
266
357
|
continue;
|
|
267
358
|
}
|
|
268
359
|
const changes = compareMenus(current.tools, next.tools);
|
|
269
360
|
if (changes.length) {
|
|
270
361
|
record.changed = true;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
: record.reason === 'refused'
|
|
279
|
-
? 'no tool call made (refused)'
|
|
280
|
-
: record.status === 'failed'
|
|
281
|
-
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
282
|
-
: undefined;
|
|
283
|
-
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
284
|
-
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
285
|
-
raw.push({
|
|
286
|
-
rule: 'session/unannounced',
|
|
287
|
-
severity: 'warn',
|
|
288
|
-
step: index,
|
|
289
|
-
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
|
|
290
|
-
});
|
|
362
|
+
const paths = variedPaths(current.tools, next.tools, changes);
|
|
363
|
+
const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
|
|
364
|
+
if (onItsOwn) {
|
|
365
|
+
// Same cause as a finding already made: said once, after the steps.
|
|
366
|
+
explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
|
|
367
|
+
const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
|
|
368
|
+
record.note = record.note ? `${record.note}; ${cause}` : cause;
|
|
291
369
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
// the run failed.
|
|
296
|
-
const probeWaited = { ms: 0 };
|
|
297
|
-
try {
|
|
298
|
-
probedAfter = index;
|
|
299
|
-
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
300
|
-
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
301
|
-
raw.push(...scopeFindings(record.scope, modern, index));
|
|
370
|
+
else {
|
|
371
|
+
if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
372
|
+
await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
|
|
302
373
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
374
|
+
const origin = step.kind === 'list'
|
|
375
|
+
? 'no tool call in between'
|
|
376
|
+
: record.reason === 'missing'
|
|
377
|
+
? 'no tool call made (the tool wasn\'t in the menu)'
|
|
378
|
+
: record.reason === 'refused'
|
|
379
|
+
? 'no tool call made (refused)'
|
|
380
|
+
: record.status === 'failed'
|
|
381
|
+
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
382
|
+
: undefined;
|
|
383
|
+
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
384
|
+
// With no call in between, whatever changed varies on its own: a later
|
|
385
|
+
// change only there has the same cause.
|
|
386
|
+
if (step.kind === 'list' || record.reason !== undefined)
|
|
387
|
+
learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
|
|
388
|
+
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
389
|
+
raw.push({
|
|
390
|
+
rule: 'session/unannounced',
|
|
391
|
+
severity: 'warn',
|
|
392
|
+
step: index,
|
|
393
|
+
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
|
|
394
|
+
fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
if (step.kind !== 'list' && probes) {
|
|
398
|
+
// The main process is still running here: a server that holds a file or
|
|
399
|
+
// a port can't start a second copy. That leaves the scope unknown, not
|
|
400
|
+
// the run failed.
|
|
401
|
+
const probeWaited = { ms: 0 };
|
|
402
|
+
try {
|
|
403
|
+
probedAfter = index;
|
|
404
|
+
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
405
|
+
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
406
|
+
raw.push(...scopeFindings(record.scope, modern, index));
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
record.scope = 'unclear';
|
|
410
|
+
scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
|
|
411
|
+
}
|
|
306
412
|
}
|
|
307
413
|
}
|
|
414
|
+
// --auto: the tools this step brought get their calls now, while they're listed.
|
|
415
|
+
const added = new Set(changes.filter((c) => c.kind === 'added').map((c) => c.tool));
|
|
416
|
+
if (options.replan && added.size)
|
|
417
|
+
scenario.steps.splice(i + 1, 0, ...options.replan(next.tools.filter((t) => added.has(t.name))));
|
|
308
418
|
current = next;
|
|
309
419
|
see(next);
|
|
310
420
|
}
|
|
311
421
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
312
422
|
record.tools = current.tools.length;
|
|
313
423
|
record.tokens = current.totalTokens;
|
|
314
|
-
|
|
424
|
+
if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
|
|
425
|
+
quiet.add(step.tool);
|
|
426
|
+
done();
|
|
315
427
|
}
|
|
428
|
+
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
429
|
+
if (explained.length)
|
|
430
|
+
raw.push(knownVariance(explained));
|
|
431
|
+
if (onWord.length)
|
|
432
|
+
raw.push(onTheirWord(onWord, options.auto !== undefined));
|
|
316
433
|
raw.push(...untested(steps, target));
|
|
317
434
|
// --auto means "find what you can": an unlock it skipped is a gap too.
|
|
318
|
-
|
|
435
|
+
const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
|
|
436
|
+
// An unlock --auto stopped trying isn't charged for the values it then left out.
|
|
437
|
+
const covered = baseline.tools.filter((t) => !stopped.has(t.name));
|
|
438
|
+
raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
|
|
319
439
|
if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
|
|
320
|
-
raw.push(nothingCalled(options.auto
|
|
440
|
+
raw.push(nothingCalled(options.auto));
|
|
321
441
|
if (scopeUnchecked.length) {
|
|
322
442
|
const which = scopeUnchecked.map((s) => s.step);
|
|
323
443
|
raw.push({
|
|
@@ -336,8 +456,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
336
456
|
final: { tools: current.tools.length, tokens: current.totalTokens },
|
|
337
457
|
connectionCheck,
|
|
338
458
|
steps,
|
|
339
|
-
findings: settle(
|
|
340
|
-
|
|
459
|
+
findings: settle(mergeRepeats(raw), options),
|
|
460
|
+
...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
|
|
461
|
+
// Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
|
|
462
|
+
union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
|
|
341
463
|
};
|
|
342
464
|
}
|
|
343
465
|
finally {
|
|
@@ -359,13 +481,17 @@ function untested(steps, target) {
|
|
|
359
481
|
for (const s of setup)
|
|
360
482
|
byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
|
|
361
483
|
const all = setup.length === calls.length;
|
|
362
|
-
|
|
484
|
+
// One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
|
|
485
|
+
const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
|
|
363
486
|
return [
|
|
364
487
|
{
|
|
365
488
|
rule: 'session/untested',
|
|
366
489
|
severity: all ? 'error' : 'warn',
|
|
367
|
-
|
|
490
|
+
// Classified from the error's words: a guess, if a good one.
|
|
491
|
+
confidence: 'unsure',
|
|
492
|
+
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
|
|
368
493
|
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
|
|
494
|
+
fix,
|
|
369
495
|
},
|
|
370
496
|
];
|
|
371
497
|
}
|
|
@@ -410,45 +536,252 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
410
536
|
? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
|
|
411
537
|
: skip === 'not read-only'
|
|
412
538
|
? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
|
|
413
|
-
: ''
|
|
539
|
+
: skip === 'unmarked'
|
|
540
|
+
? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
|
|
541
|
+
: '';
|
|
414
542
|
out.push({
|
|
415
543
|
rule: 'session/unlock-coverage',
|
|
416
544
|
severity: 'warn',
|
|
417
545
|
tool: u.tool.name,
|
|
546
|
+
// "Looks like it unlocks": from its name, parameter and description.
|
|
547
|
+
confidence: 'unsure',
|
|
418
548
|
...(last !== undefined ? { step: last } : {}),
|
|
419
|
-
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}
|
|
549
|
+
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
|
|
420
550
|
detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
|
|
551
|
+
fix: skip === 'open world'
|
|
552
|
+
? 'Rerun with --open-world.'
|
|
553
|
+
: skip === 'unmarked'
|
|
554
|
+
? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
|
|
555
|
+
: skip === 'not read-only'
|
|
556
|
+
? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
|
|
557
|
+
: 'Unlock every value: session --init puts each one in the starter.',
|
|
421
558
|
});
|
|
422
559
|
}
|
|
423
560
|
return out;
|
|
424
561
|
}
|
|
425
562
|
/**
|
|
426
563
|
* --auto that called nothing: the run only listed the menu, so a clean result
|
|
427
|
-
* says nothing about what the tools do to it. Why each tool was left out
|
|
564
|
+
* says nothing about what the tools do to it. Why each tool was left out, and the
|
|
565
|
+
* flag that reaches the most of them.
|
|
428
566
|
*/
|
|
429
|
-
function nothingCalled(
|
|
430
|
-
const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
567
|
+
function nothingCalled(auto) {
|
|
568
|
+
const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
431
569
|
const parts = [];
|
|
432
570
|
const openWorld = by('open world');
|
|
433
571
|
const needs = by('needs values');
|
|
572
|
+
const unmarked = by('unmarked');
|
|
434
573
|
const writes = by('not read-only');
|
|
435
|
-
if (openWorld.length)
|
|
436
|
-
parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
|
|
437
574
|
if (needs.length)
|
|
438
|
-
parts.push(`${needs.length} need values the schema doesn't give
|
|
575
|
+
parts.push(`${needs.length} ${needs.length === 1 ? 'needs' : 'need'} values the schema doesn't give`);
|
|
576
|
+
if (unmarked.length)
|
|
577
|
+
parts.push(`${unmarked.length} ${unmarked.length === 1 ? "isn't" : "aren't"} marked readOnlyHint`);
|
|
578
|
+
if (openWorld.length)
|
|
579
|
+
parts.push(`${openWorld.length} marked openWorldHint (${openWorld.length === 1 ? 'it' : 'they'} may cost API credits)`);
|
|
439
580
|
if (writes.length)
|
|
440
|
-
parts.push(`${writes.length}
|
|
581
|
+
parts.push(`${writes.length} marked or named as writes`);
|
|
441
582
|
const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
|
|
583
|
+
const params = neededValues(auto.skipped);
|
|
584
|
+
const fix = autoNextStep(auto);
|
|
442
585
|
return {
|
|
443
586
|
rule: 'session/nothing-called',
|
|
444
587
|
severity: 'warn',
|
|
445
588
|
message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
|
|
446
589
|
detail: [
|
|
590
|
+
...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
|
|
591
|
+
...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
|
|
447
592
|
...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
|
|
448
|
-
...(needs.length ? [`need values: ${list(needs)}`] : []),
|
|
449
593
|
],
|
|
594
|
+
...(fix ? { fix } : {}),
|
|
450
595
|
};
|
|
451
596
|
}
|
|
597
|
+
/**
|
|
598
|
+
* The same error text from different tools is one finding (two tools of a feature
|
|
599
|
+
* that "isn't set up on this deployment"). An empty text says nothing in common.
|
|
600
|
+
*/
|
|
601
|
+
function sameError(failures) {
|
|
602
|
+
const out = [];
|
|
603
|
+
const byText = new Map();
|
|
604
|
+
for (const f of failures) {
|
|
605
|
+
const key = `${f.rule}\0${f.text}`;
|
|
606
|
+
const first = f.text ? byText.get(key) : undefined;
|
|
607
|
+
if (!first) {
|
|
608
|
+
out.push({ ...f, steps: [...f.steps] });
|
|
609
|
+
if (f.text)
|
|
610
|
+
byText.set(key, out[out.length - 1]);
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
|
|
614
|
+
first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
|
|
615
|
+
}
|
|
616
|
+
return out;
|
|
617
|
+
}
|
|
618
|
+
// The server says the feature isn't there, or isn't for this account: no argument
|
|
619
|
+
// fixes that. A guess from its words.
|
|
620
|
+
// Only definite statements: a hedge in the server's own advice ("the account may lack
|
|
621
|
+
// permission") says nothing about this call, and a bare "permission" or "role" turns up in
|
|
622
|
+
// ordinary validation messages.
|
|
623
|
+
const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\b(?:permission|access)s? denied\b|\binsufficient (?:permissions?|privileges?)\b|\b(?:don't|doesn't|do not|does not) have (?:the )?(?:required |necessary )?(?:permissions?|access)\b|\bforbidden\b|\bnot allowed\b/i;
|
|
624
|
+
/** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
|
|
625
|
+
export function stepsText(steps) {
|
|
626
|
+
if (steps.length === 1)
|
|
627
|
+
return `step ${steps[0]}`;
|
|
628
|
+
const runs = [];
|
|
629
|
+
for (let i = 0; i < steps.length;) {
|
|
630
|
+
let j = i;
|
|
631
|
+
while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
|
|
632
|
+
j++;
|
|
633
|
+
if (j - i >= 2)
|
|
634
|
+
runs.push(`${steps[i]}–${steps[j]}`);
|
|
635
|
+
else
|
|
636
|
+
for (let k = i; k <= j; k++)
|
|
637
|
+
runs.push(String(steps[k]));
|
|
638
|
+
i = j + 1;
|
|
639
|
+
}
|
|
640
|
+
return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
|
|
641
|
+
}
|
|
642
|
+
/**
|
|
643
|
+
* The same error from the same tool is one finding that names every step it
|
|
644
|
+
* happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
|
|
645
|
+
* words, the call that got it, and what to try.
|
|
646
|
+
*/
|
|
647
|
+
function failureFindings(failures, auto) {
|
|
648
|
+
return failures.map((f) => {
|
|
649
|
+
const many = f.steps.length > 1;
|
|
650
|
+
const calls = [f, ...(f.others ?? [])];
|
|
651
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
652
|
+
const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
|
|
653
|
+
const words = errorWords(f.text);
|
|
654
|
+
const shown = clip(words, 160);
|
|
655
|
+
const said = f.text ? quotedSentence(shown) : 'no error text.';
|
|
656
|
+
const firstArg = Object.keys(f.args)[0];
|
|
657
|
+
const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
|
|
658
|
+
// No argument can fix a call to a tool that takes none, or a feature the server
|
|
659
|
+
// says isn't there. From the server's words: a guess.
|
|
660
|
+
const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
|
|
661
|
+
const them = tools.length === 1 ? 'its' : 'their';
|
|
662
|
+
const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
|
|
663
|
+
const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
|
|
664
|
+
const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
|
|
665
|
+
const fix = f.timedOutMs
|
|
666
|
+
? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
|
|
667
|
+
: !argsCan
|
|
668
|
+
? undefined
|
|
669
|
+
: unavailable
|
|
670
|
+
? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
671
|
+
: noParams
|
|
672
|
+
? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
673
|
+
: auto?.withValues?.includes(f.tool)
|
|
674
|
+
? `Check the values you gave ${f.tool} (--value) against what the server says above.`
|
|
675
|
+
: auto
|
|
676
|
+
? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
|
|
677
|
+
: `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
|
|
678
|
+
const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
|
|
679
|
+
const detail = calls.slice(0, 8).map((c) => {
|
|
680
|
+
const call = `${c.tool} ${JSON.stringify(c.args)}`;
|
|
681
|
+
return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
|
|
682
|
+
});
|
|
683
|
+
return {
|
|
684
|
+
rule: f.rule,
|
|
685
|
+
severity: f.severity,
|
|
686
|
+
step: f.steps[0],
|
|
687
|
+
...(many ? { steps: f.steps } : {}),
|
|
688
|
+
tool: f.tool,
|
|
689
|
+
// The server's whole text, when the message above shows less of it.
|
|
690
|
+
...(f.text && shown !== f.text ? { serverText: f.text } : {}),
|
|
691
|
+
...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
|
|
692
|
+
message: f.rule === 'session/tool-error'
|
|
693
|
+
? tools.length > 1
|
|
694
|
+
? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
695
|
+
: `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
696
|
+
: tools.length > 1
|
|
697
|
+
? `${lead}Calling ${who} failed the same way: ${said}`
|
|
698
|
+
: `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
|
|
699
|
+
detail: [...new Set(detail)],
|
|
700
|
+
...(fix ? { fix } : {}),
|
|
701
|
+
};
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Changes that touched only values already seen to vary on their own: one info
|
|
706
|
+
* finding for the run, pointing to the one that reported the cause.
|
|
707
|
+
*/
|
|
708
|
+
function knownVariance(explained) {
|
|
709
|
+
const steps = explained.map((e) => e.step);
|
|
710
|
+
const src = explained[0].source;
|
|
711
|
+
const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
|
|
712
|
+
// The values at the first such step; the rest only repeat the pattern.
|
|
713
|
+
const seen = explained[0].seen;
|
|
714
|
+
return {
|
|
715
|
+
rule: 'session/known-variance',
|
|
716
|
+
severity: 'info',
|
|
717
|
+
step: steps[0],
|
|
718
|
+
message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
|
|
719
|
+
detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
|
|
720
|
+
fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
/** Tools called because the user said they only read: said once, loudly. */
|
|
724
|
+
function onTheirWord(calls, auto) {
|
|
725
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
726
|
+
return {
|
|
727
|
+
rule: 'session/assumed-read-only',
|
|
728
|
+
severity: 'info',
|
|
729
|
+
step: calls[0].step,
|
|
730
|
+
message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
|
|
731
|
+
detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
|
|
732
|
+
fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* Every leaf value that differs between the old and new definition of each edited
|
|
737
|
+
* tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
|
|
738
|
+
*/
|
|
739
|
+
function variedPaths(before, after, changes) {
|
|
740
|
+
const out = new Map();
|
|
741
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
742
|
+
for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
|
|
743
|
+
const a = old.get(name);
|
|
744
|
+
const b = after.find((t) => t.name === name);
|
|
745
|
+
if (!a || !b)
|
|
746
|
+
continue;
|
|
747
|
+
for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
|
|
748
|
+
out.set(`${name}\0${path}`, d);
|
|
749
|
+
}
|
|
750
|
+
return out;
|
|
751
|
+
}
|
|
752
|
+
function leafDiffs(a, b, path = '', out = new Map()) {
|
|
753
|
+
if (canonical(a) === canonical(b))
|
|
754
|
+
return out;
|
|
755
|
+
const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
756
|
+
if (obj(a) && obj(b)) {
|
|
757
|
+
for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
|
|
758
|
+
leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
|
|
759
|
+
}
|
|
760
|
+
else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
|
|
761
|
+
a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
|
|
762
|
+
}
|
|
763
|
+
else
|
|
764
|
+
out.set(path, { before: a, after: b });
|
|
765
|
+
return out;
|
|
766
|
+
}
|
|
767
|
+
function show(value) {
|
|
768
|
+
if (value === undefined)
|
|
769
|
+
return '(absent)';
|
|
770
|
+
const text = JSON.stringify(value);
|
|
771
|
+
return text.length > 60 ? text.slice(0, 57) + '…' : text;
|
|
772
|
+
}
|
|
773
|
+
const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
|
|
774
|
+
/** The next step for a menu that differs between processes, from what differs. */
|
|
775
|
+
function varianceFix(main, other, changes) {
|
|
776
|
+
const diffs = [...variedPaths(main, other, changes)];
|
|
777
|
+
if (diffs.length === 0)
|
|
778
|
+
return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
|
|
779
|
+
const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
|
|
780
|
+
if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
|
|
781
|
+
return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
|
|
782
|
+
}
|
|
783
|
+
return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
|
|
784
|
+
}
|
|
452
785
|
function serverOf(c) {
|
|
453
786
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
454
787
|
}
|
|
@@ -477,6 +810,7 @@ function waitFor(done, ms) {
|
|
|
477
810
|
}, 20);
|
|
478
811
|
});
|
|
479
812
|
}
|
|
813
|
+
const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
|
|
480
814
|
const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
481
815
|
/** Turn a menu change into findings, with the cost of each. */
|
|
482
816
|
export function changeFindings(before, after, changes, step, origin) {
|
|
@@ -489,7 +823,7 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
489
823
|
const cost = !brk
|
|
490
824
|
? []
|
|
491
825
|
: [
|
|
492
|
-
`
|
|
826
|
+
`cost: the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it; the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}`,
|
|
493
827
|
];
|
|
494
828
|
const why = origin ? [origin] : [];
|
|
495
829
|
const added = changes.filter((c) => c.kind === 'added');
|
|
@@ -504,8 +838,10 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
504
838
|
rule: 'session/append',
|
|
505
839
|
severity: 'warn',
|
|
506
840
|
step,
|
|
507
|
-
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')})
|
|
841
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
|
|
508
842
|
detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
|
|
843
|
+
each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
|
|
844
|
+
fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
|
|
509
845
|
});
|
|
510
846
|
}
|
|
511
847
|
else {
|
|
@@ -516,22 +852,56 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
516
852
|
step,
|
|
517
853
|
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
518
854
|
detail: [...cost, ...why],
|
|
855
|
+
fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
|
|
519
856
|
});
|
|
520
857
|
}
|
|
521
858
|
}
|
|
522
859
|
const moved = changes.filter((c) => c.kind === 'moved');
|
|
523
860
|
if (moved.length) {
|
|
524
|
-
out.push({
|
|
861
|
+
out.push({
|
|
862
|
+
rule: 'session/reorder',
|
|
863
|
+
severity: 'error',
|
|
864
|
+
step,
|
|
865
|
+
message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
|
|
866
|
+
detail: [...cost, ...why],
|
|
867
|
+
fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
|
|
868
|
+
});
|
|
525
869
|
}
|
|
526
870
|
const removed = changes.filter((c) => c.kind === 'removed');
|
|
527
871
|
if (removed.length) {
|
|
528
|
-
out.push({
|
|
872
|
+
out.push({
|
|
873
|
+
rule: 'session/remove',
|
|
874
|
+
severity: 'error',
|
|
875
|
+
step,
|
|
876
|
+
message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
|
|
877
|
+
detail: [...cost, ...why],
|
|
878
|
+
fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
|
|
879
|
+
});
|
|
529
880
|
}
|
|
530
881
|
const edits = new Map();
|
|
531
882
|
for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
|
|
532
883
|
edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
|
|
884
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
533
885
|
for (const [tool, fields] of edits) {
|
|
534
|
-
|
|
886
|
+
const a = old.get(tool);
|
|
887
|
+
const b = after.find((t) => t.name === tool);
|
|
888
|
+
// What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
|
|
889
|
+
const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
|
|
890
|
+
// One cause: the same tool, the same changed values and the same origin (a
|
|
891
|
+
// timestamp in a description, rewritten at every call). A change that touches
|
|
892
|
+
// anything more is a change of its own.
|
|
893
|
+
const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
|
|
894
|
+
out.push({
|
|
895
|
+
rule: 'session/edit',
|
|
896
|
+
severity: 'error',
|
|
897
|
+
step,
|
|
898
|
+
tool,
|
|
899
|
+
cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
|
|
900
|
+
each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
|
|
901
|
+
message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
902
|
+
detail: [...seen, ...cost, ...why],
|
|
903
|
+
fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
|
|
904
|
+
});
|
|
535
905
|
}
|
|
536
906
|
return out;
|
|
537
907
|
}
|
|
@@ -576,6 +946,7 @@ function scopeFindings(scope, modern, step) {
|
|
|
576
946
|
message: modern
|
|
577
947
|
? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
|
|
578
948
|
: `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
|
|
949
|
+
fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
579
950
|
},
|
|
580
951
|
];
|
|
581
952
|
}
|
|
@@ -586,20 +957,57 @@ function scopeFindings(scope, modern, step) {
|
|
|
586
957
|
severity: 'warn',
|
|
587
958
|
step,
|
|
588
959
|
message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
|
|
960
|
+
fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
589
961
|
},
|
|
590
962
|
];
|
|
591
963
|
}
|
|
592
964
|
return [];
|
|
593
965
|
}
|
|
594
|
-
/**
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
966
|
+
/**
|
|
967
|
+
* Findings that say the same thing at every step are said once, naming the steps:
|
|
968
|
+
* stdio's side-effect warning, and an unlock per domain, where each value appends
|
|
969
|
+
* tools for this connection only (9 steps made 18 near-identical blocks). What
|
|
970
|
+
* differs per step, the tools and their cost, goes in the detail. Mid-inserts
|
|
971
|
+
* and removals stay per step: each is its own change. Edits merge only when the
|
|
972
|
+
* tool, the changed values and their origin are the same: one cause.
|
|
973
|
+
*/
|
|
974
|
+
const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
|
|
975
|
+
export function mergeRepeats(raw) {
|
|
976
|
+
let out = raw;
|
|
977
|
+
for (const rule of REPEATS) {
|
|
978
|
+
const same = out.filter((f) => f.rule === rule);
|
|
979
|
+
if (same.length < 2)
|
|
980
|
+
continue;
|
|
981
|
+
const steps = same.map((f) => f.step);
|
|
982
|
+
const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
|
|
983
|
+
let message = `${lead}: ${same[0].message}`;
|
|
984
|
+
let detail = same[0].detail;
|
|
985
|
+
if (rule === 'session/append') {
|
|
986
|
+
const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
|
|
987
|
+
message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
|
|
988
|
+
const each = same.map((f) => f.each).filter(Boolean);
|
|
989
|
+
const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
|
|
990
|
+
detail = [...each, ...why];
|
|
991
|
+
}
|
|
992
|
+
const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
|
|
993
|
+
out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
994
|
+
}
|
|
995
|
+
// The same edit at several steps is one cause: one finding naming the steps,
|
|
996
|
+
// with each step's position and cost.
|
|
997
|
+
const byCause = new Map();
|
|
998
|
+
for (const f of out)
|
|
999
|
+
if (f.rule === 'session/edit' && f.cause)
|
|
1000
|
+
byCause.set(f.cause, [...(byCause.get(f.cause) ?? []), f]);
|
|
1001
|
+
for (const same of byCause.values()) {
|
|
1002
|
+
if (same.length < 2)
|
|
1003
|
+
continue;
|
|
1004
|
+
const steps = same.map((f) => f.step);
|
|
1005
|
+
// The per-step value and cost are in each step's line; keep the rest (the origin) once.
|
|
1006
|
+
const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.startsWith('cost: ') && !f.each.endsWith(` · ${d}`))))];
|
|
1007
|
+
const merged = { ...same[0], steps, message: `${stepsText(steps).replace(/^s/, 'S')}: ${same[0].message}`, detail: [...same.map((f) => f.each), ...why] };
|
|
1008
|
+
out = out.filter((f) => !same.includes(f) || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
1009
|
+
}
|
|
1010
|
+
return out;
|
|
603
1011
|
}
|
|
604
1012
|
function describe(changes) {
|
|
605
1013
|
return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
|
|
@@ -608,7 +1016,7 @@ function settle(raw, options) {
|
|
|
608
1016
|
const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
609
1017
|
return raw
|
|
610
1018
|
.filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
|
|
611
|
-
.map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
1019
|
+
.map(({ each: _each, cause: _cause, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
612
1020
|
.sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
613
1021
|
}
|
|
614
1022
|
/*
|
|
@@ -630,7 +1038,25 @@ const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand
|
|
|
630
1038
|
// capabilities alone aren't tools.
|
|
631
1039
|
const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
|
|
632
1040
|
// Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
|
|
633
|
-
|
|
1041
|
+
// "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
|
|
1042
|
+
const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
|
|
1043
|
+
/**
|
|
1044
|
+
* Does the tool say that `param` unlocks tools? Its name does, or a sentence of
|
|
1045
|
+
* its description that says it unlocks tools and doesn't pin that on another
|
|
1046
|
+
* parameter ("A query match also enables the tools…" is about `query`, not the
|
|
1047
|
+
* `domain` filter next to it).
|
|
1048
|
+
*/
|
|
1049
|
+
function claimsUnlock(tool, param) {
|
|
1050
|
+
if (UNLOCK_NAME.test(tool.name))
|
|
1051
|
+
return true;
|
|
1052
|
+
const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
|
|
1053
|
+
const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
|
|
1054
|
+
const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
|
|
1055
|
+
return (tool.description ?? '')
|
|
1056
|
+
.split(/(?<=[.!?])\s+/)
|
|
1057
|
+
.filter((s) => UNLOCK_SAYS.test(s))
|
|
1058
|
+
.some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
|
|
1059
|
+
}
|
|
634
1060
|
const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
635
1061
|
/**
|
|
636
1062
|
* At most `max` characters, cut by code point at a word boundary, never inside a
|
|
@@ -672,7 +1098,8 @@ export function unlockers(tools) {
|
|
|
672
1098
|
// A lookup names toolsets without changing them (GitHub's get_toolset_tools,
|
|
673
1099
|
// toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
|
|
674
1100
|
const verb = verbOf(tool.name);
|
|
675
|
-
|
|
1101
|
+
const lookup = verb !== undefined && LOOKUPS.has(verb);
|
|
1102
|
+
if (lookup && !UNLOCK_SAYS.test(description))
|
|
676
1103
|
continue;
|
|
677
1104
|
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
|
|
678
1105
|
const required = tool.inputSchema?.required ?? [];
|
|
@@ -681,6 +1108,10 @@ export function unlockers(tools) {
|
|
|
681
1108
|
// Named like an unlock, one required parameter: that's what it takes
|
|
682
1109
|
// (toolception's enable_toolset { name }).
|
|
683
1110
|
(UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
|
|
1111
|
+
const claims = claimsUnlock(tool, param);
|
|
1112
|
+
// A lookup that says a *different* parameter unlocks: its enum is a filter.
|
|
1113
|
+
if (lookup && !claims)
|
|
1114
|
+
continue;
|
|
684
1115
|
let score = 0;
|
|
685
1116
|
if (param)
|
|
686
1117
|
score += enumOf(props[param]).length ? 3 : 2;
|
|
@@ -689,7 +1120,7 @@ export function unlockers(tools) {
|
|
|
689
1120
|
if (UNLOCK_DESC.test(tool.description ?? ''))
|
|
690
1121
|
score += 1;
|
|
691
1122
|
if (score >= 2)
|
|
692
|
-
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
|
|
1123
|
+
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
|
|
693
1124
|
}
|
|
694
1125
|
return found.sort((a, b) => b.score - a.score);
|
|
695
1126
|
}
|