toolmenu 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -20
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +53 -17
- package/dist/auto.d.ts +70 -6
- package/dist/auto.js +248 -17
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +62 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +80 -170
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +526 -114
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +9 -0
- package/dist/failures.js +17 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +92 -2
- package/dist/init.js +507 -20
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +85 -42
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +145 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +12 -4
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.js +33 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/session.d.ts +18 -8
- package/dist/session.js +484 -92
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +17 -2
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/session.js
CHANGED
|
@@ -5,9 +5,11 @@ import { listTools } from './connect.js';
|
|
|
5
5
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
6
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
-
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently,
|
|
8
|
+
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
9
9
|
import { SEVERITY_RANK } from './types.js';
|
|
10
10
|
import { leadingJson } from './catalog.js';
|
|
11
|
+
import { describeToolDifference } from './difference.js';
|
|
12
|
+
import { autoNextStep, neededValues, writeSign } from './auto.js';
|
|
11
13
|
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
12
14
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
13
15
|
/** A server saying the session is gone: "Session not found or expired" (toolception). */
|
|
@@ -31,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
31
33
|
if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
|
|
32
34
|
throw new Error(`${where}: allow_writes must be true or false`);
|
|
33
35
|
}
|
|
36
|
+
const assumed = doc.assume_read_only;
|
|
37
|
+
if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
|
|
38
|
+
throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
|
|
39
|
+
}
|
|
34
40
|
const steps = doc.steps.map((raw, i) => {
|
|
35
41
|
const at = `${where}: step ${i + 1}`;
|
|
36
42
|
if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
|
|
@@ -55,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
55
61
|
}
|
|
56
62
|
throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
|
|
57
63
|
});
|
|
58
|
-
return { allowWrites: doc.allow_writes === true, steps };
|
|
64
|
+
return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
|
|
59
65
|
}
|
|
60
66
|
export function stepLabel(step) {
|
|
61
67
|
if (step.kind === 'list')
|
|
@@ -104,9 +110,13 @@ export async function session(target, scenario, options = {}) {
|
|
|
104
110
|
rule: 'session/rate-limited',
|
|
105
111
|
severity: 'error',
|
|
106
112
|
step: index,
|
|
113
|
+
...(tool ? { tool } : {}),
|
|
107
114
|
message: tool
|
|
108
|
-
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}
|
|
109
|
-
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked}
|
|
115
|
+
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
|
|
116
|
+
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
|
|
117
|
+
fix: tool
|
|
118
|
+
? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
|
|
119
|
+
: 'Give the run its own server instance, or raise the limit for it.',
|
|
110
120
|
});
|
|
111
121
|
};
|
|
112
122
|
let current = await listPatiently();
|
|
@@ -115,17 +125,35 @@ export async function session(target, scenario, options = {}) {
|
|
|
115
125
|
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
116
126
|
see(current);
|
|
117
127
|
const baseline = current;
|
|
128
|
+
// Values that differ from one tools/list to the next on their own (a default
|
|
129
|
+
// computed from the clock: PayPal's list_transactions.end_date), by tool and
|
|
130
|
+
// path, with the finding that reported them first. A later change only there
|
|
131
|
+
// has the same cause and points to it (SPEC §25: one root cause, one finding).
|
|
132
|
+
const varying = new Map();
|
|
133
|
+
const learn = (before, after, changes, source) => {
|
|
134
|
+
for (const [key] of variedPaths(before, after, changes))
|
|
135
|
+
if (!varying.has(key))
|
|
136
|
+
varying.set(key, source);
|
|
137
|
+
};
|
|
118
138
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
119
139
|
let connectionCheck;
|
|
120
140
|
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
|
|
121
141
|
if (probe.error) {
|
|
122
|
-
raw.push({
|
|
142
|
+
raw.push({
|
|
143
|
+
rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
|
|
144
|
+
severity: 'info',
|
|
145
|
+
step: 0,
|
|
146
|
+
message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
|
|
147
|
+
fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
|
|
148
|
+
});
|
|
123
149
|
continue;
|
|
124
150
|
}
|
|
125
151
|
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
126
152
|
connectionCheck = f ? 'different' : connectionCheck ?? 'same';
|
|
127
153
|
if (f) {
|
|
128
|
-
|
|
154
|
+
const changes = compareMenus(current.tools, probe.tools ?? []);
|
|
155
|
+
raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
|
|
156
|
+
learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
|
|
129
157
|
break;
|
|
130
158
|
}
|
|
131
159
|
}
|
|
@@ -146,26 +174,75 @@ export async function session(target, scenario, options = {}) {
|
|
|
146
174
|
rule: 'session/session-lost',
|
|
147
175
|
severity: 'error',
|
|
148
176
|
step: index,
|
|
149
|
-
|
|
177
|
+
confidence: 'unsure',
|
|
178
|
+
message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
|
|
179
|
+
fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
|
|
150
180
|
});
|
|
151
181
|
return true;
|
|
152
182
|
};
|
|
153
183
|
const callWaited = { ms: 0 };
|
|
154
184
|
const listWaited = { ms: 0 };
|
|
185
|
+
const assumed = new Set(scenario.assumeReadOnly ?? []);
|
|
186
|
+
// Called on the user's word (assume_read_only), by step.
|
|
187
|
+
const onWord = [];
|
|
188
|
+
// The same failure from the same tool at several steps is one finding that lists them.
|
|
189
|
+
const failures = new Map();
|
|
190
|
+
const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
|
|
191
|
+
const key = `${rule}\0${step.tool}\0${text}`;
|
|
192
|
+
const before = failures.get(key);
|
|
193
|
+
if (before)
|
|
194
|
+
before.steps.push(index);
|
|
195
|
+
else {
|
|
196
|
+
const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
|
|
197
|
+
failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
// Steps whose change touched only values already known to vary on their own.
|
|
201
|
+
const explained = [];
|
|
202
|
+
// --auto's guessed unlocks: tried once, and those whose first call changed nothing.
|
|
203
|
+
const tried = new Set();
|
|
204
|
+
const quiet = new Set();
|
|
205
|
+
const stopped = new Map();
|
|
155
206
|
for (const [i, step] of scenario.steps.entries()) {
|
|
156
207
|
const index = i + 1;
|
|
208
|
+
if (step.kind === 'call' && step.tentative) {
|
|
209
|
+
if (quiet.has(step.tool)) {
|
|
210
|
+
stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
|
|
215
|
+
if (firstTry)
|
|
216
|
+
tried.add(step.tool);
|
|
157
217
|
const label = stepLabel(step);
|
|
158
218
|
const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
|
|
159
219
|
const mark = conn.wire.notifications.length;
|
|
220
|
+
let word = false;
|
|
221
|
+
const done = () => {
|
|
222
|
+
if (word)
|
|
223
|
+
record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
|
|
224
|
+
steps.push(record);
|
|
225
|
+
};
|
|
160
226
|
if (step.kind === 'call') {
|
|
161
227
|
const tool = current.tools.find((t) => t.name === step.tool);
|
|
228
|
+
const write = tool ? writeSign(tool) : undefined;
|
|
229
|
+
word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
|
|
162
230
|
if (!tool) {
|
|
163
231
|
record.status = 'failed';
|
|
164
232
|
record.reason = 'missing';
|
|
165
233
|
record.note = `${step.tool} isn't in the menu at this point`;
|
|
166
|
-
|
|
234
|
+
const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
|
|
235
|
+
const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
|
|
236
|
+
raw.push({
|
|
237
|
+
rule: 'session/step-failed',
|
|
238
|
+
severity: 'error',
|
|
239
|
+
step: index,
|
|
240
|
+
tool: step.tool,
|
|
241
|
+
message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
|
|
242
|
+
fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
|
|
243
|
+
});
|
|
167
244
|
}
|
|
168
|
-
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
|
|
245
|
+
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
|
|
169
246
|
record.status = 'refused';
|
|
170
247
|
record.reason = 'refused';
|
|
171
248
|
record.note = 'not marked readOnlyHint';
|
|
@@ -174,15 +251,22 @@ export async function session(target, scenario, options = {}) {
|
|
|
174
251
|
severity: 'error',
|
|
175
252
|
step: index,
|
|
176
253
|
tool: step.tool,
|
|
177
|
-
message:
|
|
254
|
+
message: assumed.has(step.tool) && write
|
|
255
|
+
? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
|
|
256
|
+
: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
|
|
257
|
+
fix: write || tool.annotations?.readOnlyHint === false
|
|
258
|
+
? 'Set "allow_writes: true" in the scenario if calling it is intended.'
|
|
259
|
+
: `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
|
|
178
260
|
});
|
|
179
261
|
}
|
|
262
|
+
if (word)
|
|
263
|
+
onWord.push({ tool: step.tool, step: index });
|
|
180
264
|
if (record.status === 'ok')
|
|
181
265
|
try {
|
|
182
266
|
// Sent again after a refusal only when that repeats nothing: the transport's
|
|
183
267
|
// 429 means the server never ran it; a tool that says it was rate-limited,
|
|
184
268
|
// thrown or as its result, may have done part of the work first, so only a
|
|
185
|
-
// read-only one is asked again.
|
|
269
|
+
// read-only one is asked again. On the server's word, not the user's.
|
|
186
270
|
const readOnly = tool?.annotations?.readOnlyHint === true;
|
|
187
271
|
callWaited.ms = 0;
|
|
188
272
|
const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
|
|
@@ -197,7 +281,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
197
281
|
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
198
282
|
record.status = 'failed';
|
|
199
283
|
record.note = 'rate-limited';
|
|
200
|
-
|
|
284
|
+
done();
|
|
201
285
|
break;
|
|
202
286
|
}
|
|
203
287
|
const short = clip(text, 120);
|
|
@@ -205,32 +289,28 @@ export async function session(target, scenario, options = {}) {
|
|
|
205
289
|
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
206
290
|
// A setup failure is reported once, for the whole run (session/untested).
|
|
207
291
|
if (!SETUP_FAILURES.has(record.failure))
|
|
208
|
-
|
|
209
|
-
rule: 'session/tool-error',
|
|
210
|
-
severity: 'warn',
|
|
211
|
-
step: index,
|
|
212
|
-
tool: step.tool,
|
|
213
|
-
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
214
|
-
});
|
|
292
|
+
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
215
293
|
}
|
|
216
294
|
}
|
|
217
295
|
catch (error) {
|
|
218
296
|
record.status = 'failed';
|
|
219
297
|
record.note = serverWords(error, 200);
|
|
220
298
|
if (lost(record.note, index)) {
|
|
221
|
-
|
|
299
|
+
done();
|
|
222
300
|
break;
|
|
223
301
|
}
|
|
224
302
|
if (tooMany(error)) {
|
|
225
303
|
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
226
304
|
record.note = 'rate-limited';
|
|
227
|
-
|
|
305
|
+
done();
|
|
228
306
|
break;
|
|
229
307
|
}
|
|
230
308
|
const code = error.code;
|
|
231
309
|
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
310
|
+
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
311
|
+
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
232
312
|
if (!SETUP_FAILURES.has(record.failure))
|
|
233
|
-
|
|
313
|
+
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
234
314
|
}
|
|
235
315
|
}
|
|
236
316
|
else if (step.kind === 'wait_for') {
|
|
@@ -252,57 +332,78 @@ export async function session(target, scenario, options = {}) {
|
|
|
252
332
|
record.status = 'failed';
|
|
253
333
|
record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
|
|
254
334
|
if (lost(why, index)) {
|
|
255
|
-
|
|
335
|
+
done();
|
|
256
336
|
break;
|
|
257
337
|
}
|
|
258
338
|
if (tooMany(error)) {
|
|
259
339
|
limited(index, why, listWaited.ms);
|
|
260
340
|
record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
|
|
261
|
-
|
|
341
|
+
done();
|
|
262
342
|
break;
|
|
263
343
|
}
|
|
264
|
-
raw.push({
|
|
265
|
-
|
|
344
|
+
raw.push({
|
|
345
|
+
rule: 'session/step-failed',
|
|
346
|
+
severity: 'error',
|
|
347
|
+
step: index,
|
|
348
|
+
message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
|
|
349
|
+
fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
|
|
350
|
+
});
|
|
351
|
+
done();
|
|
266
352
|
continue;
|
|
267
353
|
}
|
|
268
354
|
const changes = compareMenus(current.tools, next.tools);
|
|
269
355
|
if (changes.length) {
|
|
270
356
|
record.changed = true;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
: record.reason === 'refused'
|
|
279
|
-
? 'no tool call made (refused)'
|
|
280
|
-
: record.status === 'failed'
|
|
281
|
-
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
282
|
-
: undefined;
|
|
283
|
-
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
284
|
-
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
285
|
-
raw.push({
|
|
286
|
-
rule: 'session/unannounced',
|
|
287
|
-
severity: 'warn',
|
|
288
|
-
step: index,
|
|
289
|
-
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
|
|
290
|
-
});
|
|
357
|
+
const paths = variedPaths(current.tools, next.tools, changes);
|
|
358
|
+
const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
|
|
359
|
+
if (onItsOwn) {
|
|
360
|
+
// Same cause as a finding already made: said once, after the steps.
|
|
361
|
+
explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
|
|
362
|
+
const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
|
|
363
|
+
record.note = record.note ? `${record.note}; ${cause}` : cause;
|
|
291
364
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
// the run failed.
|
|
296
|
-
const probeWaited = { ms: 0 };
|
|
297
|
-
try {
|
|
298
|
-
probedAfter = index;
|
|
299
|
-
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
300
|
-
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
301
|
-
raw.push(...scopeFindings(record.scope, modern, index));
|
|
365
|
+
else {
|
|
366
|
+
if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
367
|
+
await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
|
|
302
368
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
369
|
+
const origin = step.kind === 'list'
|
|
370
|
+
? 'no tool call in between'
|
|
371
|
+
: record.reason === 'missing'
|
|
372
|
+
? 'no tool call made (the tool wasn\'t in the menu)'
|
|
373
|
+
: record.reason === 'refused'
|
|
374
|
+
? 'no tool call made (refused)'
|
|
375
|
+
: record.status === 'failed'
|
|
376
|
+
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
377
|
+
: undefined;
|
|
378
|
+
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
379
|
+
// With no call in between, whatever changed varies on its own: a later
|
|
380
|
+
// change only there has the same cause.
|
|
381
|
+
if (step.kind === 'list' || record.reason !== undefined)
|
|
382
|
+
learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
|
|
383
|
+
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
384
|
+
raw.push({
|
|
385
|
+
rule: 'session/unannounced',
|
|
386
|
+
severity: 'warn',
|
|
387
|
+
step: index,
|
|
388
|
+
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
|
|
389
|
+
fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
if (step.kind !== 'list' && probes) {
|
|
393
|
+
// The main process is still running here: a server that holds a file or
|
|
394
|
+
// a port can't start a second copy. That leaves the scope unknown, not
|
|
395
|
+
// the run failed.
|
|
396
|
+
const probeWaited = { ms: 0 };
|
|
397
|
+
try {
|
|
398
|
+
probedAfter = index;
|
|
399
|
+
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
400
|
+
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
401
|
+
raw.push(...scopeFindings(record.scope, modern, index));
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
record.scope = 'unclear';
|
|
405
|
+
scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
|
|
406
|
+
}
|
|
306
407
|
}
|
|
307
408
|
}
|
|
308
409
|
current = next;
|
|
@@ -311,13 +412,23 @@ export async function session(target, scenario, options = {}) {
|
|
|
311
412
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
312
413
|
record.tools = current.tools.length;
|
|
313
414
|
record.tokens = current.totalTokens;
|
|
314
|
-
|
|
415
|
+
if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
|
|
416
|
+
quiet.add(step.tool);
|
|
417
|
+
done();
|
|
315
418
|
}
|
|
419
|
+
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
420
|
+
if (explained.length)
|
|
421
|
+
raw.push(knownVariance(explained));
|
|
422
|
+
if (onWord.length)
|
|
423
|
+
raw.push(onTheirWord(onWord, options.auto !== undefined));
|
|
316
424
|
raw.push(...untested(steps, target));
|
|
317
425
|
// --auto means "find what you can": an unlock it skipped is a gap too.
|
|
318
|
-
|
|
426
|
+
const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
|
|
427
|
+
// An unlock --auto stopped trying isn't charged for the values it then left out.
|
|
428
|
+
const covered = baseline.tools.filter((t) => !stopped.has(t.name));
|
|
429
|
+
raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
|
|
319
430
|
if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
|
|
320
|
-
raw.push(nothingCalled(options.auto
|
|
431
|
+
raw.push(nothingCalled(options.auto));
|
|
321
432
|
if (scopeUnchecked.length) {
|
|
322
433
|
const which = scopeUnchecked.map((s) => s.step);
|
|
323
434
|
raw.push({
|
|
@@ -336,7 +447,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
336
447
|
final: { tools: current.tools.length, tokens: current.totalTokens },
|
|
337
448
|
connectionCheck,
|
|
338
449
|
steps,
|
|
339
|
-
findings: settle(
|
|
450
|
+
findings: settle(mergeRepeats(raw), options),
|
|
451
|
+
...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
|
|
340
452
|
union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
|
|
341
453
|
};
|
|
342
454
|
}
|
|
@@ -359,13 +471,17 @@ function untested(steps, target) {
|
|
|
359
471
|
for (const s of setup)
|
|
360
472
|
byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
|
|
361
473
|
const all = setup.length === calls.length;
|
|
362
|
-
|
|
474
|
+
// One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
|
|
475
|
+
const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
|
|
363
476
|
return [
|
|
364
477
|
{
|
|
365
478
|
rule: 'session/untested',
|
|
366
479
|
severity: all ? 'error' : 'warn',
|
|
367
|
-
|
|
480
|
+
// Classified from the error's words: a guess, if a good one.
|
|
481
|
+
confidence: 'unsure',
|
|
482
|
+
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
|
|
368
483
|
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
|
|
484
|
+
fix,
|
|
369
485
|
},
|
|
370
486
|
];
|
|
371
487
|
}
|
|
@@ -410,45 +526,245 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
410
526
|
? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
|
|
411
527
|
: skip === 'not read-only'
|
|
412
528
|
? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
|
|
413
|
-
: ''
|
|
529
|
+
: skip === 'unmarked'
|
|
530
|
+
? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
|
|
531
|
+
: '';
|
|
414
532
|
out.push({
|
|
415
533
|
rule: 'session/unlock-coverage',
|
|
416
534
|
severity: 'warn',
|
|
417
535
|
tool: u.tool.name,
|
|
536
|
+
// "Looks like it unlocks": from its name, parameter and description.
|
|
537
|
+
confidence: 'unsure',
|
|
418
538
|
...(last !== undefined ? { step: last } : {}),
|
|
419
|
-
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}
|
|
539
|
+
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
|
|
420
540
|
detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
|
|
541
|
+
fix: skip === 'open world'
|
|
542
|
+
? 'Rerun with --open-world.'
|
|
543
|
+
: skip === 'unmarked'
|
|
544
|
+
? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
|
|
545
|
+
: skip === 'not read-only'
|
|
546
|
+
? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
|
|
547
|
+
: 'Unlock every value: session --init puts each one in the starter.',
|
|
421
548
|
});
|
|
422
549
|
}
|
|
423
550
|
return out;
|
|
424
551
|
}
|
|
425
552
|
/**
|
|
426
553
|
* --auto that called nothing: the run only listed the menu, so a clean result
|
|
427
|
-
* says nothing about what the tools do to it. Why each tool was left out
|
|
554
|
+
* says nothing about what the tools do to it. Why each tool was left out, and the
|
|
555
|
+
* flag that reaches the most of them.
|
|
428
556
|
*/
|
|
429
|
-
function nothingCalled(
|
|
430
|
-
const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
557
|
+
function nothingCalled(auto) {
|
|
558
|
+
const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
431
559
|
const parts = [];
|
|
432
560
|
const openWorld = by('open world');
|
|
433
561
|
const needs = by('needs values');
|
|
562
|
+
const unmarked = by('unmarked');
|
|
434
563
|
const writes = by('not read-only');
|
|
435
|
-
if (openWorld.length)
|
|
436
|
-
parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
|
|
437
564
|
if (needs.length)
|
|
438
|
-
parts.push(`${needs.length} need values the schema doesn't give
|
|
565
|
+
parts.push(`${needs.length} need values the schema doesn't give`);
|
|
566
|
+
if (unmarked.length)
|
|
567
|
+
parts.push(`${unmarked.length} aren't marked readOnlyHint`);
|
|
568
|
+
if (openWorld.length)
|
|
569
|
+
parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
|
|
439
570
|
if (writes.length)
|
|
440
|
-
parts.push(`${writes.length}
|
|
571
|
+
parts.push(`${writes.length} marked or named as writes`);
|
|
441
572
|
const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
|
|
573
|
+
const params = neededValues(auto.skipped);
|
|
574
|
+
const fix = autoNextStep(auto);
|
|
442
575
|
return {
|
|
443
576
|
rule: 'session/nothing-called',
|
|
444
577
|
severity: 'warn',
|
|
445
578
|
message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
|
|
446
579
|
detail: [
|
|
580
|
+
...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
|
|
581
|
+
...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
|
|
447
582
|
...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
|
|
448
|
-
...(needs.length ? [`need values: ${list(needs)}`] : []),
|
|
449
583
|
],
|
|
584
|
+
...(fix ? { fix } : {}),
|
|
450
585
|
};
|
|
451
586
|
}
|
|
587
|
+
/**
|
|
588
|
+
* The same error text from different tools is one finding (two tools of a feature
|
|
589
|
+
* that "isn't set up on this deployment"). An empty text says nothing in common.
|
|
590
|
+
*/
|
|
591
|
+
function sameError(failures) {
|
|
592
|
+
const out = [];
|
|
593
|
+
const byText = new Map();
|
|
594
|
+
for (const f of failures) {
|
|
595
|
+
const key = `${f.rule}\0${f.text}`;
|
|
596
|
+
const first = f.text ? byText.get(key) : undefined;
|
|
597
|
+
if (!first) {
|
|
598
|
+
out.push({ ...f, steps: [...f.steps] });
|
|
599
|
+
if (f.text)
|
|
600
|
+
byText.set(key, out[out.length - 1]);
|
|
601
|
+
continue;
|
|
602
|
+
}
|
|
603
|
+
first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
|
|
604
|
+
first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
|
|
605
|
+
}
|
|
606
|
+
return out;
|
|
607
|
+
}
|
|
608
|
+
// The server says the feature isn't there, or isn't for this account: no argument
|
|
609
|
+
// fixes that. A guess from its words.
|
|
610
|
+
const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\bpermission\b|\bforbidden\b|\bnot allowed\b|\brole\b/i;
|
|
611
|
+
/** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
|
|
612
|
+
function stepsText(steps) {
|
|
613
|
+
if (steps.length === 1)
|
|
614
|
+
return `step ${steps[0]}`;
|
|
615
|
+
const runs = [];
|
|
616
|
+
for (let i = 0; i < steps.length;) {
|
|
617
|
+
let j = i;
|
|
618
|
+
while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
|
|
619
|
+
j++;
|
|
620
|
+
if (j - i >= 2)
|
|
621
|
+
runs.push(`${steps[i]}–${steps[j]}`);
|
|
622
|
+
else
|
|
623
|
+
for (let k = i; k <= j; k++)
|
|
624
|
+
runs.push(String(steps[k]));
|
|
625
|
+
i = j + 1;
|
|
626
|
+
}
|
|
627
|
+
return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* The same error from the same tool is one finding that names every step it
|
|
631
|
+
* happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
|
|
632
|
+
* words, the call that got it, and what to try.
|
|
633
|
+
*/
|
|
634
|
+
function failureFindings(failures, auto) {
|
|
635
|
+
return failures.map((f) => {
|
|
636
|
+
const many = f.steps.length > 1;
|
|
637
|
+
const calls = [f, ...(f.others ?? [])];
|
|
638
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
639
|
+
const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
|
|
640
|
+
const said = f.text ? quotedSentence(clip(f.text, 160)) : 'no error text.';
|
|
641
|
+
const firstArg = Object.keys(f.args)[0];
|
|
642
|
+
const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
|
|
643
|
+
// No argument can fix a call to a tool that takes none, or a feature the server
|
|
644
|
+
// says isn't there. From the server's words: a guess.
|
|
645
|
+
const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
|
|
646
|
+
const them = tools.length === 1 ? 'its' : 'their';
|
|
647
|
+
const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
|
|
648
|
+
const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
|
|
649
|
+
const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
|
|
650
|
+
const fix = f.timedOutMs
|
|
651
|
+
? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
|
|
652
|
+
: !argsCan
|
|
653
|
+
? undefined
|
|
654
|
+
: unavailable
|
|
655
|
+
? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
656
|
+
: noParams
|
|
657
|
+
? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
658
|
+
: auto?.withValues?.includes(f.tool)
|
|
659
|
+
? `Check the values you gave ${f.tool} (--value) against what the server says above.`
|
|
660
|
+
: auto
|
|
661
|
+
? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
|
|
662
|
+
: `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
|
|
663
|
+
const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
|
|
664
|
+
const detail = calls.slice(0, 8).map((c) => {
|
|
665
|
+
const call = `${c.tool} ${JSON.stringify(c.args)}`;
|
|
666
|
+
return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
|
|
667
|
+
});
|
|
668
|
+
return {
|
|
669
|
+
rule: f.rule,
|
|
670
|
+
severity: f.severity,
|
|
671
|
+
step: f.steps[0],
|
|
672
|
+
...(many ? { steps: f.steps } : {}),
|
|
673
|
+
tool: f.tool,
|
|
674
|
+
...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
|
|
675
|
+
message: f.rule === 'session/tool-error'
|
|
676
|
+
? tools.length > 1
|
|
677
|
+
? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
678
|
+
: `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
679
|
+
: tools.length > 1
|
|
680
|
+
? `${lead}Calling ${who} failed the same way: ${said}`
|
|
681
|
+
: `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
|
|
682
|
+
detail: [...new Set(detail)],
|
|
683
|
+
...(fix ? { fix } : {}),
|
|
684
|
+
};
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Changes that touched only values already seen to vary on their own: one info
|
|
689
|
+
* finding for the run, pointing to the one that reported the cause.
|
|
690
|
+
*/
|
|
691
|
+
function knownVariance(explained) {
|
|
692
|
+
const steps = explained.map((e) => e.step);
|
|
693
|
+
const src = explained[0].source;
|
|
694
|
+
const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
|
|
695
|
+
// The values at the first such step; the rest only repeat the pattern.
|
|
696
|
+
const seen = explained[0].seen;
|
|
697
|
+
return {
|
|
698
|
+
rule: 'session/known-variance',
|
|
699
|
+
severity: 'info',
|
|
700
|
+
step: steps[0],
|
|
701
|
+
message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
|
|
702
|
+
detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
|
|
703
|
+
fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
/** Tools called because the user said they only read: said once, loudly. */
|
|
707
|
+
function onTheirWord(calls, auto) {
|
|
708
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
709
|
+
return {
|
|
710
|
+
rule: 'session/assumed-read-only',
|
|
711
|
+
severity: 'info',
|
|
712
|
+
step: calls[0].step,
|
|
713
|
+
message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
|
|
714
|
+
detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
|
|
715
|
+
fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Every leaf value that differs between the old and new definition of each edited
|
|
720
|
+
* tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
|
|
721
|
+
*/
|
|
722
|
+
function variedPaths(before, after, changes) {
|
|
723
|
+
const out = new Map();
|
|
724
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
725
|
+
for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
|
|
726
|
+
const a = old.get(name);
|
|
727
|
+
const b = after.find((t) => t.name === name);
|
|
728
|
+
if (!a || !b)
|
|
729
|
+
continue;
|
|
730
|
+
for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
|
|
731
|
+
out.set(`${name}\0${path}`, d);
|
|
732
|
+
}
|
|
733
|
+
return out;
|
|
734
|
+
}
|
|
735
|
+
function leafDiffs(a, b, path = '', out = new Map()) {
|
|
736
|
+
if (canonical(a) === canonical(b))
|
|
737
|
+
return out;
|
|
738
|
+
const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
739
|
+
if (obj(a) && obj(b)) {
|
|
740
|
+
for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
|
|
741
|
+
leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
|
|
742
|
+
}
|
|
743
|
+
else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
|
|
744
|
+
a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
|
|
745
|
+
}
|
|
746
|
+
else
|
|
747
|
+
out.set(path, { before: a, after: b });
|
|
748
|
+
return out;
|
|
749
|
+
}
|
|
750
|
+
function show(value) {
|
|
751
|
+
if (value === undefined)
|
|
752
|
+
return '(absent)';
|
|
753
|
+
const text = JSON.stringify(value);
|
|
754
|
+
return text.length > 60 ? text.slice(0, 57) + '…' : text;
|
|
755
|
+
}
|
|
756
|
+
const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
|
|
757
|
+
/** The next step for a menu that differs between processes, from what differs. */
|
|
758
|
+
function varianceFix(main, other, changes) {
|
|
759
|
+
const diffs = [...variedPaths(main, other, changes)];
|
|
760
|
+
if (diffs.length === 0)
|
|
761
|
+
return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
|
|
762
|
+
const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
|
|
763
|
+
if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
|
|
764
|
+
return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
|
|
765
|
+
}
|
|
766
|
+
return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
|
|
767
|
+
}
|
|
452
768
|
function serverOf(c) {
|
|
453
769
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
454
770
|
}
|
|
@@ -477,6 +793,7 @@ function waitFor(done, ms) {
|
|
|
477
793
|
}, 20);
|
|
478
794
|
});
|
|
479
795
|
}
|
|
796
|
+
const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
|
|
480
797
|
const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
481
798
|
/** Turn a menu change into findings, with the cost of each. */
|
|
482
799
|
export function changeFindings(before, after, changes, step, origin) {
|
|
@@ -504,8 +821,10 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
504
821
|
rule: 'session/append',
|
|
505
822
|
severity: 'warn',
|
|
506
823
|
step,
|
|
507
|
-
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')})
|
|
824
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
|
|
508
825
|
detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
|
|
826
|
+
each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
|
|
827
|
+
fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
|
|
509
828
|
});
|
|
510
829
|
}
|
|
511
830
|
else {
|
|
@@ -516,22 +835,50 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
516
835
|
step,
|
|
517
836
|
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
518
837
|
detail: [...cost, ...why],
|
|
838
|
+
fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
|
|
519
839
|
});
|
|
520
840
|
}
|
|
521
841
|
}
|
|
522
842
|
const moved = changes.filter((c) => c.kind === 'moved');
|
|
523
843
|
if (moved.length) {
|
|
524
|
-
out.push({
|
|
844
|
+
out.push({
|
|
845
|
+
rule: 'session/reorder',
|
|
846
|
+
severity: 'error',
|
|
847
|
+
step,
|
|
848
|
+
message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
|
|
849
|
+
detail: [...cost, ...why],
|
|
850
|
+
fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
|
|
851
|
+
});
|
|
525
852
|
}
|
|
526
853
|
const removed = changes.filter((c) => c.kind === 'removed');
|
|
527
854
|
if (removed.length) {
|
|
528
|
-
out.push({
|
|
855
|
+
out.push({
|
|
856
|
+
rule: 'session/remove',
|
|
857
|
+
severity: 'error',
|
|
858
|
+
step,
|
|
859
|
+
message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
|
|
860
|
+
detail: [...cost, ...why],
|
|
861
|
+
fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
|
|
862
|
+
});
|
|
529
863
|
}
|
|
530
864
|
const edits = new Map();
|
|
531
865
|
for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
|
|
532
866
|
edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
|
|
867
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
533
868
|
for (const [tool, fields] of edits) {
|
|
534
|
-
|
|
869
|
+
const a = old.get(tool);
|
|
870
|
+
const b = after.find((t) => t.name === tool);
|
|
871
|
+
// What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
|
|
872
|
+
const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
|
|
873
|
+
out.push({
|
|
874
|
+
rule: 'session/edit',
|
|
875
|
+
severity: 'error',
|
|
876
|
+
step,
|
|
877
|
+
tool,
|
|
878
|
+
message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
879
|
+
detail: [...seen, ...cost, ...why],
|
|
880
|
+
fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
|
|
881
|
+
});
|
|
535
882
|
}
|
|
536
883
|
return out;
|
|
537
884
|
}
|
|
@@ -576,6 +923,7 @@ function scopeFindings(scope, modern, step) {
|
|
|
576
923
|
message: modern
|
|
577
924
|
? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
|
|
578
925
|
: `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
|
|
926
|
+
fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
579
927
|
},
|
|
580
928
|
];
|
|
581
929
|
}
|
|
@@ -586,20 +934,41 @@ function scopeFindings(scope, modern, step) {
|
|
|
586
934
|
severity: 'warn',
|
|
587
935
|
step,
|
|
588
936
|
message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
|
|
937
|
+
fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
589
938
|
},
|
|
590
939
|
];
|
|
591
940
|
}
|
|
592
941
|
return [];
|
|
593
942
|
}
|
|
594
|
-
/**
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
943
|
+
/**
|
|
944
|
+
* Findings that say the same thing at every step are said once, naming the steps:
|
|
945
|
+
* stdio's side-effect warning, and an unlock per domain, where each value appends
|
|
946
|
+
* tools for this connection only (9 steps made 18 near-identical blocks). What
|
|
947
|
+
* differs per step, the tools and their cost, goes in the detail. Mid-inserts,
|
|
948
|
+
* removals and edits stay per step: each is its own change.
|
|
949
|
+
*/
|
|
950
|
+
const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
|
|
951
|
+
function mergeRepeats(raw) {
|
|
952
|
+
let out = raw;
|
|
953
|
+
for (const rule of REPEATS) {
|
|
954
|
+
const same = out.filter((f) => f.rule === rule);
|
|
955
|
+
if (same.length < 2)
|
|
956
|
+
continue;
|
|
957
|
+
const steps = same.map((f) => f.step);
|
|
958
|
+
const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
|
|
959
|
+
let message = `${lead}: ${same[0].message}`;
|
|
960
|
+
let detail = same[0].detail;
|
|
961
|
+
if (rule === 'session/append') {
|
|
962
|
+
const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
|
|
963
|
+
message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
|
|
964
|
+
const each = same.map((f) => f.each).filter(Boolean);
|
|
965
|
+
const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
|
|
966
|
+
detail = [...each, ...why];
|
|
967
|
+
}
|
|
968
|
+
const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
|
|
969
|
+
out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
970
|
+
}
|
|
971
|
+
return out;
|
|
603
972
|
}
|
|
604
973
|
function describe(changes) {
|
|
605
974
|
return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
|
|
@@ -608,7 +977,7 @@ function settle(raw, options) {
|
|
|
608
977
|
const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
609
978
|
return raw
|
|
610
979
|
.filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
|
|
611
|
-
.map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
980
|
+
.map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
612
981
|
.sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
613
982
|
}
|
|
614
983
|
/*
|
|
@@ -630,7 +999,25 @@ const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand
|
|
|
630
999
|
// capabilities alone aren't tools.
|
|
631
1000
|
const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
|
|
632
1001
|
// Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
|
|
633
|
-
|
|
1002
|
+
// "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
|
|
1003
|
+
const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
|
|
1004
|
+
/**
|
|
1005
|
+
* Does the tool say that `param` unlocks tools? Its name does, or a sentence of
|
|
1006
|
+
* its description that says it unlocks tools and doesn't pin that on another
|
|
1007
|
+
* parameter ("A query match also enables the tools…" is about `query`, not the
|
|
1008
|
+
* `domain` filter next to it).
|
|
1009
|
+
*/
|
|
1010
|
+
function claimsUnlock(tool, param) {
|
|
1011
|
+
if (UNLOCK_NAME.test(tool.name))
|
|
1012
|
+
return true;
|
|
1013
|
+
const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
|
|
1014
|
+
const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
|
|
1015
|
+
const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
|
|
1016
|
+
return (tool.description ?? '')
|
|
1017
|
+
.split(/(?<=[.!?])\s+/)
|
|
1018
|
+
.filter((s) => UNLOCK_SAYS.test(s))
|
|
1019
|
+
.some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
|
|
1020
|
+
}
|
|
634
1021
|
const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
635
1022
|
/**
|
|
636
1023
|
* At most `max` characters, cut by code point at a word boundary, never inside a
|
|
@@ -672,7 +1059,8 @@ export function unlockers(tools) {
|
|
|
672
1059
|
// A lookup names toolsets without changing them (GitHub's get_toolset_tools,
|
|
673
1060
|
// toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
|
|
674
1061
|
const verb = verbOf(tool.name);
|
|
675
|
-
|
|
1062
|
+
const lookup = verb !== undefined && LOOKUPS.has(verb);
|
|
1063
|
+
if (lookup && !UNLOCK_SAYS.test(description))
|
|
676
1064
|
continue;
|
|
677
1065
|
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
|
|
678
1066
|
const required = tool.inputSchema?.required ?? [];
|
|
@@ -681,6 +1069,10 @@ export function unlockers(tools) {
|
|
|
681
1069
|
// Named like an unlock, one required parameter: that's what it takes
|
|
682
1070
|
// (toolception's enable_toolset { name }).
|
|
683
1071
|
(UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
|
|
1072
|
+
const claims = claimsUnlock(tool, param);
|
|
1073
|
+
// A lookup that says a *different* parameter unlocks: its enum is a filter.
|
|
1074
|
+
if (lookup && !claims)
|
|
1075
|
+
continue;
|
|
684
1076
|
let score = 0;
|
|
685
1077
|
if (param)
|
|
686
1078
|
score += enumOf(props[param]).length ? 3 : 2;
|
|
@@ -689,7 +1081,7 @@ export function unlockers(tools) {
|
|
|
689
1081
|
if (UNLOCK_DESC.test(tool.description ?? ''))
|
|
690
1082
|
score += 1;
|
|
691
1083
|
if (score >= 2)
|
|
692
|
-
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
|
|
1084
|
+
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
|
|
693
1085
|
}
|
|
694
1086
|
return found.sort((a, b) => b.score - a.score);
|
|
695
1087
|
}
|