toolmenu 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +84 -23
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +59 -21
  6. package/dist/auto.d.ts +80 -6
  7. package/dist/auto.js +255 -19
  8. package/dist/breakdown.d.ts +2 -0
  9. package/dist/breakdown.js +64 -19
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +170 -185
  13. package/dist/close.d.ts +8 -0
  14. package/dist/close.js +18 -0
  15. package/dist/config.d.ts +7 -1
  16. package/dist/config.js +17 -0
  17. package/dist/connect.d.ts +14 -0
  18. package/dist/connect.js +150 -16
  19. package/dist/diff.d.ts +23 -1
  20. package/dist/diff.js +574 -120
  21. package/dist/difference.d.ts +2 -0
  22. package/dist/difference.js +13 -12
  23. package/dist/explain.d.ts +113 -0
  24. package/dist/explain.js +461 -0
  25. package/dist/failures.d.ts +15 -0
  26. package/dist/failures.js +59 -2
  27. package/dist/help.d.ts +11 -0
  28. package/dist/help.js +205 -0
  29. package/dist/history.d.ts +13 -0
  30. package/dist/history.js +86 -6
  31. package/dist/init.d.ts +93 -2
  32. package/dist/init.js +508 -21
  33. package/dist/menu.js +10 -2
  34. package/dist/options.d.ts +141 -0
  35. package/dist/options.js +50 -0
  36. package/dist/report.d.ts +2 -2
  37. package/dist/report.js +305 -101
  38. package/dist/rules/description.d.ts +7 -2
  39. package/dist/rules/description.js +86 -42
  40. package/dist/rules/determinism.d.ts +4 -1
  41. package/dist/rules/determinism.js +175 -11
  42. package/dist/rules/gated.d.ts +7 -0
  43. package/dist/rules/gated.js +77 -0
  44. package/dist/rules/ids.d.ts +1 -1
  45. package/dist/rules/ids.js +9 -4
  46. package/dist/rules/index.d.ts +6 -0
  47. package/dist/rules/index.js +18 -6
  48. package/dist/rules/naming.d.ts +1 -1
  49. package/dist/rules/naming.js +91 -21
  50. package/dist/rules/rule.d.ts +25 -0
  51. package/dist/rules/rule.js +25 -1
  52. package/dist/rules/spec.d.ts +6 -0
  53. package/dist/rules/spec.js +56 -9
  54. package/dist/rules/write.d.ts +1 -1
  55. package/dist/rules/write.js +64 -14
  56. package/dist/session.d.ts +34 -9
  57. package/dist/session.js +525 -94
  58. package/dist/snapshot.d.ts +8 -2
  59. package/dist/snapshot.js +33 -2
  60. package/dist/types.d.ts +21 -0
  61. package/dist/words.d.ts +5 -0
  62. package/dist/words.js +14 -0
  63. package/package.json +2 -2
package/dist/session.js CHANGED
@@ -5,9 +5,11 @@ import { listTools } from './connect.js';
5
5
  import { buildMenu, toolDefinition } from './menu.js';
6
6
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
7
  import { varianceFinding } from './rules/determinism.js';
8
- import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_ADVICE, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, tooMany, waitedFor, whyNot } from './failures.js';
8
+ import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
9
  import { SEVERITY_RANK } from './types.js';
10
10
  import { leadingJson } from './catalog.js';
11
+ import { allDifferences, describeToolDifference } from './difference.js';
12
+ import { autoNextStep, neededValues, writeSign } from './auto.js';
11
13
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
12
14
  const LIST_CHANGED = 'notifications/tools/list_changed';
13
15
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
@@ -31,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
31
33
  if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
32
34
  throw new Error(`${where}: allow_writes must be true or false`);
33
35
  }
36
+ const assumed = doc.assume_read_only;
37
+ if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
38
+ throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
39
+ }
34
40
  const steps = doc.steps.map((raw, i) => {
35
41
  const at = `${where}: step ${i + 1}`;
36
42
  if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
@@ -55,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
55
61
  }
56
62
  throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
57
63
  });
58
- return { allowWrites: doc.allow_writes === true, steps };
64
+ return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
59
65
  }
60
66
  export function stepLabel(step) {
61
67
  if (step.kind === 'list')
@@ -104,9 +110,13 @@ export async function session(target, scenario, options = {}) {
104
110
  rule: 'session/rate-limited',
105
111
  severity: 'error',
106
112
  step: index,
113
+ ...(tool ? { tool } : {}),
107
114
  message: tool
108
- ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked} If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota.`
109
- : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} ${RATE_LIMIT_ADVICE}`,
115
+ ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
116
+ : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
117
+ fix: tool
118
+ ? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
119
+ : 'Give the run its own server instance, or raise the limit for it.',
110
120
  });
111
121
  };
112
122
  let current = await listPatiently();
@@ -115,17 +125,35 @@ export async function session(target, scenario, options = {}) {
115
125
  const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
116
126
  see(current);
117
127
  const baseline = current;
128
+ // Values that differ from one tools/list to the next on their own (a default
129
+ // computed from the clock: PayPal's list_transactions.end_date), by tool and
130
+ // path, with the finding that reported them first. A later change only there
131
+ // has the same cause and points to it (SPEC §25: one root cause, one finding).
132
+ const varying = new Map();
133
+ const learn = (before, after, changes, source) => {
134
+ for (const [key] of variedPaths(before, after, changes))
135
+ if (!varying.has(key))
136
+ varying.set(key, source);
137
+ };
118
138
  // Fresh processes or connections, same credentials, must see the same menu.
119
139
  let connectionCheck;
120
140
  for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
121
141
  if (probe.error) {
122
- raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}` });
142
+ raw.push({
143
+ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
144
+ severity: 'info',
145
+ step: 0,
146
+ message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
147
+ fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
148
+ });
123
149
  continue;
124
150
  }
125
151
  const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
126
152
  connectionCheck = f ? 'different' : connectionCheck ?? 'same';
127
153
  if (f) {
128
- raw.push({ ...f, step: 0 });
154
+ const changes = compareMenus(current.tools, probe.tools ?? []);
155
+ raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
156
+ learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
129
157
  break;
130
158
  }
131
159
  }
@@ -146,43 +174,102 @@ export async function session(target, scenario, options = {}) {
146
174
  rule: 'session/session-lost',
147
175
  severity: 'error',
148
176
  step: index,
149
- message: `The server ended this session (“${said}”) after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that, so the rest of the scenario didn't run. Run with --processes 1: toolmenu then opens no second one.`,
177
+ confidence: 'unsure',
178
+ message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
179
+ fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
150
180
  });
151
181
  return true;
152
182
  };
153
183
  const callWaited = { ms: 0 };
154
184
  const listWaited = { ms: 0 };
185
+ // Read live: --auto vouches for tools that appear mid-session too.
186
+ const assumed = { has: (tool) => (scenario.assumeReadOnly ?? []).includes(tool) };
187
+ // Called on the user's word (assume_read_only), by step.
188
+ const onWord = [];
189
+ // The same failure from the same tool at several steps is one finding that lists them.
190
+ const failures = new Map();
191
+ const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
192
+ const key = `${rule}\0${step.tool}\0${text}`;
193
+ const before = failures.get(key);
194
+ if (before)
195
+ before.steps.push(index);
196
+ else {
197
+ const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
198
+ failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
199
+ }
200
+ };
201
+ // Steps whose change touched only values already known to vary on their own.
202
+ const explained = [];
203
+ // --auto's guessed unlocks: tried once, and those whose first call changed nothing.
204
+ const tried = new Set();
205
+ const quiet = new Set();
206
+ const stopped = new Map();
155
207
  for (const [i, step] of scenario.steps.entries()) {
156
208
  const index = i + 1;
209
+ if (step.kind === 'call' && step.tentative) {
210
+ if (quiet.has(step.tool)) {
211
+ stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
212
+ continue;
213
+ }
214
+ }
215
+ const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
216
+ if (firstTry)
217
+ tried.add(step.tool);
157
218
  const label = stepLabel(step);
158
219
  const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
159
220
  const mark = conn.wire.notifications.length;
221
+ let word = false;
222
+ const done = () => {
223
+ if (word)
224
+ record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
225
+ steps.push(record);
226
+ };
160
227
  if (step.kind === 'call') {
161
228
  const tool = current.tools.find((t) => t.name === step.tool);
229
+ const write = tool ? writeSign(tool) : undefined;
230
+ word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
162
231
  if (!tool) {
163
232
  record.status = 'failed';
164
233
  record.reason = 'missing';
234
+ record.outcome = 'not-sent';
165
235
  record.note = `${step.tool} isn't in the menu at this point`;
166
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `Step ${index} calls ${step.tool}, which isn't in the menu at this point.` });
236
+ const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
237
+ const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
238
+ raw.push({
239
+ rule: 'session/step-failed',
240
+ severity: 'error',
241
+ step: index,
242
+ tool: step.tool,
243
+ message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
244
+ fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
245
+ });
167
246
  }
168
- else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
247
+ else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
169
248
  record.status = 'refused';
170
249
  record.reason = 'refused';
250
+ record.outcome = 'not-sent';
171
251
  record.note = 'not marked readOnlyHint';
172
252
  raw.push({
173
253
  rule: 'session/refused',
174
254
  severity: 'error',
175
255
  step: index,
176
256
  tool: step.tool,
177
- message: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real. Set "allow_writes: true" in the scenario if that's intended.`,
257
+ message: assumed.has(step.tool) && write
258
+ ? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
259
+ : `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
260
+ fix: write || tool.annotations?.readOnlyHint === false
261
+ ? 'Set "allow_writes: true" in the scenario if calling it is intended.'
262
+ : `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
178
263
  });
179
264
  }
265
+ if (word)
266
+ onWord.push({ tool: step.tool, step: index });
180
267
  if (record.status === 'ok')
181
268
  try {
182
269
  // Sent again after a refusal only when that repeats nothing: the transport's
183
270
  // 429 means the server never ran it; a tool that says it was rate-limited,
184
271
  // thrown or as its result, may have done part of the work first, so only a
185
- // read-only one is asked again.
272
+ // read-only one is asked again. On the server's word, not the user's.
186
273
  const readOnly = tool?.annotations?.readOnlyHint === true;
187
274
  callWaited.ms = 0;
188
275
  const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
@@ -191,13 +278,14 @@ export async function session(target, scenario, options = {}) {
191
278
  error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
192
279
  result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
193
280
  });
281
+ record.outcome = result.isError ? 'tool-error' : 'answered';
194
282
  if (result.isError) {
195
283
  const text = errorText(result);
196
284
  if (RATE_LIMITED.test(text)) {
197
285
  limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
198
286
  record.status = 'failed';
199
287
  record.note = 'rate-limited';
200
- steps.push(record);
288
+ done();
201
289
  break;
202
290
  }
203
291
  const short = clip(text, 120);
@@ -205,32 +293,29 @@ export async function session(target, scenario, options = {}) {
205
293
  record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
206
294
  // A setup failure is reported once, for the whole run (session/untested).
207
295
  if (!SETUP_FAILURES.has(record.failure))
208
- raw.push({
209
- rule: 'session/tool-error',
210
- severity: 'warn',
211
- step: index,
212
- tool: step.tool,
213
- message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
214
- });
296
+ fail('session/tool-error', 'warn', index, step, text, record.failure);
215
297
  }
216
298
  }
217
299
  catch (error) {
218
300
  record.status = 'failed';
301
+ record.outcome = 'failed';
219
302
  record.note = serverWords(error, 200);
220
303
  if (lost(record.note, index)) {
221
- steps.push(record);
304
+ done();
222
305
  break;
223
306
  }
224
307
  if (tooMany(error)) {
225
308
  limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
226
309
  record.note = 'rate-limited';
227
- steps.push(record);
310
+ done();
228
311
  break;
229
312
  }
230
313
  const code = error.code;
231
314
  record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
315
+ // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
316
+ const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
232
317
  if (!SETUP_FAILURES.has(record.failure))
233
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
318
+ fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
234
319
  }
235
320
  }
236
321
  else if (step.kind === 'wait_for') {
@@ -252,72 +337,107 @@ export async function session(target, scenario, options = {}) {
252
337
  record.status = 'failed';
253
338
  record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
254
339
  if (lost(why, index)) {
255
- steps.push(record);
340
+ done();
256
341
  break;
257
342
  }
258
343
  if (tooMany(error)) {
259
344
  limited(index, why, listWaited.ms);
260
345
  record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
261
- steps.push(record);
346
+ done();
262
347
  break;
263
348
  }
264
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
265
- steps.push(record);
349
+ raw.push({
350
+ rule: 'session/step-failed',
351
+ severity: 'error',
352
+ step: index,
353
+ message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
354
+ fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
355
+ });
356
+ done();
266
357
  continue;
267
358
  }
268
359
  const changes = compareMenus(current.tools, next.tools);
269
360
  if (changes.length) {
270
361
  record.changed = true;
271
- if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
272
- await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
273
- }
274
- const origin = step.kind === 'list'
275
- ? 'no tool call in between'
276
- : record.reason === 'missing'
277
- ? 'no tool call made (the tool wasn\'t in the menu)'
278
- : record.reason === 'refused'
279
- ? 'no tool call made (refused)'
280
- : record.status === 'failed'
281
- ? 'the call failed, but the menu changed: the server may have applied it anyway'
282
- : undefined;
283
- raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
284
- if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
285
- raw.push({
286
- rule: 'session/unannounced',
287
- severity: 'warn',
288
- step: index,
289
- message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
290
- });
362
+ const paths = variedPaths(current.tools, next.tools, changes);
363
+ const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
364
+ if (onItsOwn) {
365
+ // Same cause as a finding already made: said once, after the steps.
366
+ explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
367
+ const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
368
+ record.note = record.note ? `${record.note}; ${cause}` : cause;
291
369
  }
292
- if (step.kind !== 'list' && probes) {
293
- // The main process is still running here: a server that holds a file or
294
- // a port can't start a second copy. That leaves the scope unknown, not
295
- // the run failed.
296
- const probeWaited = { ms: 0 };
297
- try {
298
- probedAfter = index;
299
- const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
300
- record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
301
- raw.push(...scopeFindings(record.scope, modern, index));
370
+ else {
371
+ if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
372
+ await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
302
373
  }
303
- catch (error) {
304
- record.scope = 'unclear';
305
- scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
374
+ const origin = step.kind === 'list'
375
+ ? 'no tool call in between'
376
+ : record.reason === 'missing'
377
+ ? 'no tool call made (the tool wasn\'t in the menu)'
378
+ : record.reason === 'refused'
379
+ ? 'no tool call made (refused)'
380
+ : record.status === 'failed'
381
+ ? 'the call failed, but the menu changed: the server may have applied it anyway'
382
+ : undefined;
383
+ raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
384
+ // With no call in between, whatever changed varies on its own: a later
385
+ // change only there has the same cause.
386
+ if (step.kind === 'list' || record.reason !== undefined)
387
+ learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
388
+ if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
389
+ raw.push({
390
+ rule: 'session/unannounced',
391
+ severity: 'warn',
392
+ step: index,
393
+ message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
394
+ fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
395
+ });
396
+ }
397
+ if (step.kind !== 'list' && probes) {
398
+ // The main process is still running here: a server that holds a file or
399
+ // a port can't start a second copy. That leaves the scope unknown, not
400
+ // the run failed.
401
+ const probeWaited = { ms: 0 };
402
+ try {
403
+ probedAfter = index;
404
+ const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
405
+ record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
406
+ raw.push(...scopeFindings(record.scope, modern, index));
407
+ }
408
+ catch (error) {
409
+ record.scope = 'unclear';
410
+ scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
411
+ }
306
412
  }
307
413
  }
414
+ // --auto: the tools this step brought get their calls now, while they're listed.
415
+ const added = new Set(changes.filter((c) => c.kind === 'added').map((c) => c.tool));
416
+ if (options.replan && added.size)
417
+ scenario.steps.splice(i + 1, 0, ...options.replan(next.tools.filter((t) => added.has(t.name))));
308
418
  current = next;
309
419
  see(next);
310
420
  }
311
421
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
312
422
  record.tools = current.tools.length;
313
423
  record.tokens = current.totalTokens;
314
- steps.push(record);
424
+ if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
425
+ quiet.add(step.tool);
426
+ done();
315
427
  }
428
+ raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
429
+ if (explained.length)
430
+ raw.push(knownVariance(explained));
431
+ if (onWord.length)
432
+ raw.push(onTheirWord(onWord, options.auto !== undefined));
316
433
  raw.push(...untested(steps, target));
317
434
  // --auto means "find what you can": an unlock it skipped is a gap too.
318
- raw.push(...unlockCoverage(baseline.tools, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
435
+ const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
436
+ // An unlock --auto stopped trying isn't charged for the values it then left out.
437
+ const covered = baseline.tools.filter((t) => !stopped.has(t.name));
438
+ raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
319
439
  if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
320
- raw.push(nothingCalled(options.auto.skipped));
440
+ raw.push(nothingCalled(options.auto));
321
441
  if (scopeUnchecked.length) {
322
442
  const which = scopeUnchecked.map((s) => s.step);
323
443
  raw.push({
@@ -336,8 +456,10 @@ export async function session(target, scenario, options = {}) {
336
456
  final: { tools: current.tools.length, tokens: current.totalTokens },
337
457
  connectionCheck,
338
458
  steps,
339
- findings: settle(mergeSideEffects(raw), options),
340
- union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
459
+ findings: settle(mergeRepeats(raw), options),
460
+ ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
461
+ // Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
462
+ union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
341
463
  };
342
464
  }
343
465
  finally {
@@ -359,13 +481,17 @@ function untested(steps, target) {
359
481
  for (const s of setup)
360
482
  byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
361
483
  const all = setup.length === calls.length;
362
- const how = target.kind === 'http' ? 'real credentials (--header "Authorization: …")' : 'real credentials (--env KEY=…)';
484
+ // One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
485
+ const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
363
486
  return [
364
487
  {
365
488
  rule: 'session/untested',
366
489
  severity: all ? 'error' : 'warn',
367
- message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.${byClass.has('auth') || byClass.has('not-found') ? ` Run with ${how}.` : ''}`,
490
+ // Classified from the error's words: a guess, if a good one.
491
+ confidence: 'unsure',
492
+ message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
368
493
  detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
494
+ fix,
369
495
  },
370
496
  ];
371
497
  }
@@ -410,45 +536,252 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
410
536
  ? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
411
537
  : skip === 'not read-only'
412
538
  ? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
413
- : '';
539
+ : skip === 'unmarked'
540
+ ? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
541
+ : '';
414
542
  out.push({
415
543
  rule: 'session/unlock-coverage',
416
544
  severity: 'warn',
417
545
  tool: u.tool.name,
546
+ // "Looks like it unlocks": from its name, parameter and description.
547
+ confidence: 'unsure',
418
548
  ...(last !== undefined ? { step: last } : {}),
419
- message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}${why ? '' : ' Unlock every value (session --init puts every one in the starter).'}`,
549
+ message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
420
550
  detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
551
+ fix: skip === 'open world'
552
+ ? 'Rerun with --open-world.'
553
+ : skip === 'unmarked'
554
+ ? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
555
+ : skip === 'not read-only'
556
+ ? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
557
+ : 'Unlock every value: session --init puts each one in the starter.',
421
558
  });
422
559
  }
423
560
  return out;
424
561
  }
425
562
  /**
426
563
  * --auto that called nothing: the run only listed the menu, so a clean result
427
- * says nothing about what the tools do to it. Why each tool was left out.
564
+ * says nothing about what the tools do to it. Why each tool was left out, and the
565
+ * flag that reaches the most of them.
428
566
  */
429
- function nothingCalled(skipped) {
430
- const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
567
+ function nothingCalled(auto) {
568
+ const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
431
569
  const parts = [];
432
570
  const openWorld = by('open world');
433
571
  const needs = by('needs values');
572
+ const unmarked = by('unmarked');
434
573
  const writes = by('not read-only');
435
- if (openWorld.length)
436
- parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
437
574
  if (needs.length)
438
- parts.push(`${needs.length} need values the schema doesn't give (--save-scenario puts them in a scenario to fill in)`);
575
+ parts.push(`${needs.length} ${needs.length === 1 ? 'needs' : 'need'} values the schema doesn't give`);
576
+ if (unmarked.length)
577
+ parts.push(`${unmarked.length} ${unmarked.length === 1 ? "isn't" : "aren't"} marked readOnlyHint`);
578
+ if (openWorld.length)
579
+ parts.push(`${openWorld.length} marked openWorldHint (${openWorld.length === 1 ? 'it' : 'they'} may cost API credits)`);
439
580
  if (writes.length)
440
- parts.push(`${writes.length} not marked readOnlyHint`);
581
+ parts.push(`${writes.length} marked or named as writes`);
441
582
  const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
583
+ const params = neededValues(auto.skipped);
584
+ const fix = autoNextStep(auto);
442
585
  return {
443
586
  rule: 'session/nothing-called',
444
587
  severity: 'warn',
445
588
  message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
446
589
  detail: [
590
+ ...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
591
+ ...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
447
592
  ...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
448
- ...(needs.length ? [`need values: ${list(needs)}`] : []),
449
593
  ],
594
+ ...(fix ? { fix } : {}),
450
595
  };
451
596
  }
597
+ /**
598
+ * The same error text from different tools is one finding (two tools of a feature
599
+ * that "isn't set up on this deployment"). An empty text says nothing in common.
600
+ */
601
+ function sameError(failures) {
602
+ const out = [];
603
+ const byText = new Map();
604
+ for (const f of failures) {
605
+ const key = `${f.rule}\0${f.text}`;
606
+ const first = f.text ? byText.get(key) : undefined;
607
+ if (!first) {
608
+ out.push({ ...f, steps: [...f.steps] });
609
+ if (f.text)
610
+ byText.set(key, out[out.length - 1]);
611
+ continue;
612
+ }
613
+ first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
614
+ first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
615
+ }
616
+ return out;
617
+ }
618
+ // The server says the feature isn't there, or isn't for this account: no argument
619
+ // fixes that. A guess from its words.
620
+ // Only definite statements: a hedge in the server's own advice ("the account may lack
621
+ // permission") says nothing about this call, and a bare "permission" or "role" turns up in
622
+ // ordinary validation messages.
623
+ const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\b(?:permission|access)s? denied\b|\binsufficient (?:permissions?|privileges?)\b|\b(?:don't|doesn't|do not|does not) have (?:the )?(?:required |necessary )?(?:permissions?|access)\b|\bforbidden\b|\bnot allowed\b/i;
624
+ /** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
625
+ export function stepsText(steps) {
626
+ if (steps.length === 1)
627
+ return `step ${steps[0]}`;
628
+ const runs = [];
629
+ for (let i = 0; i < steps.length;) {
630
+ let j = i;
631
+ while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
632
+ j++;
633
+ if (j - i >= 2)
634
+ runs.push(`${steps[i]}–${steps[j]}`);
635
+ else
636
+ for (let k = i; k <= j; k++)
637
+ runs.push(String(steps[k]));
638
+ i = j + 1;
639
+ }
640
+ return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
641
+ }
642
+ /**
643
+ * The same error from the same tool is one finding that names every step it
644
+ * happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
645
+ * words, the call that got it, and what to try.
646
+ */
647
+ function failureFindings(failures, auto) {
648
+ return failures.map((f) => {
649
+ const many = f.steps.length > 1;
650
+ const calls = [f, ...(f.others ?? [])];
651
+ const tools = [...new Set(calls.map((c) => c.tool))];
652
+ const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
653
+ const words = errorWords(f.text);
654
+ const shown = clip(words, 160);
655
+ const said = f.text ? quotedSentence(shown) : 'no error text.';
656
+ const firstArg = Object.keys(f.args)[0];
657
+ const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
658
+ // No argument can fix a call to a tool that takes none, or a feature the server
659
+ // says isn't there. From the server's words: a guess.
660
+ const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
661
+ const them = tools.length === 1 ? 'its' : 'their';
662
+ const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
663
+ const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
664
+ const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
665
+ const fix = f.timedOutMs
666
+ ? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
667
+ : !argsCan
668
+ ? undefined
669
+ : unavailable
670
+ ? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
671
+ : noParams
672
+ ? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
673
+ : auto?.withValues?.includes(f.tool)
674
+ ? `Check the values you gave ${f.tool} (--value) against what the server says above.`
675
+ : auto
676
+ ? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
677
+ : `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
678
+ const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
679
+ const detail = calls.slice(0, 8).map((c) => {
680
+ const call = `${c.tool} ${JSON.stringify(c.args)}`;
681
+ return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
682
+ });
683
+ return {
684
+ rule: f.rule,
685
+ severity: f.severity,
686
+ step: f.steps[0],
687
+ ...(many ? { steps: f.steps } : {}),
688
+ tool: f.tool,
689
+ // The server's whole text, when the message above shows less of it.
690
+ ...(f.text && shown !== f.text ? { serverText: f.text } : {}),
691
+ ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
692
+ message: f.rule === 'session/tool-error'
693
+ ? tools.length > 1
694
+ ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
695
+ : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
696
+ : tools.length > 1
697
+ ? `${lead}Calling ${who} failed the same way: ${said}`
698
+ : `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
699
+ detail: [...new Set(detail)],
700
+ ...(fix ? { fix } : {}),
701
+ };
702
+ });
703
+ }
704
+ /**
705
+ * Changes that touched only values already seen to vary on their own: one info
706
+ * finding for the run, pointing to the one that reported the cause.
707
+ */
708
+ function knownVariance(explained) {
709
+ const steps = explained.map((e) => e.step);
710
+ const src = explained[0].source;
711
+ const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
712
+ // The values at the first such step; the rest only repeat the pattern.
713
+ const seen = explained[0].seen;
714
+ return {
715
+ rule: 'session/known-variance',
716
+ severity: 'info',
717
+ step: steps[0],
718
+ message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
719
+ detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
720
+ fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
721
+ };
722
+ }
723
+ /** Tools called because the user said they only read: said once, loudly. */
724
+ function onTheirWord(calls, auto) {
725
+ const tools = [...new Set(calls.map((c) => c.tool))];
726
+ return {
727
+ rule: 'session/assumed-read-only',
728
+ severity: 'info',
729
+ step: calls[0].step,
730
+ message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
731
+ detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
732
+ fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
733
+ };
734
+ }
735
+ /**
736
+ * Every leaf value that differs between the old and new definition of each edited
737
+ * tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
738
+ */
739
+ function variedPaths(before, after, changes) {
740
+ const out = new Map();
741
+ const old = new Map(before.map((t) => [t.name, t]));
742
+ for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
743
+ const a = old.get(name);
744
+ const b = after.find((t) => t.name === name);
745
+ if (!a || !b)
746
+ continue;
747
+ for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
748
+ out.set(`${name}\0${path}`, d);
749
+ }
750
+ return out;
751
+ }
752
+ function leafDiffs(a, b, path = '', out = new Map()) {
753
+ if (canonical(a) === canonical(b))
754
+ return out;
755
+ const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
756
+ if (obj(a) && obj(b)) {
757
+ for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
758
+ leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
759
+ }
760
+ else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
761
+ a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
762
+ }
763
+ else
764
+ out.set(path, { before: a, after: b });
765
+ return out;
766
+ }
767
+ function show(value) {
768
+ if (value === undefined)
769
+ return '(absent)';
770
+ const text = JSON.stringify(value);
771
+ return text.length > 60 ? text.slice(0, 57) + '…' : text;
772
+ }
773
+ const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
774
+ /** The next step for a menu that differs between processes, from what differs. */
775
+ function varianceFix(main, other, changes) {
776
+ const diffs = [...variedPaths(main, other, changes)];
777
+ if (diffs.length === 0)
778
+ return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
779
+ const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
780
+ if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
781
+ return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
782
+ }
783
+ return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
784
+ }
452
785
  function serverOf(c) {
453
786
  return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
454
787
  }
@@ -477,6 +810,7 @@ function waitFor(done, ms) {
477
810
  }, 20);
478
811
  });
479
812
  }
813
+ const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
480
814
  const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
481
815
  /** Turn a menu change into findings, with the cost of each. */
482
816
  export function changeFindings(before, after, changes, step, origin) {
@@ -489,7 +823,7 @@ export function changeFindings(before, after, changes, step, origin) {
489
823
  const cost = !brk
490
824
  ? []
491
825
  : [
492
- `the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}; any change to the tool list invalidates the cached prompt, so the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it are processed again`,
826
+ `cost: the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it; the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}`,
493
827
  ];
494
828
  const why = origin ? [origin] : [];
495
829
  const added = changes.filter((c) => c.kind === 'added');
@@ -504,8 +838,10 @@ export function changeFindings(before, after, changes, step, origin) {
504
838
  rule: 'session/append',
505
839
  severity: 'warn',
506
840
  step,
507
- message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}). The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.`,
841
+ message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
508
842
  detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
843
+ each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
844
+ fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
509
845
  });
510
846
  }
511
847
  else {
@@ -516,22 +852,56 @@ export function changeFindings(before, after, changes, step, origin) {
516
852
  step,
517
853
  message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
518
854
  detail: [...cost, ...why],
855
+ fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
519
856
  });
520
857
  }
521
858
  }
522
859
  const moved = changes.filter((c) => c.kind === 'moved');
523
860
  if (moved.length) {
524
- out.push({ rule: 'session/reorder', severity: 'error', step, message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}).`, detail: [...cost, ...why] });
861
+ out.push({
862
+ rule: 'session/reorder',
863
+ severity: 'error',
864
+ step,
865
+ message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
866
+ detail: [...cost, ...why],
867
+ fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
868
+ });
525
869
  }
526
870
  const removed = changes.filter((c) => c.kind === 'removed');
527
871
  if (removed.length) {
528
- out.push({ rule: 'session/remove', severity: 'error', step, message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session.`, detail: [...cost, ...why] });
872
+ out.push({
873
+ rule: 'session/remove',
874
+ severity: 'error',
875
+ step,
876
+ message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
877
+ detail: [...cost, ...why],
878
+ fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
879
+ });
529
880
  }
530
881
  const edits = new Map();
531
882
  for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
532
883
  edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
884
+ const old = new Map(before.map((t) => [t.name, t]));
533
885
  for (const [tool, fields] of edits) {
534
- out.push({ rule: 'session/edit', severity: 'error', step, tool, message: `${tool}: ${fields.join(', ')} changed mid-session.`, detail: [...cost, ...why] });
886
+ const a = old.get(tool);
887
+ const b = after.find((t) => t.name === tool);
888
+ // What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
889
+ const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
890
+ // One cause: the same tool, the same changed values and the same origin (a
891
+ // timestamp in a description, rewritten at every call). A change that touches
892
+ // anything more is a change of its own.
893
+ const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
894
+ out.push({
895
+ rule: 'session/edit',
896
+ severity: 'error',
897
+ step,
898
+ tool,
899
+ cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
900
+ each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
901
+ message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
902
+ detail: [...seen, ...cost, ...why],
903
+ fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
904
+ });
535
905
  }
536
906
  return out;
537
907
  }
@@ -576,6 +946,7 @@ function scopeFindings(scope, modern, step) {
576
946
  message: modern
577
947
  ? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
578
948
  : `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
949
+ fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
579
950
  },
580
951
  ];
581
952
  }
@@ -586,20 +957,57 @@ function scopeFindings(scope, modern, step) {
586
957
  severity: 'warn',
587
958
  step,
588
959
  message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
960
+ fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
589
961
  },
590
962
  ];
591
963
  }
592
964
  return [];
593
965
  }
594
- /** stdio's side-effect warning is the same for every step: say it once, name the steps. */
595
- function mergeSideEffects(raw) {
596
- const side = raw.filter((f) => f.rule === 'session/side-effect');
597
- if (side.length < 2)
598
- return raw;
599
- const steps = side.map((f) => f.step);
600
- return raw
601
- .filter((f) => f.rule !== 'session/side-effect' || f === side[0])
602
- .map((f) => (f === side[0] ? { ...f, message: `Steps ${steps.join(', ')}: ${f.message}` } : f));
966
+ /**
967
+ * Findings that say the same thing at every step are said once, naming the steps:
968
+ * stdio's side-effect warning, and an unlock per domain, where each value appends
969
+ * tools for this connection only (9 steps made 18 near-identical blocks). What
970
+ * differs per step, the tools and their cost, goes in the detail. Mid-inserts
971
+ * and removals stay per step: each is its own change. Edits merge only when the
972
+ * tool, the changed values and their origin are the same: one cause.
973
+ */
974
+ const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
975
+ export function mergeRepeats(raw) {
976
+ let out = raw;
977
+ for (const rule of REPEATS) {
978
+ const same = out.filter((f) => f.rule === rule);
979
+ if (same.length < 2)
980
+ continue;
981
+ const steps = same.map((f) => f.step);
982
+ const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
983
+ let message = `${lead}: ${same[0].message}`;
984
+ let detail = same[0].detail;
985
+ if (rule === 'session/append') {
986
+ const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
987
+ message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
988
+ const each = same.map((f) => f.each).filter(Boolean);
989
+ const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
990
+ detail = [...each, ...why];
991
+ }
992
+ const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
993
+ out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
994
+ }
995
+ // The same edit at several steps is one cause: one finding naming the steps,
996
+ // with each step's position and cost.
997
+ const byCause = new Map();
998
+ for (const f of out)
999
+ if (f.rule === 'session/edit' && f.cause)
1000
+ byCause.set(f.cause, [...(byCause.get(f.cause) ?? []), f]);
1001
+ for (const same of byCause.values()) {
1002
+ if (same.length < 2)
1003
+ continue;
1004
+ const steps = same.map((f) => f.step);
1005
+ // The per-step value and cost are in each step's line; keep the rest (the origin) once.
1006
+ const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.startsWith('cost: ') && !f.each.endsWith(` · ${d}`))))];
1007
+ const merged = { ...same[0], steps, message: `${stepsText(steps).replace(/^s/, 'S')}: ${same[0].message}`, detail: [...same.map((f) => f.each), ...why] };
1008
+ out = out.filter((f) => !same.includes(f) || f === same[0]).map((f) => (f === same[0] ? merged : f));
1009
+ }
1010
+ return out;
603
1011
  }
604
1012
  function describe(changes) {
605
1013
  return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
@@ -608,7 +1016,7 @@ function settle(raw, options) {
608
1016
  const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
609
1017
  return raw
610
1018
  .filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
611
- .map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
1019
+ .map(({ each: _each, cause: _cause, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
612
1020
  .sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
613
1021
  }
614
1022
  /*
@@ -630,7 +1038,25 @@ const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand
630
1038
  // capabilities alone aren't tools.
631
1039
  const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
632
1040
  // Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
633
- const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
1041
+ // "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
1042
+ const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
1043
+ /**
1044
+ * Does the tool say that `param` unlocks tools? Its name does, or a sentence of
1045
+ * its description that says it unlocks tools and doesn't pin that on another
1046
+ * parameter ("A query match also enables the tools…" is about `query`, not the
1047
+ * `domain` filter next to it).
1048
+ */
1049
+ function claimsUnlock(tool, param) {
1050
+ if (UNLOCK_NAME.test(tool.name))
1051
+ return true;
1052
+ const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
1053
+ const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
1054
+ const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
1055
+ return (tool.description ?? '')
1056
+ .split(/(?<=[.!?])\s+/)
1057
+ .filter((s) => UNLOCK_SAYS.test(s))
1058
+ .some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
1059
+ }
634
1060
  const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
635
1061
  /**
636
1062
  * At most `max` characters, cut by code point at a word boundary, never inside a
@@ -672,7 +1098,8 @@ export function unlockers(tools) {
672
1098
  // A lookup names toolsets without changing them (GitHub's get_toolset_tools,
673
1099
  // toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
674
1100
  const verb = verbOf(tool.name);
675
- if (verb && LOOKUPS.has(verb) && !UNLOCK_SAYS.test(description))
1101
+ const lookup = verb !== undefined && LOOKUPS.has(verb);
1102
+ if (lookup && !UNLOCK_SAYS.test(description))
676
1103
  continue;
677
1104
  const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
678
1105
  const required = tool.inputSchema?.required ?? [];
@@ -681,6 +1108,10 @@ export function unlockers(tools) {
681
1108
  // Named like an unlock, one required parameter: that's what it takes
682
1109
  // (toolception's enable_toolset { name }).
683
1110
  (UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
1111
+ const claims = claimsUnlock(tool, param);
1112
+ // A lookup that says a *different* parameter unlocks: its enum is a filter.
1113
+ if (lookup && !claims)
1114
+ continue;
684
1115
  let score = 0;
685
1116
  if (param)
686
1117
  score += enumOf(props[param]).length ? 3 : 2;
@@ -689,7 +1120,7 @@ export function unlockers(tools) {
689
1120
  if (UNLOCK_DESC.test(tool.description ?? ''))
690
1121
  score += 1;
691
1122
  if (score >= 2)
692
- found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
1123
+ found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
693
1124
  }
694
1125
  return found.sort((a, b) => b.score - a.score);
695
1126
  }