toolmenu 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +76 -20
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +53 -17
  6. package/dist/auto.d.ts +70 -6
  7. package/dist/auto.js +248 -17
  8. package/dist/breakdown.d.ts +2 -0
  9. package/dist/breakdown.js +62 -19
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +80 -170
  13. package/dist/config.d.ts +5 -1
  14. package/dist/config.js +14 -0
  15. package/dist/connect.d.ts +4 -0
  16. package/dist/connect.js +132 -11
  17. package/dist/diff.d.ts +23 -1
  18. package/dist/diff.js +526 -114
  19. package/dist/difference.d.ts +2 -0
  20. package/dist/difference.js +13 -12
  21. package/dist/explain.d.ts +103 -0
  22. package/dist/explain.js +434 -0
  23. package/dist/failures.d.ts +9 -0
  24. package/dist/failures.js +17 -0
  25. package/dist/help.d.ts +9 -0
  26. package/dist/help.js +198 -0
  27. package/dist/history.d.ts +13 -0
  28. package/dist/history.js +86 -6
  29. package/dist/init.d.ts +92 -2
  30. package/dist/init.js +507 -20
  31. package/dist/menu.js +10 -2
  32. package/dist/options.d.ts +135 -0
  33. package/dist/options.js +48 -0
  34. package/dist/report.d.ts +1 -1
  35. package/dist/report.js +229 -99
  36. package/dist/rules/description.d.ts +7 -2
  37. package/dist/rules/description.js +85 -42
  38. package/dist/rules/determinism.d.ts +3 -1
  39. package/dist/rules/determinism.js +145 -11
  40. package/dist/rules/ids.d.ts +1 -1
  41. package/dist/rules/ids.js +9 -4
  42. package/dist/rules/index.d.ts +6 -0
  43. package/dist/rules/index.js +12 -4
  44. package/dist/rules/naming.d.ts +1 -1
  45. package/dist/rules/naming.js +66 -19
  46. package/dist/rules/rule.d.ts +18 -0
  47. package/dist/rules/rule.js +25 -1
  48. package/dist/rules/spec.js +33 -9
  49. package/dist/rules/write.d.ts +1 -1
  50. package/dist/rules/write.js +42 -10
  51. package/dist/session.d.ts +18 -8
  52. package/dist/session.js +484 -92
  53. package/dist/snapshot.d.ts +2 -2
  54. package/dist/snapshot.js +17 -2
  55. package/dist/types.d.ts +14 -0
  56. package/dist/words.d.ts +5 -0
  57. package/dist/words.js +14 -0
  58. package/package.json +1 -1
package/dist/session.js CHANGED
@@ -5,9 +5,11 @@ import { listTools } from './connect.js';
5
5
  import { buildMenu, toolDefinition } from './menu.js';
6
6
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
7
  import { varianceFinding } from './rules/determinism.js';
8
- import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_ADVICE, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, tooMany, waitedFor, whyNot } from './failures.js';
8
+ import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
9
  import { SEVERITY_RANK } from './types.js';
10
10
  import { leadingJson } from './catalog.js';
11
+ import { describeToolDifference } from './difference.js';
12
+ import { autoNextStep, neededValues, writeSign } from './auto.js';
11
13
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
12
14
  const LIST_CHANGED = 'notifications/tools/list_changed';
13
15
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
@@ -31,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
31
33
  if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
32
34
  throw new Error(`${where}: allow_writes must be true or false`);
33
35
  }
36
+ const assumed = doc.assume_read_only;
37
+ if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
38
+ throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
39
+ }
34
40
  const steps = doc.steps.map((raw, i) => {
35
41
  const at = `${where}: step ${i + 1}`;
36
42
  if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
@@ -55,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
55
61
  }
56
62
  throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
57
63
  });
58
- return { allowWrites: doc.allow_writes === true, steps };
64
+ return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
59
65
  }
60
66
  export function stepLabel(step) {
61
67
  if (step.kind === 'list')
@@ -104,9 +110,13 @@ export async function session(target, scenario, options = {}) {
104
110
  rule: 'session/rate-limited',
105
111
  severity: 'error',
106
112
  step: index,
113
+ ...(tool ? { tool } : {}),
107
114
  message: tool
108
- ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked} If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota.`
109
- : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} ${RATE_LIMIT_ADVICE}`,
115
+ ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
116
+ : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
117
+ fix: tool
118
+ ? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
119
+ : 'Give the run its own server instance, or raise the limit for it.',
110
120
  });
111
121
  };
112
122
  let current = await listPatiently();
@@ -115,17 +125,35 @@ export async function session(target, scenario, options = {}) {
115
125
  const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
116
126
  see(current);
117
127
  const baseline = current;
128
+ // Values that differ from one tools/list to the next on their own (a default
129
+ // computed from the clock: PayPal's list_transactions.end_date), by tool and
130
+ // path, with the finding that reported them first. A later change only there
131
+ // has the same cause and points to it (SPEC §25: one root cause, one finding).
132
+ const varying = new Map();
133
+ const learn = (before, after, changes, source) => {
134
+ for (const [key] of variedPaths(before, after, changes))
135
+ if (!varying.has(key))
136
+ varying.set(key, source);
137
+ };
118
138
  // Fresh processes or connections, same credentials, must see the same menu.
119
139
  let connectionCheck;
120
140
  for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
121
141
  if (probe.error) {
122
- raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}` });
142
+ raw.push({
143
+ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
144
+ severity: 'info',
145
+ step: 0,
146
+ message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
147
+ fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
148
+ });
123
149
  continue;
124
150
  }
125
151
  const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
126
152
  connectionCheck = f ? 'different' : connectionCheck ?? 'same';
127
153
  if (f) {
128
- raw.push({ ...f, step: 0 });
154
+ const changes = compareMenus(current.tools, probe.tools ?? []);
155
+ raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
156
+ learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
129
157
  break;
130
158
  }
131
159
  }
@@ -146,26 +174,75 @@ export async function session(target, scenario, options = {}) {
146
174
  rule: 'session/session-lost',
147
175
  severity: 'error',
148
176
  step: index,
149
- message: `The server ended this session (“${said}”) after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that, so the rest of the scenario didn't run. Run with --processes 1: toolmenu then opens no second one.`,
177
+ confidence: 'unsure',
178
+ message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
179
+ fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
150
180
  });
151
181
  return true;
152
182
  };
153
183
  const callWaited = { ms: 0 };
154
184
  const listWaited = { ms: 0 };
185
+ const assumed = new Set(scenario.assumeReadOnly ?? []);
186
+ // Called on the user's word (assume_read_only), by step.
187
+ const onWord = [];
188
+ // The same failure from the same tool at several steps is one finding that lists them.
189
+ const failures = new Map();
190
+ const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
191
+ const key = `${rule}\0${step.tool}\0${text}`;
192
+ const before = failures.get(key);
193
+ if (before)
194
+ before.steps.push(index);
195
+ else {
196
+ const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
197
+ failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
198
+ }
199
+ };
200
+ // Steps whose change touched only values already known to vary on their own.
201
+ const explained = [];
202
+ // --auto's guessed unlocks: tried once, and those whose first call changed nothing.
203
+ const tried = new Set();
204
+ const quiet = new Set();
205
+ const stopped = new Map();
155
206
  for (const [i, step] of scenario.steps.entries()) {
156
207
  const index = i + 1;
208
+ if (step.kind === 'call' && step.tentative) {
209
+ if (quiet.has(step.tool)) {
210
+ stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
211
+ continue;
212
+ }
213
+ }
214
+ const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
215
+ if (firstTry)
216
+ tried.add(step.tool);
157
217
  const label = stepLabel(step);
158
218
  const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
159
219
  const mark = conn.wire.notifications.length;
220
+ let word = false;
221
+ const done = () => {
222
+ if (word)
223
+ record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
224
+ steps.push(record);
225
+ };
160
226
  if (step.kind === 'call') {
161
227
  const tool = current.tools.find((t) => t.name === step.tool);
228
+ const write = tool ? writeSign(tool) : undefined;
229
+ word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
162
230
  if (!tool) {
163
231
  record.status = 'failed';
164
232
  record.reason = 'missing';
165
233
  record.note = `${step.tool} isn't in the menu at this point`;
166
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `Step ${index} calls ${step.tool}, which isn't in the menu at this point.` });
234
+ const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
235
+ const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
236
+ raw.push({
237
+ rule: 'session/step-failed',
238
+ severity: 'error',
239
+ step: index,
240
+ tool: step.tool,
241
+ message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
242
+ fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
243
+ });
167
244
  }
168
- else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
245
+ else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
169
246
  record.status = 'refused';
170
247
  record.reason = 'refused';
171
248
  record.note = 'not marked readOnlyHint';
@@ -174,15 +251,22 @@ export async function session(target, scenario, options = {}) {
174
251
  severity: 'error',
175
252
  step: index,
176
253
  tool: step.tool,
177
- message: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real. Set "allow_writes: true" in the scenario if that's intended.`,
254
+ message: assumed.has(step.tool) && write
255
+ ? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
256
+ : `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
257
+ fix: write || tool.annotations?.readOnlyHint === false
258
+ ? 'Set "allow_writes: true" in the scenario if calling it is intended.'
259
+ : `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
178
260
  });
179
261
  }
262
+ if (word)
263
+ onWord.push({ tool: step.tool, step: index });
180
264
  if (record.status === 'ok')
181
265
  try {
182
266
  // Sent again after a refusal only when that repeats nothing: the transport's
183
267
  // 429 means the server never ran it; a tool that says it was rate-limited,
184
268
  // thrown or as its result, may have done part of the work first, so only a
185
- // read-only one is asked again.
269
+ // read-only one is asked again. On the server's word, not the user's.
186
270
  const readOnly = tool?.annotations?.readOnlyHint === true;
187
271
  callWaited.ms = 0;
188
272
  const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
@@ -197,7 +281,7 @@ export async function session(target, scenario, options = {}) {
197
281
  limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
198
282
  record.status = 'failed';
199
283
  record.note = 'rate-limited';
200
- steps.push(record);
284
+ done();
201
285
  break;
202
286
  }
203
287
  const short = clip(text, 120);
@@ -205,32 +289,28 @@ export async function session(target, scenario, options = {}) {
205
289
  record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
206
290
  // A setup failure is reported once, for the whole run (session/untested).
207
291
  if (!SETUP_FAILURES.has(record.failure))
208
- raw.push({
209
- rule: 'session/tool-error',
210
- severity: 'warn',
211
- step: index,
212
- tool: step.tool,
213
- message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
214
- });
292
+ fail('session/tool-error', 'warn', index, step, text, record.failure);
215
293
  }
216
294
  }
217
295
  catch (error) {
218
296
  record.status = 'failed';
219
297
  record.note = serverWords(error, 200);
220
298
  if (lost(record.note, index)) {
221
- steps.push(record);
299
+ done();
222
300
  break;
223
301
  }
224
302
  if (tooMany(error)) {
225
303
  limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
226
304
  record.note = 'rate-limited';
227
- steps.push(record);
305
+ done();
228
306
  break;
229
307
  }
230
308
  const code = error.code;
231
309
  record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
310
+ // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
311
+ const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
232
312
  if (!SETUP_FAILURES.has(record.failure))
233
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
313
+ fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
234
314
  }
235
315
  }
236
316
  else if (step.kind === 'wait_for') {
@@ -252,57 +332,78 @@ export async function session(target, scenario, options = {}) {
252
332
  record.status = 'failed';
253
333
  record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
254
334
  if (lost(why, index)) {
255
- steps.push(record);
335
+ done();
256
336
  break;
257
337
  }
258
338
  if (tooMany(error)) {
259
339
  limited(index, why, listWaited.ms);
260
340
  record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
261
- steps.push(record);
341
+ done();
262
342
  break;
263
343
  }
264
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
265
- steps.push(record);
344
+ raw.push({
345
+ rule: 'session/step-failed',
346
+ severity: 'error',
347
+ step: index,
348
+ message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
349
+ fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
350
+ });
351
+ done();
266
352
  continue;
267
353
  }
268
354
  const changes = compareMenus(current.tools, next.tools);
269
355
  if (changes.length) {
270
356
  record.changed = true;
271
- if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
272
- await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
273
- }
274
- const origin = step.kind === 'list'
275
- ? 'no tool call in between'
276
- : record.reason === 'missing'
277
- ? 'no tool call made (the tool wasn\'t in the menu)'
278
- : record.reason === 'refused'
279
- ? 'no tool call made (refused)'
280
- : record.status === 'failed'
281
- ? 'the call failed, but the menu changed: the server may have applied it anyway'
282
- : undefined;
283
- raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
284
- if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
285
- raw.push({
286
- rule: 'session/unannounced',
287
- severity: 'warn',
288
- step: index,
289
- message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
290
- });
357
+ const paths = variedPaths(current.tools, next.tools, changes);
358
+ const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
359
+ if (onItsOwn) {
360
+ // Same cause as a finding already made: said once, after the steps.
361
+ explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
362
+ const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
363
+ record.note = record.note ? `${record.note}; ${cause}` : cause;
291
364
  }
292
- if (step.kind !== 'list' && probes) {
293
- // The main process is still running here: a server that holds a file or
294
- // a port can't start a second copy. That leaves the scope unknown, not
295
- // the run failed.
296
- const probeWaited = { ms: 0 };
297
- try {
298
- probedAfter = index;
299
- const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
300
- record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
301
- raw.push(...scopeFindings(record.scope, modern, index));
365
+ else {
366
+ if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
367
+ await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
302
368
  }
303
- catch (error) {
304
- record.scope = 'unclear';
305
- scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
369
+ const origin = step.kind === 'list'
370
+ ? 'no tool call in between'
371
+ : record.reason === 'missing'
372
+ ? 'no tool call made (the tool wasn\'t in the menu)'
373
+ : record.reason === 'refused'
374
+ ? 'no tool call made (refused)'
375
+ : record.status === 'failed'
376
+ ? 'the call failed, but the menu changed: the server may have applied it anyway'
377
+ : undefined;
378
+ raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
379
+ // With no call in between, whatever changed varies on its own: a later
380
+ // change only there has the same cause.
381
+ if (step.kind === 'list' || record.reason !== undefined)
382
+ learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
383
+ if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
384
+ raw.push({
385
+ rule: 'session/unannounced',
386
+ severity: 'warn',
387
+ step: index,
388
+ message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
389
+ fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
390
+ });
391
+ }
392
+ if (step.kind !== 'list' && probes) {
393
+ // The main process is still running here: a server that holds a file or
394
+ // a port can't start a second copy. That leaves the scope unknown, not
395
+ // the run failed.
396
+ const probeWaited = { ms: 0 };
397
+ try {
398
+ probedAfter = index;
399
+ const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
400
+ record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
401
+ raw.push(...scopeFindings(record.scope, modern, index));
402
+ }
403
+ catch (error) {
404
+ record.scope = 'unclear';
405
+ scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
406
+ }
306
407
  }
307
408
  }
308
409
  current = next;
@@ -311,13 +412,23 @@ export async function session(target, scenario, options = {}) {
311
412
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
312
413
  record.tools = current.tools.length;
313
414
  record.tokens = current.totalTokens;
314
- steps.push(record);
415
+ if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
416
+ quiet.add(step.tool);
417
+ done();
315
418
  }
419
+ raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
420
+ if (explained.length)
421
+ raw.push(knownVariance(explained));
422
+ if (onWord.length)
423
+ raw.push(onTheirWord(onWord, options.auto !== undefined));
316
424
  raw.push(...untested(steps, target));
317
425
  // --auto means "find what you can": an unlock it skipped is a gap too.
318
- raw.push(...unlockCoverage(baseline.tools, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
426
+ const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
427
+ // An unlock --auto stopped trying isn't charged for the values it then left out.
428
+ const covered = baseline.tools.filter((t) => !stopped.has(t.name));
429
+ raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
319
430
  if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
320
- raw.push(nothingCalled(options.auto.skipped));
431
+ raw.push(nothingCalled(options.auto));
321
432
  if (scopeUnchecked.length) {
322
433
  const which = scopeUnchecked.map((s) => s.step);
323
434
  raw.push({
@@ -336,7 +447,8 @@ export async function session(target, scenario, options = {}) {
336
447
  final: { tools: current.tools.length, tokens: current.totalTokens },
337
448
  connectionCheck,
338
449
  steps,
339
- findings: settle(mergeSideEffects(raw), options),
450
+ findings: settle(mergeRepeats(raw), options),
451
+ ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
340
452
  union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
341
453
  };
342
454
  }
@@ -359,13 +471,17 @@ function untested(steps, target) {
359
471
  for (const s of setup)
360
472
  byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
361
473
  const all = setup.length === calls.length;
362
- const how = target.kind === 'http' ? 'real credentials (--header "Authorization: …")' : 'real credentials (--env KEY=…)';
474
+ // One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
475
+ const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
363
476
  return [
364
477
  {
365
478
  rule: 'session/untested',
366
479
  severity: all ? 'error' : 'warn',
367
- message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.${byClass.has('auth') || byClass.has('not-found') ? ` Run with ${how}.` : ''}`,
480
+ // Classified from the error's words: a guess, if a good one.
481
+ confidence: 'unsure',
482
+ message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
368
483
  detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
484
+ fix,
369
485
  },
370
486
  ];
371
487
  }
@@ -410,45 +526,245 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
410
526
  ? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
411
527
  : skip === 'not read-only'
412
528
  ? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
413
- : '';
529
+ : skip === 'unmarked'
530
+ ? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
531
+ : '';
414
532
  out.push({
415
533
  rule: 'session/unlock-coverage',
416
534
  severity: 'warn',
417
535
  tool: u.tool.name,
536
+ // "Looks like it unlocks": from its name, parameter and description.
537
+ confidence: 'unsure',
418
538
  ...(last !== undefined ? { step: last } : {}),
419
- message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}${why ? '' : ' Unlock every value (session --init puts every one in the starter).'}`,
539
+ message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
420
540
  detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
541
+ fix: skip === 'open world'
542
+ ? 'Rerun with --open-world.'
543
+ : skip === 'unmarked'
544
+ ? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
545
+ : skip === 'not read-only'
546
+ ? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
547
+ : 'Unlock every value: session --init puts each one in the starter.',
421
548
  });
422
549
  }
423
550
  return out;
424
551
  }
425
552
  /**
426
553
  * --auto that called nothing: the run only listed the menu, so a clean result
427
- * says nothing about what the tools do to it. Why each tool was left out.
554
+ * says nothing about what the tools do to it. Why each tool was left out, and the
555
+ * flag that reaches the most of them.
428
556
  */
429
- function nothingCalled(skipped) {
430
- const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
557
+ function nothingCalled(auto) {
558
+ const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
431
559
  const parts = [];
432
560
  const openWorld = by('open world');
433
561
  const needs = by('needs values');
562
+ const unmarked = by('unmarked');
434
563
  const writes = by('not read-only');
435
- if (openWorld.length)
436
- parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
437
564
  if (needs.length)
438
- parts.push(`${needs.length} need values the schema doesn't give (--save-scenario puts them in a scenario to fill in)`);
565
+ parts.push(`${needs.length} need values the schema doesn't give`);
566
+ if (unmarked.length)
567
+ parts.push(`${unmarked.length} aren't marked readOnlyHint`);
568
+ if (openWorld.length)
569
+ parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
439
570
  if (writes.length)
440
- parts.push(`${writes.length} not marked readOnlyHint`);
571
+ parts.push(`${writes.length} marked or named as writes`);
441
572
  const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
573
+ const params = neededValues(auto.skipped);
574
+ const fix = autoNextStep(auto);
442
575
  return {
443
576
  rule: 'session/nothing-called',
444
577
  severity: 'warn',
445
578
  message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
446
579
  detail: [
580
+ ...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
581
+ ...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
447
582
  ...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
448
- ...(needs.length ? [`need values: ${list(needs)}`] : []),
449
583
  ],
584
+ ...(fix ? { fix } : {}),
450
585
  };
451
586
  }
587
+ /**
588
+ * The same error text from different tools is one finding (two tools of a feature
589
+ * that "isn't set up on this deployment"). An empty text says nothing in common.
590
+ */
591
+ function sameError(failures) {
592
+ const out = [];
593
+ const byText = new Map();
594
+ for (const f of failures) {
595
+ const key = `${f.rule}\0${f.text}`;
596
+ const first = f.text ? byText.get(key) : undefined;
597
+ if (!first) {
598
+ out.push({ ...f, steps: [...f.steps] });
599
+ if (f.text)
600
+ byText.set(key, out[out.length - 1]);
601
+ continue;
602
+ }
603
+ first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
604
+ first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
605
+ }
606
+ return out;
607
+ }
608
+ // The server says the feature isn't there, or isn't for this account: no argument
609
+ // fixes that. A guess from its words.
610
+ const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\bpermission\b|\bforbidden\b|\bnot allowed\b|\brole\b/i;
611
+ /** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
612
+ function stepsText(steps) {
613
+ if (steps.length === 1)
614
+ return `step ${steps[0]}`;
615
+ const runs = [];
616
+ for (let i = 0; i < steps.length;) {
617
+ let j = i;
618
+ while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
619
+ j++;
620
+ if (j - i >= 2)
621
+ runs.push(`${steps[i]}–${steps[j]}`);
622
+ else
623
+ for (let k = i; k <= j; k++)
624
+ runs.push(String(steps[k]));
625
+ i = j + 1;
626
+ }
627
+ return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
628
+ }
629
+ /**
630
+ * The same error from the same tool is one finding that names every step it
631
+ * happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
632
+ * words, the call that got it, and what to try.
633
+ */
634
+ function failureFindings(failures, auto) {
635
+ return failures.map((f) => {
636
+ const many = f.steps.length > 1;
637
+ const calls = [f, ...(f.others ?? [])];
638
+ const tools = [...new Set(calls.map((c) => c.tool))];
639
+ const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
640
+ const said = f.text ? quotedSentence(clip(f.text, 160)) : 'no error text.';
641
+ const firstArg = Object.keys(f.args)[0];
642
+ const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
643
+ // No argument can fix a call to a tool that takes none, or a feature the server
644
+ // says isn't there. From the server's words: a guess.
645
+ const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
646
+ const them = tools.length === 1 ? 'its' : 'their';
647
+ const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
648
+ const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
649
+ const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
650
+ const fix = f.timedOutMs
651
+ ? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
652
+ : !argsCan
653
+ ? undefined
654
+ : unavailable
655
+ ? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
656
+ : noParams
657
+ ? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
658
+ : auto?.withValues?.includes(f.tool)
659
+ ? `Check the values you gave ${f.tool} (--value) against what the server says above.`
660
+ : auto
661
+ ? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
662
+ : `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
663
+ const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
664
+ const detail = calls.slice(0, 8).map((c) => {
665
+ const call = `${c.tool} ${JSON.stringify(c.args)}`;
666
+ return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
667
+ });
668
+ return {
669
+ rule: f.rule,
670
+ severity: f.severity,
671
+ step: f.steps[0],
672
+ ...(many ? { steps: f.steps } : {}),
673
+ tool: f.tool,
674
+ ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
675
+ message: f.rule === 'session/tool-error'
676
+ ? tools.length > 1
677
+ ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
678
+ : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
679
+ : tools.length > 1
680
+ ? `${lead}Calling ${who} failed the same way: ${said}`
681
+ : `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
682
+ detail: [...new Set(detail)],
683
+ ...(fix ? { fix } : {}),
684
+ };
685
+ });
686
+ }
687
+ /**
688
+ * Changes that touched only values already seen to vary on their own: one info
689
+ * finding for the run, pointing to the one that reported the cause.
690
+ */
691
+ function knownVariance(explained) {
692
+ const steps = explained.map((e) => e.step);
693
+ const src = explained[0].source;
694
+ const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
695
+ // The values at the first such step; the rest only repeat the pattern.
696
+ const seen = explained[0].seen;
697
+ return {
698
+ rule: 'session/known-variance',
699
+ severity: 'info',
700
+ step: steps[0],
701
+ message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
702
+ detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
703
+ fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
704
+ };
705
+ }
706
+ /** Tools called because the user said they only read: said once, loudly. */
707
+ function onTheirWord(calls, auto) {
708
+ const tools = [...new Set(calls.map((c) => c.tool))];
709
+ return {
710
+ rule: 'session/assumed-read-only',
711
+ severity: 'info',
712
+ step: calls[0].step,
713
+ message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
714
+ detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
715
+ fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
716
+ };
717
+ }
718
+ /**
719
+ * Every leaf value that differs between the old and new definition of each edited
720
+ * tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
721
+ */
722
+ function variedPaths(before, after, changes) {
723
+ const out = new Map();
724
+ const old = new Map(before.map((t) => [t.name, t]));
725
+ for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
726
+ const a = old.get(name);
727
+ const b = after.find((t) => t.name === name);
728
+ if (!a || !b)
729
+ continue;
730
+ for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
731
+ out.set(`${name}\0${path}`, d);
732
+ }
733
+ return out;
734
+ }
735
+ function leafDiffs(a, b, path = '', out = new Map()) {
736
+ if (canonical(a) === canonical(b))
737
+ return out;
738
+ const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
739
+ if (obj(a) && obj(b)) {
740
+ for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
741
+ leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
742
+ }
743
+ else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
744
+ a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
745
+ }
746
+ else
747
+ out.set(path, { before: a, after: b });
748
+ return out;
749
+ }
750
+ function show(value) {
751
+ if (value === undefined)
752
+ return '(absent)';
753
+ const text = JSON.stringify(value);
754
+ return text.length > 60 ? text.slice(0, 57) + '…' : text;
755
+ }
756
+ const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
757
+ /** The next step for a menu that differs between processes, from what differs. */
758
+ function varianceFix(main, other, changes) {
759
+ const diffs = [...variedPaths(main, other, changes)];
760
+ if (diffs.length === 0)
761
+ return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
762
+ const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
763
+ if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
764
+ return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
765
+ }
766
+ return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
767
+ }
452
768
  function serverOf(c) {
453
769
  return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
454
770
  }
@@ -477,6 +793,7 @@ function waitFor(done, ms) {
477
793
  }, 20);
478
794
  });
479
795
  }
796
+ const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
480
797
  const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
481
798
  /** Turn a menu change into findings, with the cost of each. */
482
799
  export function changeFindings(before, after, changes, step, origin) {
@@ -504,8 +821,10 @@ export function changeFindings(before, after, changes, step, origin) {
504
821
  rule: 'session/append',
505
822
  severity: 'warn',
506
823
  step,
507
- message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}). The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.`,
824
+ message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
508
825
  detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
826
+ each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
827
+ fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
509
828
  });
510
829
  }
511
830
  else {
@@ -516,22 +835,50 @@ export function changeFindings(before, after, changes, step, origin) {
516
835
  step,
517
836
  message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
518
837
  detail: [...cost, ...why],
838
+ fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
519
839
  });
520
840
  }
521
841
  }
522
842
  const moved = changes.filter((c) => c.kind === 'moved');
523
843
  if (moved.length) {
524
- out.push({ rule: 'session/reorder', severity: 'error', step, message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}).`, detail: [...cost, ...why] });
844
+ out.push({
845
+ rule: 'session/reorder',
846
+ severity: 'error',
847
+ step,
848
+ message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
849
+ detail: [...cost, ...why],
850
+ fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
851
+ });
525
852
  }
526
853
  const removed = changes.filter((c) => c.kind === 'removed');
527
854
  if (removed.length) {
528
- out.push({ rule: 'session/remove', severity: 'error', step, message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session.`, detail: [...cost, ...why] });
855
+ out.push({
856
+ rule: 'session/remove',
857
+ severity: 'error',
858
+ step,
859
+ message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
860
+ detail: [...cost, ...why],
861
+ fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
862
+ });
529
863
  }
530
864
  const edits = new Map();
531
865
  for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
532
866
  edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
867
+ const old = new Map(before.map((t) => [t.name, t]));
533
868
  for (const [tool, fields] of edits) {
534
- out.push({ rule: 'session/edit', severity: 'error', step, tool, message: `${tool}: ${fields.join(', ')} changed mid-session.`, detail: [...cost, ...why] });
869
+ const a = old.get(tool);
870
+ const b = after.find((t) => t.name === tool);
871
+ // What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
872
+ const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
873
+ out.push({
874
+ rule: 'session/edit',
875
+ severity: 'error',
876
+ step,
877
+ tool,
878
+ message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
879
+ detail: [...seen, ...cost, ...why],
880
+ fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
881
+ });
535
882
  }
536
883
  return out;
537
884
  }
@@ -576,6 +923,7 @@ function scopeFindings(scope, modern, step) {
576
923
  message: modern
577
924
  ? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
578
925
  : `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
926
+ fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
579
927
  },
580
928
  ];
581
929
  }
@@ -586,20 +934,41 @@ function scopeFindings(scope, modern, step) {
586
934
  severity: 'warn',
587
935
  step,
588
936
  message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
937
+ fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
589
938
  },
590
939
  ];
591
940
  }
592
941
  return [];
593
942
  }
594
- /** stdio's side-effect warning is the same for every step: say it once, name the steps. */
595
- function mergeSideEffects(raw) {
596
- const side = raw.filter((f) => f.rule === 'session/side-effect');
597
- if (side.length < 2)
598
- return raw;
599
- const steps = side.map((f) => f.step);
600
- return raw
601
- .filter((f) => f.rule !== 'session/side-effect' || f === side[0])
602
- .map((f) => (f === side[0] ? { ...f, message: `Steps ${steps.join(', ')}: ${f.message}` } : f));
943
+ /**
944
+ * Findings that say the same thing at every step are said once, naming the steps:
945
+ * stdio's side-effect warning, and an unlock per domain, where each value appends
946
+ * tools for this connection only (9 steps made 18 near-identical blocks). What
947
+ * differs per step, the tools and their cost, goes in the detail. Mid-inserts,
948
+ * removals and edits stay per step: each is its own change.
949
+ */
950
+ const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
951
+ function mergeRepeats(raw) {
952
+ let out = raw;
953
+ for (const rule of REPEATS) {
954
+ const same = out.filter((f) => f.rule === rule);
955
+ if (same.length < 2)
956
+ continue;
957
+ const steps = same.map((f) => f.step);
958
+ const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
959
+ let message = `${lead}: ${same[0].message}`;
960
+ let detail = same[0].detail;
961
+ if (rule === 'session/append') {
962
+ const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
963
+ message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
964
+ const each = same.map((f) => f.each).filter(Boolean);
965
+ const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
966
+ detail = [...each, ...why];
967
+ }
968
+ const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
969
+ out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
970
+ }
971
+ return out;
603
972
  }
604
973
  function describe(changes) {
605
974
  return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
@@ -608,7 +977,7 @@ function settle(raw, options) {
608
977
  const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
609
978
  return raw
610
979
  .filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
611
- .map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
980
+ .map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
612
981
  .sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
613
982
  }
614
983
  /*
@@ -630,7 +999,25 @@ const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand
630
999
  // capabilities alone aren't tools.
631
1000
  const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
632
1001
  // Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
633
- const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
1002
+ // "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
1003
+ const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
1004
+ /**
1005
+ * Does the tool say that `param` unlocks tools? Its name does, or a sentence of
1006
+ * its description that says it unlocks tools and doesn't pin that on another
1007
+ * parameter ("A query match also enables the tools…" is about `query`, not the
1008
+ * `domain` filter next to it).
1009
+ */
1010
+ function claimsUnlock(tool, param) {
1011
+ if (UNLOCK_NAME.test(tool.name))
1012
+ return true;
1013
+ const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
1014
+ const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
1015
+ const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
1016
+ return (tool.description ?? '')
1017
+ .split(/(?<=[.!?])\s+/)
1018
+ .filter((s) => UNLOCK_SAYS.test(s))
1019
+ .some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
1020
+ }
634
1021
  const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
635
1022
  /**
636
1023
  * At most `max` characters, cut by code point at a word boundary, never inside a
@@ -672,7 +1059,8 @@ export function unlockers(tools) {
672
1059
  // A lookup names toolsets without changing them (GitHub's get_toolset_tools,
673
1060
  // toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
674
1061
  const verb = verbOf(tool.name);
675
- if (verb && LOOKUPS.has(verb) && !UNLOCK_SAYS.test(description))
1062
+ const lookup = verb !== undefined && LOOKUPS.has(verb);
1063
+ if (lookup && !UNLOCK_SAYS.test(description))
676
1064
  continue;
677
1065
  const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
678
1066
  const required = tool.inputSchema?.required ?? [];
@@ -681,6 +1069,10 @@ export function unlockers(tools) {
681
1069
  // Named like an unlock, one required parameter: that's what it takes
682
1070
  // (toolception's enable_toolset { name }).
683
1071
  (UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
1072
+ const claims = claimsUnlock(tool, param);
1073
+ // A lookup that says a *different* parameter unlocks: its enum is a filter.
1074
+ if (lookup && !claims)
1075
+ continue;
684
1076
  let score = 0;
685
1077
  if (param)
686
1078
  score += enumOf(props[param]).length ? 3 : 2;
@@ -689,7 +1081,7 @@ export function unlockers(tools) {
689
1081
  if (UNLOCK_DESC.test(tool.description ?? ''))
690
1082
  score += 1;
691
1083
  if (score >= 2)
692
- found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
1084
+ found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
693
1085
  }
694
1086
  return found.sort((a, b) => b.score - a.score);
695
1087
  }