toolmenu 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +150 -30
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +61 -18
  6. package/dist/auto.d.ts +71 -7
  7. package/dist/auto.js +267 -13
  8. package/dist/breakdown.d.ts +35 -6
  9. package/dist/breakdown.js +179 -23
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +116 -178
  13. package/dist/compare.js +20 -5
  14. package/dist/config.d.ts +5 -1
  15. package/dist/config.js +14 -0
  16. package/dist/connect.d.ts +4 -0
  17. package/dist/connect.js +132 -11
  18. package/dist/diff.d.ts +35 -1
  19. package/dist/diff.js +1132 -121
  20. package/dist/difference.d.ts +2 -0
  21. package/dist/difference.js +13 -12
  22. package/dist/explain.d.ts +103 -0
  23. package/dist/explain.js +434 -0
  24. package/dist/failures.d.ts +55 -0
  25. package/dist/failures.js +103 -0
  26. package/dist/help.d.ts +9 -0
  27. package/dist/help.js +198 -0
  28. package/dist/history.d.ts +13 -0
  29. package/dist/history.js +86 -6
  30. package/dist/init.d.ts +100 -5
  31. package/dist/init.js +520 -26
  32. package/dist/menu.js +10 -2
  33. package/dist/options.d.ts +135 -0
  34. package/dist/options.js +48 -0
  35. package/dist/probe.d.ts +15 -3
  36. package/dist/probe.js +35 -11
  37. package/dist/report.d.ts +1 -1
  38. package/dist/report.js +229 -99
  39. package/dist/rules/description.d.ts +7 -2
  40. package/dist/rules/description.js +89 -43
  41. package/dist/rules/determinism.d.ts +3 -1
  42. package/dist/rules/determinism.js +146 -11
  43. package/dist/rules/ids.d.ts +1 -1
  44. package/dist/rules/ids.js +9 -4
  45. package/dist/rules/index.d.ts +6 -0
  46. package/dist/rules/index.js +15 -5
  47. package/dist/rules/naming.d.ts +1 -1
  48. package/dist/rules/naming.js +66 -19
  49. package/dist/rules/rule.d.ts +18 -0
  50. package/dist/rules/rule.js +25 -1
  51. package/dist/rules/spec.d.ts +2 -0
  52. package/dist/rules/spec.js +61 -8
  53. package/dist/rules/write.d.ts +1 -1
  54. package/dist/rules/write.js +42 -10
  55. package/dist/semver.js +1 -1
  56. package/dist/session.d.ts +56 -7
  57. package/dist/session.js +800 -94
  58. package/dist/snapshot.d.ts +2 -2
  59. package/dist/snapshot.js +25 -7
  60. package/dist/types.d.ts +14 -0
  61. package/dist/words.d.ts +5 -0
  62. package/dist/words.js +14 -0
  63. package/package.json +1 -1
package/dist/session.js CHANGED
@@ -1,14 +1,19 @@
1
1
  import { readFile } from 'node:fs/promises';
2
2
  import { parse } from 'yaml';
3
- import { cacheBreak, compareMenus } from './compare.js';
4
- import { connect, listTools } from './connect.js';
3
+ import { cacheBreak, canonical, compareMenus } from './compare.js';
4
+ import { listTools } from './connect.js';
5
5
  import { buildMenu, toolDefinition } from './menu.js';
6
- import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
6
+ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
7
  import { varianceFinding } from './rules/determinism.js';
8
- import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
8
+ import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
9
  import { SEVERITY_RANK } from './types.js';
10
- import { verbOf, WRITE_VERBS } from './words.js';
10
+ import { leadingJson } from './catalog.js';
11
+ import { describeToolDifference } from './difference.js';
12
+ import { autoNextStep, neededValues, writeSign } from './auto.js';
13
+ import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
11
14
  const LIST_CHANGED = 'notifications/tools/list_changed';
15
+ /** A server saying the session is gone: "Session not found or expired" (toolception). */
16
+ const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
12
17
  /** Read and check a scenario file. Mistakes are reported before anything runs. */
13
18
  export async function loadScenario(path) {
14
19
  let data;
@@ -28,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
28
33
  if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
29
34
  throw new Error(`${where}: allow_writes must be true or false`);
30
35
  }
36
+ const assumed = doc.assume_read_only;
37
+ if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
38
+ throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
39
+ }
31
40
  const steps = doc.steps.map((raw, i) => {
32
41
  const at = `${where}: step ${i + 1}`;
33
42
  if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
@@ -52,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
52
61
  }
53
62
  throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
54
63
  });
55
- return { allowWrites: doc.allow_writes === true, steps };
64
+ return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
56
65
  }
57
66
  export function stepLabel(step) {
58
67
  if (step.kind === 'list')
@@ -69,7 +78,10 @@ export async function session(target, scenario, options = {}) {
69
78
  // The main process runs with a pinned hash seed, and so does every scope probe:
70
79
  // a probe with a different seed would count ordering variance as a change.
71
80
  const mainTarget = seeded(target, MAIN_SEED);
72
- const conn = await connect(mainTarget, { timeoutMs });
81
+ const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
82
+ // Connecting and listing are read-only: a refusal for being too many is waited
83
+ // out and they're sent again.
84
+ const conn = await connectPatiently(mainTarget, { timeoutMs, waits });
73
85
  try {
74
86
  const era = conn.era;
75
87
  const modern = era === 'modern';
@@ -77,7 +89,7 @@ export async function session(target, scenario, options = {}) {
77
89
  let listening = !modern; // 2025-era notifications arrive on the session itself
78
90
  if (modern && declared) {
79
91
  try {
80
- await conn.client.listen({ toolsListChanged: true });
92
+ await patiently(() => conn.client.listen({ toolsListChanged: true }), { waits, error: tooMany });
81
93
  listening = true;
82
94
  }
83
95
  catch {
@@ -85,41 +97,152 @@ export async function session(target, scenario, options = {}) {
85
97
  }
86
98
  }
87
99
  const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
88
- let current = await menuOf(conn);
100
+ const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
101
+ // A refusal that outlasts the waits, or a write's that isn't sent again, ends
102
+ // the run with one finding, not one per step. `tool`: the write not sent again.
103
+ let limitedAt;
104
+ const limited = (index, why, waitedMs, tool) => {
105
+ limitedAt = index;
106
+ const said = serverWords(why);
107
+ const last = scenario.steps.length;
108
+ const unchecked = index < last ? `Steps ${index}–${last} weren't checked.` : `Step ${index} wasn't checked.`;
109
+ raw.push({
110
+ rule: 'session/rate-limited',
111
+ severity: 'error',
112
+ step: index,
113
+ ...(tool ? { tool } : {}),
114
+ message: tool
115
+ ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
116
+ : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
117
+ fix: tool
118
+ ? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
119
+ : 'Give the run its own server instance, or raise the limit for it.',
120
+ });
121
+ };
122
+ let current = await listPatiently();
89
123
  // Map keeps first-insertion order; set() on a known name updates it in place.
90
124
  const seen = new Map();
91
125
  const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
92
126
  see(current);
93
127
  const baseline = current;
128
+ // Values that differ from one tools/list to the next on their own (a default
129
+ // computed from the clock: PayPal's list_transactions.end_date), by tool and
130
+ // path, with the finding that reported them first. A later change only there
131
+ // has the same cause and points to it (SPEC §25: one root cause, one finding).
132
+ const varying = new Map();
133
+ const learn = (before, after, changes, source) => {
134
+ for (const [key] of variedPaths(before, after, changes))
135
+ if (!varying.has(key))
136
+ varying.set(key, source);
137
+ };
94
138
  // Fresh processes or connections, same credentials, must see the same menu.
95
139
  let connectionCheck;
96
- for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
140
+ for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
97
141
  if (probe.error) {
98
- raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
142
+ raw.push({
143
+ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
144
+ severity: 'info',
145
+ step: 0,
146
+ message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
147
+ fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
148
+ });
99
149
  continue;
100
150
  }
101
151
  const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
102
152
  connectionCheck = f ? 'different' : connectionCheck ?? 'same';
103
153
  if (f) {
104
- raw.push({ ...f, step: 0 });
154
+ const changes = compareMenus(current.tools, probe.tools ?? []);
155
+ raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
156
+ learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
105
157
  break;
106
158
  }
107
159
  }
108
160
  const steps = [];
161
+ const scopeUnchecked = [];
162
+ // processes: 1 turns off every second process or connection, the scope probe too.
163
+ const probes = (options.processes ?? 2) > 1;
164
+ // The step after which toolmenu last opened a second process or connection (0:
165
+ // before the first step). A server that keeps one session per client can end
166
+ // this one when that happens.
167
+ let probedAfter = probes ? 0 : undefined;
168
+ const lost = (why, index) => {
169
+ if (probedAfter === undefined || !SESSION_LOST.test(why))
170
+ return false;
171
+ // The server's own words, not the transport's wrapping of its JSON-RPC error.
172
+ const said = serverWords(why);
173
+ raw.push({
174
+ rule: 'session/session-lost',
175
+ severity: 'error',
176
+ step: index,
177
+ confidence: 'unsure',
178
+ message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
179
+ fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
180
+ });
181
+ return true;
182
+ };
183
+ const callWaited = { ms: 0 };
184
+ const listWaited = { ms: 0 };
185
+ const assumed = new Set(scenario.assumeReadOnly ?? []);
186
+ // Called on the user's word (assume_read_only), by step.
187
+ const onWord = [];
188
+ // The same failure from the same tool at several steps is one finding that lists them.
189
+ const failures = new Map();
190
+ const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
191
+ const key = `${rule}\0${step.tool}\0${text}`;
192
+ const before = failures.get(key);
193
+ if (before)
194
+ before.steps.push(index);
195
+ else {
196
+ const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
197
+ failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
198
+ }
199
+ };
200
+ // Steps whose change touched only values already known to vary on their own.
201
+ const explained = [];
202
+ // --auto's guessed unlocks: tried once, and those whose first call changed nothing.
203
+ const tried = new Set();
204
+ const quiet = new Set();
205
+ const stopped = new Map();
109
206
  for (const [i, step] of scenario.steps.entries()) {
110
207
  const index = i + 1;
208
+ if (step.kind === 'call' && step.tentative) {
209
+ if (quiet.has(step.tool)) {
210
+ stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
211
+ continue;
212
+ }
213
+ }
214
+ const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
215
+ if (firstTry)
216
+ tried.add(step.tool);
111
217
  const label = stepLabel(step);
112
218
  const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
113
219
  const mark = conn.wire.notifications.length;
220
+ let word = false;
221
+ const done = () => {
222
+ if (word)
223
+ record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
224
+ steps.push(record);
225
+ };
114
226
  if (step.kind === 'call') {
115
227
  const tool = current.tools.find((t) => t.name === step.tool);
228
+ const write = tool ? writeSign(tool) : undefined;
229
+ word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
116
230
  if (!tool) {
117
231
  record.status = 'failed';
118
232
  record.reason = 'missing';
119
233
  record.note = `${step.tool} isn't in the menu at this point`;
120
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `Step ${index} calls ${step.tool}, which isn't in the menu at this point.` });
234
+ const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
235
+ const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
236
+ raw.push({
237
+ rule: 'session/step-failed',
238
+ severity: 'error',
239
+ step: index,
240
+ tool: step.tool,
241
+ message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
242
+ fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
243
+ });
121
244
  }
122
- else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
245
+ else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
123
246
  record.status = 'refused';
124
247
  record.reason = 'refused';
125
248
  record.note = 'not marked readOnlyHint';
@@ -128,39 +251,66 @@ export async function session(target, scenario, options = {}) {
128
251
  severity: 'error',
129
252
  step: index,
130
253
  tool: step.tool,
131
- message: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real. Set "allow_writes: true" in the scenario if that's intended.`,
254
+ message: assumed.has(step.tool) && write
255
+ ? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
256
+ : `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
257
+ fix: write || tool.annotations?.readOnlyHint === false
258
+ ? 'Set "allow_writes: true" in the scenario if calling it is intended.'
259
+ : `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
132
260
  });
133
261
  }
262
+ if (word)
263
+ onWord.push({ tool: step.tool, step: index });
134
264
  if (record.status === 'ok')
135
265
  try {
136
- const result = await conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs });
266
+ // Sent again after a refusal only when that repeats nothing: the transport's
267
+ // 429 means the server never ran it; a tool that says it was rate-limited,
268
+ // thrown or as its result, may have done part of the work first, so only a
269
+ // read-only one is asked again. On the server's word, not the user's.
270
+ const readOnly = tool?.annotations?.readOnlyHint === true;
271
+ callWaited.ms = 0;
272
+ const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
273
+ waits,
274
+ waited: callWaited,
275
+ error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
276
+ result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
277
+ });
137
278
  if (result.isError) {
138
- const text = (Array.isArray(result.content) ? result.content : [])
139
- .map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
140
- .join(' ')
141
- .replace(/\s+/g, ' ')
142
- .trim();
279
+ const text = errorText(result);
280
+ if (RATE_LIMITED.test(text)) {
281
+ limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
282
+ record.status = 'failed';
283
+ record.note = 'rate-limited';
284
+ done();
285
+ break;
286
+ }
143
287
  const short = clip(text, 120);
144
288
  record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
145
289
  record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
146
290
  // A setup failure is reported once, for the whole run (session/untested).
147
291
  if (!SETUP_FAILURES.has(record.failure))
148
- raw.push({
149
- rule: 'session/tool-error',
150
- severity: 'warn',
151
- step: index,
152
- tool: step.tool,
153
- message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
154
- });
292
+ fail('session/tool-error', 'warn', index, step, text, record.failure);
155
293
  }
156
294
  }
157
295
  catch (error) {
158
296
  record.status = 'failed';
159
- record.note = error instanceof Error ? error.message.split('\n')[0] : String(error);
297
+ record.note = serverWords(error, 200);
298
+ if (lost(record.note, index)) {
299
+ done();
300
+ break;
301
+ }
302
+ if (tooMany(error)) {
303
+ limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
304
+ record.note = 'rate-limited';
305
+ done();
306
+ break;
307
+ }
160
308
  const code = error.code;
161
309
  record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
310
+ // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
311
+ const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
162
312
  if (!SETUP_FAILURES.has(record.failure))
163
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, tool: step.tool, message: `Step ${index} (${label}) failed: ${record.note}` });
313
+ fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
164
314
  }
165
315
  }
166
316
  else if (step.kind === 'wait_for') {
@@ -174,44 +324,87 @@ export async function session(target, scenario, options = {}) {
174
324
  // have changed the menu, and the change belongs to this step, not the next.
175
325
  let next;
176
326
  try {
177
- next = await menuOf(conn);
327
+ listWaited.ms = 0;
328
+ next = await listPatiently(listWaited);
178
329
  }
179
330
  catch (error) {
180
- const why = error instanceof Error ? error.message.split('\n')[0] : String(error);
331
+ const why = serverWords(error, 200);
181
332
  record.status = 'failed';
182
333
  record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
183
- raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
184
- steps.push(record);
334
+ if (lost(why, index)) {
335
+ done();
336
+ break;
337
+ }
338
+ if (tooMany(error)) {
339
+ limited(index, why, listWaited.ms);
340
+ record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
341
+ done();
342
+ break;
343
+ }
344
+ raw.push({
345
+ rule: 'session/step-failed',
346
+ severity: 'error',
347
+ step: index,
348
+ message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
349
+ fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
350
+ });
351
+ done();
185
352
  continue;
186
353
  }
187
354
  const changes = compareMenus(current.tools, next.tools);
188
355
  if (changes.length) {
189
356
  record.changed = true;
190
- if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
191
- await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
357
+ const paths = variedPaths(current.tools, next.tools, changes);
358
+ const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
359
+ if (onItsOwn) {
360
+ // Same cause as a finding already made: said once, after the steps.
361
+ explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
362
+ const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
363
+ record.note = record.note ? `${record.note}; ${cause}` : cause;
192
364
  }
193
- const origin = step.kind === 'list'
194
- ? 'no tool call in between'
195
- : record.reason === 'missing'
196
- ? 'no tool call made (the tool wasn\'t in the menu)'
197
- : record.reason === 'refused'
198
- ? 'no tool call made (refused)'
199
- : record.status === 'failed'
200
- ? 'the call failed, but the menu changed: the server may have applied it anyway'
201
- : undefined;
202
- raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
203
- if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
204
- raw.push({
205
- rule: 'session/unannounced',
206
- severity: 'warn',
207
- step: index,
208
- message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
209
- });
210
- }
211
- if (step.kind !== 'list') {
212
- const probe = await probeMenu(mainTarget, timeoutMs);
213
- record.scope = scopeOf(changes, next.tools, probe.tools, target.kind);
214
- raw.push(...scopeFindings(record.scope, modern, index));
365
+ else {
366
+ if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
367
+ await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
368
+ }
369
+ const origin = step.kind === 'list'
370
+ ? 'no tool call in between'
371
+ : record.reason === 'missing'
372
+ ? 'no tool call made (the tool wasn\'t in the menu)'
373
+ : record.reason === 'refused'
374
+ ? 'no tool call made (refused)'
375
+ : record.status === 'failed'
376
+ ? 'the call failed, but the menu changed: the server may have applied it anyway'
377
+ : undefined;
378
+ raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
379
+ // With no call in between, whatever changed varies on its own: a later
380
+ // change only there has the same cause.
381
+ if (step.kind === 'list' || record.reason !== undefined)
382
+ learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
383
+ if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
384
+ raw.push({
385
+ rule: 'session/unannounced',
386
+ severity: 'warn',
387
+ step: index,
388
+ message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
389
+ fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
390
+ });
391
+ }
392
+ if (step.kind !== 'list' && probes) {
393
+ // The main process is still running here: a server that holds a file or
394
+ // a port can't start a second copy. That leaves the scope unknown, not
395
+ // the run failed.
396
+ const probeWaited = { ms: 0 };
397
+ try {
398
+ probedAfter = index;
399
+ const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
400
+ record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
401
+ raw.push(...scopeFindings(record.scope, modern, index));
402
+ }
403
+ catch (error) {
404
+ record.scope = 'unclear';
405
+ scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
406
+ }
407
+ }
215
408
  }
216
409
  current = next;
217
410
  see(next);
@@ -219,9 +412,32 @@ export async function session(target, scenario, options = {}) {
219
412
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
220
413
  record.tools = current.tools.length;
221
414
  record.tokens = current.totalTokens;
222
- steps.push(record);
415
+ if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
416
+ quiet.add(step.tool);
417
+ done();
223
418
  }
419
+ raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
420
+ if (explained.length)
421
+ raw.push(knownVariance(explained));
422
+ if (onWord.length)
423
+ raw.push(onTheirWord(onWord, options.auto !== undefined));
224
424
  raw.push(...untested(steps, target));
425
+ // --auto means "find what you can": an unlock it skipped is a gap too.
426
+ const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
427
+ // An unlock --auto stopped trying isn't charged for the values it then left out.
428
+ const covered = baseline.tools.filter((t) => !stopped.has(t.name));
429
+ raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
430
+ if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
431
+ raw.push(nothingCalled(options.auto));
432
+ if (scopeUnchecked.length) {
433
+ const which = scopeUnchecked.map((s) => s.step);
434
+ raw.push({
435
+ rule: 'session/scope-unchecked',
436
+ severity: 'info',
437
+ step: which[0],
438
+ message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} while the session ran, so whether the change${which.length === 1 ? ` at step ${which[0]}` : `s at steps ${which.join(', ')}`} reach${which.length === 1 ? 'es' : ''} a fresh ${target.kind === 'stdio' ? 'process' : 'connection'} wasn't checked: ${scopeUnchecked[0].why}`,
439
+ });
440
+ }
225
441
  return {
226
442
  scenario: options.scenarioName ?? 'scenario',
227
443
  server: baseline.server,
@@ -231,7 +447,8 @@ export async function session(target, scenario, options = {}) {
231
447
  final: { tools: current.tools.length, tokens: current.totalTokens },
232
448
  connectionCheck,
233
449
  steps,
234
- findings: settle(mergeSideEffects(raw), options),
450
+ findings: settle(mergeRepeats(raw), options),
451
+ ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
235
452
  union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
236
453
  };
237
454
  }
@@ -254,19 +471,311 @@ function untested(steps, target) {
254
471
  for (const s of setup)
255
472
  byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
256
473
  const all = setup.length === calls.length;
257
- const how = target.kind === 'http' ? 'real credentials (--header "Authorization: …")' : 'real credentials (--env KEY=…)';
474
+ // One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
475
+ const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
258
476
  return [
259
477
  {
260
478
  rule: 'session/untested',
261
479
  severity: all ? 'error' : 'warn',
262
- message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.${byClass.has('auth') || byClass.has('not-found') ? ` Run with ${how}.` : ''}`,
480
+ // Classified from the error's words: a guess, if a good one.
481
+ confidence: 'unsure',
482
+ message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
263
483
  detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
484
+ fix,
264
485
  },
265
486
  ];
266
487
  }
488
+ /**
489
+ * Unlocks whose schema lists every value (an enum), and how many of those values
490
+ * the run got through. The session sees, and --union-out keeps, only the tools
491
+ * behind the values it unlocked: GitHub's enable_toolset has 19, the first starter
492
+ * unlocked 2, and 72 of 81 tools never reached the baseline.
493
+ */
494
+ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], stoppedAt) {
495
+ const out = [];
496
+ // A run a rate limit stopped (session/rate-limited says so) is charged only for
497
+ // values its scenario leaves out, not for ones in steps it never got to.
498
+ const unrun = stoppedAt === undefined ? [] : scenario.steps.map((step, i) => ({ step, index: i + 1 })).filter((s) => s.index >= stoppedAt);
499
+ for (const u of unlockers(menu)) {
500
+ if (!u.param || u.values.length === 0)
501
+ continue;
502
+ const reached = new Set();
503
+ let last;
504
+ const add = (arg) => {
505
+ for (const v of Array.isArray(arg) ? arg : [arg])
506
+ if (v !== undefined)
507
+ reached.add(canonical(v));
508
+ };
509
+ for (const record of steps) {
510
+ const step = scenario.steps[record.index - 1];
511
+ if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure)
512
+ continue;
513
+ last = record.index;
514
+ add(step.args[u.param]);
515
+ }
516
+ for (const { step } of unrun)
517
+ if (step.kind === 'call' && step.tool === u.tool.name)
518
+ add(step.args[u.param]);
519
+ const missing = u.values.filter((v) => !reached.has(canonical(v)));
520
+ // Never called: a scenario about something else, unless it's building the baseline.
521
+ if (missing.length === 0 || (reached.size === 0 && !baselineWanted))
522
+ continue;
523
+ const got = u.values.length - missing.length;
524
+ const skip = skipped.find((s) => s.tool === u.tool.name)?.reason;
525
+ const why = skip === 'open world'
526
+ ? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
527
+ : skip === 'not read-only'
528
+ ? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
529
+ : skip === 'unmarked'
530
+ ? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
531
+ : '';
532
+ out.push({
533
+ rule: 'session/unlock-coverage',
534
+ severity: 'warn',
535
+ tool: u.tool.name,
536
+ // "Looks like it unlocks": from its name, parameter and description.
537
+ confidence: 'unsure',
538
+ ...(last !== undefined ? { step: last } : {}),
539
+ message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
540
+ detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
541
+ fix: skip === 'open world'
542
+ ? 'Rerun with --open-world.'
543
+ : skip === 'unmarked'
544
+ ? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
545
+ : skip === 'not read-only'
546
+ ? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
547
+ : 'Unlock every value: session --init puts each one in the starter.',
548
+ });
549
+ }
550
+ return out;
551
+ }
552
+ /**
553
+ * --auto that called nothing: the run only listed the menu, so a clean result
554
+ * says nothing about what the tools do to it. Why each tool was left out, and the
555
+ * flag that reaches the most of them.
556
+ */
557
+ function nothingCalled(auto) {
558
+ const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
559
+ const parts = [];
560
+ const openWorld = by('open world');
561
+ const needs = by('needs values');
562
+ const unmarked = by('unmarked');
563
+ const writes = by('not read-only');
564
+ if (needs.length)
565
+ parts.push(`${needs.length} need values the schema doesn't give`);
566
+ if (unmarked.length)
567
+ parts.push(`${unmarked.length} aren't marked readOnlyHint`);
568
+ if (openWorld.length)
569
+ parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
570
+ if (writes.length)
571
+ parts.push(`${writes.length} marked or named as writes`);
572
+ const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
573
+ const params = neededValues(auto.skipped);
574
+ const fix = autoNextStep(auto);
575
+ return {
576
+ rule: 'session/nothing-called',
577
+ severity: 'warn',
578
+ message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
579
+ detail: [
580
+ ...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
581
+ ...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
582
+ ...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
583
+ ],
584
+ ...(fix ? { fix } : {}),
585
+ };
586
+ }
587
+ /**
588
+ * The same error text from different tools is one finding (two tools of a feature
589
+ * that "isn't set up on this deployment"). An empty text says nothing in common.
590
+ */
591
+ function sameError(failures) {
592
+ const out = [];
593
+ const byText = new Map();
594
+ for (const f of failures) {
595
+ const key = `${f.rule}\0${f.text}`;
596
+ const first = f.text ? byText.get(key) : undefined;
597
+ if (!first) {
598
+ out.push({ ...f, steps: [...f.steps] });
599
+ if (f.text)
600
+ byText.set(key, out[out.length - 1]);
601
+ continue;
602
+ }
603
+ first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
604
+ first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
605
+ }
606
+ return out;
607
+ }
608
+ // The server says the feature isn't there, or isn't for this account: no argument
609
+ // fixes that. A guess from its words.
610
+ const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\bpermission\b|\bforbidden\b|\bnot allowed\b|\brole\b/i;
611
+ /** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
612
+ function stepsText(steps) {
613
+ if (steps.length === 1)
614
+ return `step ${steps[0]}`;
615
+ const runs = [];
616
+ for (let i = 0; i < steps.length;) {
617
+ let j = i;
618
+ while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
619
+ j++;
620
+ if (j - i >= 2)
621
+ runs.push(`${steps[i]}–${steps[j]}`);
622
+ else
623
+ for (let k = i; k <= j; k++)
624
+ runs.push(String(steps[k]));
625
+ i = j + 1;
626
+ }
627
+ return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
628
+ }
629
+ /**
630
+ * The same error from the same tool is one finding that names every step it
631
+ * happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
632
+ * words, the call that got it, and what to try.
633
+ */
634
+ function failureFindings(failures, auto) {
635
+ return failures.map((f) => {
636
+ const many = f.steps.length > 1;
637
+ const calls = [f, ...(f.others ?? [])];
638
+ const tools = [...new Set(calls.map((c) => c.tool))];
639
+ const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
640
+ const said = f.text ? quotedSentence(clip(f.text, 160)) : 'no error text.';
641
+ const firstArg = Object.keys(f.args)[0];
642
+ const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
643
+ // No argument can fix a call to a tool that takes none, or a feature the server
644
+ // says isn't there. From the server's words: a guess.
645
+ const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
646
+ const them = tools.length === 1 ? 'its' : 'their';
647
+ const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
648
+ const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
649
+ const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
650
+ const fix = f.timedOutMs
651
+ ? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
652
+ : !argsCan
653
+ ? undefined
654
+ : unavailable
655
+ ? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
656
+ : noParams
657
+ ? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
658
+ : auto?.withValues?.includes(f.tool)
659
+ ? `Check the values you gave ${f.tool} (--value) against what the server says above.`
660
+ : auto
661
+ ? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
662
+ : `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
663
+ const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
664
+ const detail = calls.slice(0, 8).map((c) => {
665
+ const call = `${c.tool} ${JSON.stringify(c.args)}`;
666
+ return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
667
+ });
668
+ return {
669
+ rule: f.rule,
670
+ severity: f.severity,
671
+ step: f.steps[0],
672
+ ...(many ? { steps: f.steps } : {}),
673
+ tool: f.tool,
674
+ ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
675
+ message: f.rule === 'session/tool-error'
676
+ ? tools.length > 1
677
+ ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
678
+ : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
679
+ : tools.length > 1
680
+ ? `${lead}Calling ${who} failed the same way: ${said}`
681
+ : `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
682
+ detail: [...new Set(detail)],
683
+ ...(fix ? { fix } : {}),
684
+ };
685
+ });
686
+ }
687
+ /**
688
+ * Changes that touched only values already seen to vary on their own: one info
689
+ * finding for the run, pointing to the one that reported the cause.
690
+ */
691
+ function knownVariance(explained) {
692
+ const steps = explained.map((e) => e.step);
693
+ const src = explained[0].source;
694
+ const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
695
+ // The values at the first such step; the rest only repeat the pattern.
696
+ const seen = explained[0].seen;
697
+ return {
698
+ rule: 'session/known-variance',
699
+ severity: 'info',
700
+ step: steps[0],
701
+ message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
702
+ detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
703
+ fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
704
+ };
705
+ }
706
+ /** Tools called because the user said they only read: said once, loudly. */
707
+ function onTheirWord(calls, auto) {
708
+ const tools = [...new Set(calls.map((c) => c.tool))];
709
+ return {
710
+ rule: 'session/assumed-read-only',
711
+ severity: 'info',
712
+ step: calls[0].step,
713
+ message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
714
+ detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
715
+ fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
716
+ };
717
+ }
718
+ /**
719
+ * Every leaf value that differs between the old and new definition of each edited
720
+ * tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
721
+ */
722
+ function variedPaths(before, after, changes) {
723
+ const out = new Map();
724
+ const old = new Map(before.map((t) => [t.name, t]));
725
+ for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
726
+ const a = old.get(name);
727
+ const b = after.find((t) => t.name === name);
728
+ if (!a || !b)
729
+ continue;
730
+ for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
731
+ out.set(`${name}\0${path}`, d);
732
+ }
733
+ return out;
734
+ }
735
+ function leafDiffs(a, b, path = '', out = new Map()) {
736
+ if (canonical(a) === canonical(b))
737
+ return out;
738
+ const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
739
+ if (obj(a) && obj(b)) {
740
+ for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
741
+ leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
742
+ }
743
+ else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
744
+ a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
745
+ }
746
+ else
747
+ out.set(path, { before: a, after: b });
748
+ return out;
749
+ }
750
+ function show(value) {
751
+ if (value === undefined)
752
+ return '(absent)';
753
+ const text = JSON.stringify(value);
754
+ return text.length > 60 ? text.slice(0, 57) + '…' : text;
755
+ }
756
+ const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
757
+ /** The next step for a menu that differs between processes, from what differs. */
758
+ function varianceFix(main, other, changes) {
759
+ const diffs = [...variedPaths(main, other, changes)];
760
+ if (diffs.length === 0)
761
+ return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
762
+ const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
763
+ if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
764
+ return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
765
+ }
766
+ return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
767
+ }
267
768
  function serverOf(c) {
268
769
  return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
269
770
  }
771
+ /** A tool result's text, whitespace collapsed. */
772
+ function errorText(result) {
773
+ return (Array.isArray(result.content) ? result.content : [])
774
+ .map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
775
+ .join(' ')
776
+ .replace(/\s+/g, ' ')
777
+ .trim();
778
+ }
270
779
  function waitFor(done, ms) {
271
780
  return new Promise((resolve) => {
272
781
  if (done())
@@ -284,6 +793,7 @@ function waitFor(done, ms) {
284
793
  }, 20);
285
794
  });
286
795
  }
796
+ const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
287
797
  const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
288
798
  /** Turn a menu change into findings, with the cost of each. */
289
799
  export function changeFindings(before, after, changes, step, origin) {
@@ -311,8 +821,10 @@ export function changeFindings(before, after, changes, step, origin) {
311
821
  rule: 'session/append',
312
822
  severity: 'warn',
313
823
  step,
314
- message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}). The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.`,
824
+ message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
315
825
  detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
826
+ each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
827
+ fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
316
828
  });
317
829
  }
318
830
  else {
@@ -323,31 +835,63 @@ export function changeFindings(before, after, changes, step, origin) {
323
835
  step,
324
836
  message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
325
837
  detail: [...cost, ...why],
838
+ fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
326
839
  });
327
840
  }
328
841
  }
329
842
  const moved = changes.filter((c) => c.kind === 'moved');
330
843
  if (moved.length) {
331
- out.push({ rule: 'session/reorder', severity: 'error', step, message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}).`, detail: [...cost, ...why] });
844
+ out.push({
845
+ rule: 'session/reorder',
846
+ severity: 'error',
847
+ step,
848
+ message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
849
+ detail: [...cost, ...why],
850
+ fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
851
+ });
332
852
  }
333
853
  const removed = changes.filter((c) => c.kind === 'removed');
334
854
  if (removed.length) {
335
- out.push({ rule: 'session/remove', severity: 'error', step, message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session.`, detail: [...cost, ...why] });
855
+ out.push({
856
+ rule: 'session/remove',
857
+ severity: 'error',
858
+ step,
859
+ message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
860
+ detail: [...cost, ...why],
861
+ fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
862
+ });
336
863
  }
337
864
  const edits = new Map();
338
865
  for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
339
866
  edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
867
+ const old = new Map(before.map((t) => [t.name, t]));
340
868
  for (const [tool, fields] of edits) {
341
- out.push({ rule: 'session/edit', severity: 'error', step, tool, message: `${tool}: ${fields.join(', ')} changed mid-session.`, detail: [...cost, ...why] });
869
+ const a = old.get(tool);
870
+ const b = after.find((t) => t.name === tool);
871
+ // What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
872
+ const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
873
+ out.push({
874
+ rule: 'session/edit',
875
+ severity: 'error',
876
+ step,
877
+ tool,
878
+ message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
879
+ detail: [...seen, ...cost, ...why],
880
+ fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
881
+ });
342
882
  }
343
883
  return out;
344
884
  }
345
885
  /**
346
886
  * Does a fresh connection see this step's changes? All of them: the server's tool
347
887
  * set changed. None: the change belongs to this connection (or, on stdio, this
348
- * process). Reorders alone can't tell, and a mix is unclear.
888
+ * process). Reorders alone can't tell, and a mix is unclear. So can't a step that
889
+ * takes the menu back to the session's `baseline` while fresh ones still serve it:
890
+ * a fresh process starts there anyway (an unlock undone reads as "seen" by it).
349
891
  */
350
- export function scopeOf(changes, next, probe, transport) {
892
+ export function scopeOf(changes, next, probe, transport, baseline) {
893
+ if (baseline && compareMenus(baseline, next).length === 0 && compareMenus(baseline, probe).length === 0)
894
+ return 'unclear';
351
895
  const probeByName = new Map(probe.map((t) => [t.name, t]));
352
896
  const nextByName = new Map(next.map((t) => [t.name, t]));
353
897
  const verdicts = new Set();
@@ -379,6 +923,7 @@ function scopeFindings(scope, modern, step) {
379
923
  message: modern
380
924
  ? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
381
925
  : `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
926
+ fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
382
927
  },
383
928
  ];
384
929
  }
@@ -389,20 +934,41 @@ function scopeFindings(scope, modern, step) {
389
934
  severity: 'warn',
390
935
  step,
391
936
  message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
937
+ fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
392
938
  },
393
939
  ];
394
940
  }
395
941
  return [];
396
942
  }
397
- /** stdio's side-effect warning is the same for every step: say it once, name the steps. */
398
- function mergeSideEffects(raw) {
399
- const side = raw.filter((f) => f.rule === 'session/side-effect');
400
- if (side.length < 2)
401
- return raw;
402
- const steps = side.map((f) => f.step);
403
- return raw
404
- .filter((f) => f.rule !== 'session/side-effect' || f === side[0])
405
- .map((f) => (f === side[0] ? { ...f, message: `Steps ${steps.join(', ')}: ${f.message}` } : f));
943
+ /**
944
+ * Findings that say the same thing at every step are said once, naming the steps:
945
+ * stdio's side-effect warning, and an unlock per domain, where each value appends
946
+ * tools for this connection only (9 steps made 18 near-identical blocks). What
947
+ * differs per step, the tools and their cost, goes in the detail. Mid-inserts,
948
+ * removals and edits stay per step: each is its own change.
949
+ */
950
+ const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
951
+ function mergeRepeats(raw) {
952
+ let out = raw;
953
+ for (const rule of REPEATS) {
954
+ const same = out.filter((f) => f.rule === rule);
955
+ if (same.length < 2)
956
+ continue;
957
+ const steps = same.map((f) => f.step);
958
+ const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
959
+ let message = `${lead}: ${same[0].message}`;
960
+ let detail = same[0].detail;
961
+ if (rule === 'session/append') {
962
+ const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
963
+ message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
964
+ const each = same.map((f) => f.each).filter(Boolean);
965
+ const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
966
+ detail = [...each, ...why];
967
+ }
968
+ const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
969
+ out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
970
+ }
971
+ return out;
406
972
  }
407
973
  function describe(changes) {
408
974
  return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
@@ -411,7 +977,7 @@ function settle(raw, options) {
411
977
  const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
412
978
  return raw
413
979
  .filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
414
- .map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
980
+ .map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
415
981
  .sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
416
982
  }
417
983
  /*
@@ -424,10 +990,35 @@ function settle(raw, options) {
424
990
  // A parameter that names what to unlock. Strong names say so alone; weak ones
425
991
  // (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
426
992
  // description to back them up.
427
- const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?|namespaces?)$/i;
428
- const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?)$/i;
993
+ // `namespace` is weak too: Kubernetes and Pinecone take one on every read.
994
+ const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
995
+ const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
429
996
  const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
430
- const UNLOCK_DESC = /(?:unlock|enable|activate|load|expose|add)s?[^.]{0,60}tools?|more tools|toolsets?|capabilit(?:y|ies)|unlock/i;
997
+ // A description about loading tools: "Adds the audit tools", "enables more
998
+ // capabilities", "unlocks…". Whole words: "download" isn't "load", and a device's
999
+ // capabilities alone aren't tools.
1000
+ const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
1001
+ // Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
1002
+ // "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
1003
+ const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
1004
+ /**
1005
+ * Does the tool say that `param` unlocks tools? Its name does, or a sentence of
1006
+ * its description that says it unlocks tools and doesn't pin that on another
1007
+ * parameter ("A query match also enables the tools…" is about `query`, not the
1008
+ * `domain` filter next to it).
1009
+ */
1010
+ function claimsUnlock(tool, param) {
1011
+ if (UNLOCK_NAME.test(tool.name))
1012
+ return true;
1013
+ const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
1014
+ const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
1015
+ const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
1016
+ return (tool.description ?? '')
1017
+ .split(/(?<=[.!?])\s+/)
1018
+ .filter((s) => UNLOCK_SAYS.test(s))
1019
+ .some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
1020
+ }
1021
+ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
431
1022
  /**
432
1023
  * At most `max` characters, cut by code point at a word boundary, never inside a
433
1024
  * character. Drops U+FFFD left over from a server that cut its own message by bytes.
@@ -441,10 +1032,15 @@ export function clip(text, max) {
441
1032
  return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
442
1033
  }
443
1034
  function isWriteLike(t) {
1035
+ // Named as an unlock: changing the menu is its job, whatever its hints say
1036
+ // (toolception marks enable_toolset destructive). It's still only suggested,
1037
+ // never called without allow_writes.
1038
+ if (UNLOCK_NAME.test(t.name))
1039
+ return false;
444
1040
  if (t.annotations?.destructiveHint === true)
445
1041
  return true;
446
1042
  const verb = verbOf(t.name);
447
- return verb !== undefined && WRITE_VERBS.has(verb) && !UNLOCK_NAME.test(t.name);
1043
+ return verb !== undefined && WRITE_VERBS.has(verb);
448
1044
  }
449
1045
  function enumOf(schema) {
450
1046
  if (!schema)
@@ -459,8 +1055,24 @@ export function unlockers(tools) {
459
1055
  const found = [];
460
1056
  for (const tool of tools) {
461
1057
  const props = tool.inputSchema?.properties ?? {};
462
- const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(tool.description ?? '');
463
- const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ?? (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined);
1058
+ const description = tool.description ?? '';
1059
+ // A lookup names toolsets without changing them (GitHub's get_toolset_tools,
1060
+ // toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
1061
+ const verb = verbOf(tool.name);
1062
+ const lookup = verb !== undefined && LOOKUPS.has(verb);
1063
+ if (lookup && !UNLOCK_SAYS.test(description))
1064
+ continue;
1065
+ const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
1066
+ const required = tool.inputSchema?.required ?? [];
1067
+ const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ??
1068
+ (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined) ??
1069
+ // Named like an unlock, one required parameter: that's what it takes
1070
+ // (toolception's enable_toolset { name }).
1071
+ (UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
1072
+ const claims = claimsUnlock(tool, param);
1073
+ // A lookup that says a *different* parameter unlocks: its enum is a filter.
1074
+ if (lookup && !claims)
1075
+ continue;
464
1076
  let score = 0;
465
1077
  if (param)
466
1078
  score += enumOf(props[param]).length ? 3 : 2;
@@ -469,25 +1081,109 @@ export function unlockers(tools) {
469
1081
  if (UNLOCK_DESC.test(tool.description ?? ''))
470
1082
  score += 1;
471
1083
  if (score >= 2)
472
- found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
1084
+ found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
473
1085
  }
474
1086
  return found.sort((a, b) => b.score - a.score);
475
1087
  }
1088
+ /**
1089
+ * Unlocks whose values the schema doesn't give (no enum), each with a read-only
1090
+ * lister that should: no required arguments, a lookup verb, and a noun in common
1091
+ * with the unlock (toolception: enable_toolset { name } ← list_toolsets).
1092
+ */
1093
+ export function unlockListers(tools) {
1094
+ const out = [];
1095
+ const nounSet = (name) => new Set(words(name).map(singular).filter((w) => !VERBS.has(w)));
1096
+ for (const u of unlockers(tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)))) {
1097
+ if (!u.param || u.values.length)
1098
+ continue;
1099
+ const mine = new Set([...nounSet(u.tool.name), ...nounSet(u.param)]);
1100
+ const lister = tools.find((t) => {
1101
+ if (t === u.tool || t.annotations?.readOnlyHint !== true || (t.inputSchema?.required ?? []).length)
1102
+ return false;
1103
+ const verb = verbOf(t.name);
1104
+ return !!verb && LOOKUPS.has(verb) && [...nounSet(t.name)].some((n) => mine.has(n));
1105
+ });
1106
+ if (lister)
1107
+ out.push({ unlock: u.tool.name, lister: lister.name });
1108
+ }
1109
+ return out;
1110
+ }
1111
+ /**
1112
+ * The values a listing offers, from its JSON (structuredContent, or text that
1113
+ * starts with JSON): the first list of strings, or of objects with a key, id,
1114
+ * slug or name, each item's own field first (toolception's `key: "quotes"`, not
1115
+ * the display name "Quotes" nested under it).
1116
+ */
1117
+ export function valuesFromListing(result) {
1118
+ const roots = [];
1119
+ if (result.structuredContent)
1120
+ roots.push(result.structuredContent);
1121
+ for (const part of Array.isArray(result.content) ? result.content : []) {
1122
+ const text = part.text;
1123
+ if (typeof text === 'string') {
1124
+ const value = leadingJson(text);
1125
+ if (value !== undefined)
1126
+ roots.push(value);
1127
+ }
1128
+ }
1129
+ const idOf = (item) => {
1130
+ if (typeof item === 'string')
1131
+ return item;
1132
+ if (!item || typeof item !== 'object')
1133
+ return undefined;
1134
+ const o = item;
1135
+ for (const k of ['key', 'id', 'slug', 'name'])
1136
+ if (typeof o[k] === 'string' && o[k])
1137
+ return o[k];
1138
+ return undefined;
1139
+ };
1140
+ const walk = (v, depth) => {
1141
+ if (depth > 5 || !v || typeof v !== 'object')
1142
+ return undefined;
1143
+ if (Array.isArray(v)) {
1144
+ const ids = v.map(idOf).filter((x) => !!x);
1145
+ if (ids.length && ids.length === v.length)
1146
+ return [...new Set(ids)];
1147
+ for (const x of v) {
1148
+ const found = walk(x, depth + 1);
1149
+ if (found)
1150
+ return found;
1151
+ }
1152
+ return undefined;
1153
+ }
1154
+ for (const x of Object.values(v)) {
1155
+ const found = walk(x, depth + 1);
1156
+ if (found)
1157
+ return found;
1158
+ }
1159
+ return undefined;
1160
+ };
1161
+ for (const r of roots) {
1162
+ const found = walk(r, 0);
1163
+ if (found)
1164
+ return found;
1165
+ }
1166
+ return [];
1167
+ }
476
1168
  /** A YAML value for an unlock parameter: an array if the schema takes one. */
477
1169
  function unlockValue(u, value) {
478
1170
  const schema = u.tool.inputSchema?.properties?.[u.param];
479
1171
  return schema?.type === 'array' || schema?.items ? `[${JSON.stringify(value)}]` : JSON.stringify(value);
480
1172
  }
481
- export function starterScenario(menu) {
1173
+ /** At most this many values of one unlock in a scenario; the rest are named in a comment. */
1174
+ export const MAX_UNLOCKS = 50;
1175
+ export function starterScenario(menu, options = {}) {
482
1176
  const readOnly = menu.tools.filter((t) => t.annotations?.readOnlyHint === true);
483
1177
  const required = (t) => t.inputSchema?.required ?? [];
484
- const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)));
1178
+ const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t))).map((u) => u.values.length === 0 && u.param && options.values?.[u.tool.name]?.length ? { ...u, values: options.values[u.tool.name] } : u);
485
1179
  const unlocking = new Set(found.filter((u) => u.param).map((u) => u.tool));
486
1180
  const ready = readOnly.filter((t) => required(t).length === 0 && !unlocking.has(t)).slice(0, 12);
487
1181
  const needsArgs = readOnly.filter((t) => required(t).length > 0);
488
- // The best candidate with a real value runs for real: two unlocks, then a repeat.
1182
+ // The best candidate with real values runs for real: every value, so the tools
1183
+ // behind each are seen (a session baseline holds only what was unlocked), then a repeat.
489
1184
  const live = found.find((u) => u.values.length > 0 && u.tool.annotations?.readOnlyHint === true && required(u.tool).every((p) => p === u.param));
490
- const changers = found.filter((u) => u !== live).map((u) => u.tool);
1185
+ const others = found.filter((u) => u !== live);
1186
+ const changers = others.map((u) => u.tool);
491
1187
  const example = (t, p) => {
492
1188
  const schema = t.inputSchema?.properties?.[p];
493
1189
  const u = found.find((x) => x.tool === t && x.param === p);
@@ -520,18 +1216,28 @@ export function starterScenario(menu) {
520
1216
  for (const t of ready)
521
1217
  lines.push(` - call: ${t.name}`);
522
1218
  if (live) {
523
- const [first, second] = live.values;
524
- lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock, list, and unlock again:`);
525
- for (const v of second === undefined ? [first] : [first, second]) {
1219
+ const shown = live.values.slice(0, MAX_UNLOCKS);
1220
+ lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock every ${live.param} (${live.values.length}), listing after each, then the first again:`);
1221
+ for (const v of shown)
526
1222
  lines.push(` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, v)} }`, ' - list');
1223
+ if (live.values.length > shown.length) {
1224
+ lines.push(` # …and ${live.values.length - shown.length} more: ${live.values.slice(shown.length).map((v) => JSON.stringify(v)).join(', ')}`);
527
1225
  }
528
- lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, first)} }`, ' - list');
1226
+ lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, shown[0])} }`, ' - list');
529
1227
  }
530
1228
  if (changers.length) {
531
1229
  lines.push('', ` # ${live ? 'These also' : 'These'} look like they could change the menu. Try them:`);
532
- for (const t of changers.slice(0, 8)) {
1230
+ for (const u of others.slice(0, 8)) {
1231
+ const t = u.tool;
533
1232
  if (t.annotations?.readOnlyHint !== true)
534
1233
  lines.push(` # (not marked readOnlyHint: needs allow_writes: true)`);
1234
+ // Every known value, as with a live unlock: each may bring different tools.
1235
+ const values = u.param && required(t).every((p) => p === u.param) ? u.values.slice(0, MAX_UNLOCKS) : [];
1236
+ if (values.length) {
1237
+ for (const v of values)
1238
+ lines.push(` # - call: ${t.name}`, ` # args: { ${u.param}: ${unlockValue(u, v)} }`, ' # - list');
1239
+ continue;
1240
+ }
535
1241
  lines.push(` # - call: ${t.name}`);
536
1242
  if (argNames(t).length)
537
1243
  lines.push(` # args: ${placeholder(t)}`);