toolmenu 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/session.js CHANGED
@@ -1,14 +1,17 @@
1
1
  import { readFile } from 'node:fs/promises';
2
2
  import { parse } from 'yaml';
3
- import { cacheBreak, compareMenus } from './compare.js';
4
- import { connect, listTools } from './connect.js';
3
+ import { cacheBreak, canonical, compareMenus } from './compare.js';
4
+ import { listTools } from './connect.js';
5
5
  import { buildMenu, toolDefinition } from './menu.js';
6
- import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
6
+ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
7
  import { varianceFinding } from './rules/determinism.js';
8
- import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
8
+ import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_ADVICE, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, tooMany, waitedFor, whyNot } from './failures.js';
9
9
  import { SEVERITY_RANK } from './types.js';
10
- import { verbOf, WRITE_VERBS } from './words.js';
10
+ import { leadingJson } from './catalog.js';
11
+ import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
11
12
  const LIST_CHANGED = 'notifications/tools/list_changed';
13
+ /** A server saying the session is gone: "Session not found or expired" (toolception). */
14
+ const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
12
15
  /** Read and check a scenario file. Mistakes are reported before anything runs. */
13
16
  export async function loadScenario(path) {
14
17
  let data;
@@ -69,7 +72,10 @@ export async function session(target, scenario, options = {}) {
69
72
  // The main process runs with a pinned hash seed, and so does every scope probe:
70
73
  // a probe with a different seed would count ordering variance as a change.
71
74
  const mainTarget = seeded(target, MAIN_SEED);
72
- const conn = await connect(mainTarget, { timeoutMs });
75
+ const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
76
+ // Connecting and listing are read-only: a refusal for being too many is waited
77
+ // out and they're sent again.
78
+ const conn = await connectPatiently(mainTarget, { timeoutMs, waits });
73
79
  try {
74
80
  const era = conn.era;
75
81
  const modern = era === 'modern';
@@ -77,7 +83,7 @@ export async function session(target, scenario, options = {}) {
77
83
  let listening = !modern; // 2025-era notifications arrive on the session itself
78
84
  if (modern && declared) {
79
85
  try {
80
- await conn.client.listen({ toolsListChanged: true });
86
+ await patiently(() => conn.client.listen({ toolsListChanged: true }), { waits, error: tooMany });
81
87
  listening = true;
82
88
  }
83
89
  catch {
@@ -85,7 +91,25 @@ export async function session(target, scenario, options = {}) {
85
91
  }
86
92
  }
87
93
  const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
88
- let current = await menuOf(conn);
94
+ const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
95
+ // A refusal that outlasts the waits, or a write's that isn't sent again, ends
96
+ // the run with one finding, not one per step. `tool`: the write not sent again.
97
+ let limitedAt;
98
+ const limited = (index, why, waitedMs, tool) => {
99
+ limitedAt = index;
100
+ const said = serverWords(why);
101
+ const last = scenario.steps.length;
102
+ const unchecked = index < last ? `Steps ${index}–${last} weren't checked.` : `Step ${index} wasn't checked.`;
103
+ raw.push({
104
+ rule: 'session/rate-limited',
105
+ severity: 'error',
106
+ step: index,
107
+ message: tool
108
+ ? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked} If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota.`
109
+ : `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} ${RATE_LIMIT_ADVICE}`,
110
+ });
111
+ };
112
+ let current = await listPatiently();
89
113
  // Map keeps first-insertion order; set() on a known name updates it in place.
90
114
  const seen = new Map();
91
115
  const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
@@ -93,9 +117,9 @@ export async function session(target, scenario, options = {}) {
93
117
  const baseline = current;
94
118
  // Fresh processes or connections, same credentials, must see the same menu.
95
119
  let connectionCheck;
96
- for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
120
+ for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
97
121
  if (probe.error) {
98
- raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error.split('\n')[0]}` });
122
+ raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}` });
99
123
  continue;
100
124
  }
101
125
  const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
@@ -106,6 +130,28 @@ export async function session(target, scenario, options = {}) {
106
130
  }
107
131
  }
108
132
  const steps = [];
133
+ const scopeUnchecked = [];
134
+ // processes: 1 turns off every second process or connection, the scope probe too.
135
+ const probes = (options.processes ?? 2) > 1;
136
+ // The step after which toolmenu last opened a second process or connection (0:
137
+ // before the first step). A server that keeps one session per client can end
138
+ // this one when that happens.
139
+ let probedAfter = probes ? 0 : undefined;
140
+ const lost = (why, index) => {
141
+ if (probedAfter === undefined || !SESSION_LOST.test(why))
142
+ return false;
143
+ // The server's own words, not the transport's wrapping of its JSON-RPC error.
144
+ const said = serverWords(why);
145
+ raw.push({
146
+ rule: 'session/session-lost',
147
+ severity: 'error',
148
+ step: index,
149
+ message: `The server ended this session (“${said}”) after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that, so the rest of the scenario didn't run. Run with --processes 1: toolmenu then opens no second one.`,
150
+ });
151
+ return true;
152
+ };
153
+ const callWaited = { ms: 0 };
154
+ const listWaited = { ms: 0 };
109
155
  for (const [i, step] of scenario.steps.entries()) {
110
156
  const index = i + 1;
111
157
  const label = stepLabel(step);
@@ -133,13 +179,27 @@ export async function session(target, scenario, options = {}) {
133
179
  }
134
180
  if (record.status === 'ok')
135
181
  try {
136
- const result = await conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs });
182
+ // Sent again after a refusal only when that repeats nothing: the transport's
183
+ // 429 means the server never ran it; a tool that says it was rate-limited,
184
+ // thrown or as its result, may have done part of the work first, so only a
185
+ // read-only one is asked again.
186
+ const readOnly = tool?.annotations?.readOnlyHint === true;
187
+ callWaited.ms = 0;
188
+ const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
189
+ waits,
190
+ waited: callWaited,
191
+ error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
192
+ result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
193
+ });
137
194
  if (result.isError) {
138
- const text = (Array.isArray(result.content) ? result.content : [])
139
- .map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
140
- .join(' ')
141
- .replace(/\s+/g, ' ')
142
- .trim();
195
+ const text = errorText(result);
196
+ if (RATE_LIMITED.test(text)) {
197
+ limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
198
+ record.status = 'failed';
199
+ record.note = 'rate-limited';
200
+ steps.push(record);
201
+ break;
202
+ }
143
203
  const short = clip(text, 120);
144
204
  record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
145
205
  record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
@@ -156,7 +216,17 @@ export async function session(target, scenario, options = {}) {
156
216
  }
157
217
  catch (error) {
158
218
  record.status = 'failed';
159
- record.note = error instanceof Error ? error.message.split('\n')[0] : String(error);
219
+ record.note = serverWords(error, 200);
220
+ if (lost(record.note, index)) {
221
+ steps.push(record);
222
+ break;
223
+ }
224
+ if (tooMany(error)) {
225
+ limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
226
+ record.note = 'rate-limited';
227
+ steps.push(record);
228
+ break;
229
+ }
160
230
  const code = error.code;
161
231
  record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
162
232
  if (!SETUP_FAILURES.has(record.failure))
@@ -174,12 +244,23 @@ export async function session(target, scenario, options = {}) {
174
244
  // have changed the menu, and the change belongs to this step, not the next.
175
245
  let next;
176
246
  try {
177
- next = await menuOf(conn);
247
+ listWaited.ms = 0;
248
+ next = await listPatiently(listWaited);
178
249
  }
179
250
  catch (error) {
180
- const why = error instanceof Error ? error.message.split('\n')[0] : String(error);
251
+ const why = serverWords(error, 200);
181
252
  record.status = 'failed';
182
253
  record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
254
+ if (lost(why, index)) {
255
+ steps.push(record);
256
+ break;
257
+ }
258
+ if (tooMany(error)) {
259
+ limited(index, why, listWaited.ms);
260
+ record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
261
+ steps.push(record);
262
+ break;
263
+ }
183
264
  raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
184
265
  steps.push(record);
185
266
  continue;
@@ -208,10 +289,21 @@ export async function session(target, scenario, options = {}) {
208
289
  message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
209
290
  });
210
291
  }
211
- if (step.kind !== 'list') {
212
- const probe = await probeMenu(mainTarget, timeoutMs);
213
- record.scope = scopeOf(changes, next.tools, probe.tools, target.kind);
214
- raw.push(...scopeFindings(record.scope, modern, index));
292
+ if (step.kind !== 'list' && probes) {
293
+ // The main process is still running here: a server that holds a file or
294
+ // a port can't start a second copy. That leaves the scope unknown, not
295
+ // the run failed.
296
+ const probeWaited = { ms: 0 };
297
+ try {
298
+ probedAfter = index;
299
+ const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
300
+ record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
301
+ raw.push(...scopeFindings(record.scope, modern, index));
302
+ }
303
+ catch (error) {
304
+ record.scope = 'unclear';
305
+ scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
306
+ }
215
307
  }
216
308
  current = next;
217
309
  see(next);
@@ -222,6 +314,19 @@ export async function session(target, scenario, options = {}) {
222
314
  steps.push(record);
223
315
  }
224
316
  raw.push(...untested(steps, target));
317
+ // --auto means "find what you can": an unlock it skipped is a gap too.
318
+ raw.push(...unlockCoverage(baseline.tools, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
319
+ if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
320
+ raw.push(nothingCalled(options.auto.skipped));
321
+ if (scopeUnchecked.length) {
322
+ const which = scopeUnchecked.map((s) => s.step);
323
+ raw.push({
324
+ rule: 'session/scope-unchecked',
325
+ severity: 'info',
326
+ step: which[0],
327
+ message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} while the session ran, so whether the change${which.length === 1 ? ` at step ${which[0]}` : `s at steps ${which.join(', ')}`} reach${which.length === 1 ? 'es' : ''} a fresh ${target.kind === 'stdio' ? 'process' : 'connection'} wasn't checked: ${scopeUnchecked[0].why}`,
328
+ });
329
+ }
225
330
  return {
226
331
  scenario: options.scenarioName ?? 'scenario',
227
332
  server: baseline.server,
@@ -264,9 +369,97 @@ function untested(steps, target) {
264
369
  },
265
370
  ];
266
371
  }
372
+ /**
373
+ * Unlocks whose schema lists every value (an enum), and how many of those values
374
+ * the run got through. The session sees, and --union-out keeps, only the tools
375
+ * behind the values it unlocked: GitHub's enable_toolset has 19, the first starter
376
+ * unlocked 2, and 72 of 81 tools never reached the baseline.
377
+ */
378
+ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], stoppedAt) {
379
+ const out = [];
380
+ // A run a rate limit stopped (session/rate-limited says so) is charged only for
381
+ // values its scenario leaves out, not for ones in steps it never got to.
382
+ const unrun = stoppedAt === undefined ? [] : scenario.steps.map((step, i) => ({ step, index: i + 1 })).filter((s) => s.index >= stoppedAt);
383
+ for (const u of unlockers(menu)) {
384
+ if (!u.param || u.values.length === 0)
385
+ continue;
386
+ const reached = new Set();
387
+ let last;
388
+ const add = (arg) => {
389
+ for (const v of Array.isArray(arg) ? arg : [arg])
390
+ if (v !== undefined)
391
+ reached.add(canonical(v));
392
+ };
393
+ for (const record of steps) {
394
+ const step = scenario.steps[record.index - 1];
395
+ if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure)
396
+ continue;
397
+ last = record.index;
398
+ add(step.args[u.param]);
399
+ }
400
+ for (const { step } of unrun)
401
+ if (step.kind === 'call' && step.tool === u.tool.name)
402
+ add(step.args[u.param]);
403
+ const missing = u.values.filter((v) => !reached.has(canonical(v)));
404
+ // Never called: a scenario about something else, unless it's building the baseline.
405
+ if (missing.length === 0 || (reached.size === 0 && !baselineWanted))
406
+ continue;
407
+ const got = u.values.length - missing.length;
408
+ const skip = skipped.find((s) => s.tool === u.tool.name)?.reason;
409
+ const why = skip === 'open world'
410
+ ? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
411
+ : skip === 'not read-only'
412
+ ? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
413
+ : '';
414
+ out.push({
415
+ rule: 'session/unlock-coverage',
416
+ severity: 'warn',
417
+ tool: u.tool.name,
418
+ ...(last !== undefined ? { step: last } : {}),
419
+ message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}${why ? '' : ' Unlock every value (session --init puts every one in the starter).'}`,
420
+ detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
421
+ });
422
+ }
423
+ return out;
424
+ }
425
+ /**
426
+ * --auto that called nothing: the run only listed the menu, so a clean result
427
+ * says nothing about what the tools do to it. Why each tool was left out.
428
+ */
429
+ function nothingCalled(skipped) {
430
+ const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
431
+ const parts = [];
432
+ const openWorld = by('open world');
433
+ const needs = by('needs values');
434
+ const writes = by('not read-only');
435
+ if (openWorld.length)
436
+ parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
437
+ if (needs.length)
438
+ parts.push(`${needs.length} need values the schema doesn't give (--save-scenario puts them in a scenario to fill in)`);
439
+ if (writes.length)
440
+ parts.push(`${writes.length} not marked readOnlyHint`);
441
+ const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
442
+ return {
443
+ rule: 'session/nothing-called',
444
+ severity: 'warn',
445
+ message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
446
+ detail: [
447
+ ...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
448
+ ...(needs.length ? [`need values: ${list(needs)}`] : []),
449
+ ],
450
+ };
451
+ }
267
452
  function serverOf(c) {
268
453
  return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
269
454
  }
455
+ /** A tool result's text, whitespace collapsed. */
456
+ function errorText(result) {
457
+ return (Array.isArray(result.content) ? result.content : [])
458
+ .map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
459
+ .join(' ')
460
+ .replace(/\s+/g, ' ')
461
+ .trim();
462
+ }
270
463
  function waitFor(done, ms) {
271
464
  return new Promise((resolve) => {
272
465
  if (done())
@@ -345,9 +538,13 @@ export function changeFindings(before, after, changes, step, origin) {
345
538
  /**
346
539
  * Does a fresh connection see this step's changes? All of them: the server's tool
347
540
  * set changed. None: the change belongs to this connection (or, on stdio, this
348
- * process). Reorders alone can't tell, and a mix is unclear.
541
+ * process). Reorders alone can't tell, and a mix is unclear. So can't a step that
542
+ * takes the menu back to the session's `baseline` while fresh ones still serve it:
543
+ * a fresh process starts there anyway (an unlock undone reads as "seen" by it).
349
544
  */
350
- export function scopeOf(changes, next, probe, transport) {
545
+ export function scopeOf(changes, next, probe, transport, baseline) {
546
+ if (baseline && compareMenus(baseline, next).length === 0 && compareMenus(baseline, probe).length === 0)
547
+ return 'unclear';
351
548
  const probeByName = new Map(probe.map((t) => [t.name, t]));
352
549
  const nextByName = new Map(next.map((t) => [t.name, t]));
353
550
  const verdicts = new Set();
@@ -424,10 +621,17 @@ function settle(raw, options) {
424
621
  // A parameter that names what to unlock. Strong names say so alone; weak ones
425
622
  // (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
426
623
  // description to back them up.
427
- const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?|namespaces?)$/i;
428
- const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?)$/i;
624
+ // `namespace` is weak too: Kubernetes and Pinecone take one on every read.
625
+ const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
626
+ const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
429
627
  const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
430
- const UNLOCK_DESC = /(?:unlock|enable|activate|load|expose|add)s?[^.]{0,60}tools?|more tools|toolsets?|capabilit(?:y|ies)|unlock/i;
628
+ // A description about loading tools: "Adds the audit tools", "enables more
629
+ // capabilities", "unlocks…". Whole words: "download" isn't "load", and a device's
630
+ // capabilities alone aren't tools.
631
+ const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
632
+ // Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
633
+ const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
634
+ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
431
635
  /**
432
636
  * At most `max` characters, cut by code point at a word boundary, never inside a
433
637
  * character. Drops U+FFFD left over from a server that cut its own message by bytes.
@@ -441,10 +645,15 @@ export function clip(text, max) {
441
645
  return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
442
646
  }
443
647
  function isWriteLike(t) {
648
+ // Named as an unlock: changing the menu is its job, whatever its hints say
649
+ // (toolception marks enable_toolset destructive). It's still only suggested,
650
+ // never called without allow_writes.
651
+ if (UNLOCK_NAME.test(t.name))
652
+ return false;
444
653
  if (t.annotations?.destructiveHint === true)
445
654
  return true;
446
655
  const verb = verbOf(t.name);
447
- return verb !== undefined && WRITE_VERBS.has(verb) && !UNLOCK_NAME.test(t.name);
656
+ return verb !== undefined && WRITE_VERBS.has(verb);
448
657
  }
449
658
  function enumOf(schema) {
450
659
  if (!schema)
@@ -459,8 +668,19 @@ export function unlockers(tools) {
459
668
  const found = [];
460
669
  for (const tool of tools) {
461
670
  const props = tool.inputSchema?.properties ?? {};
462
- const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(tool.description ?? '');
463
- const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ?? (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined);
671
+ const description = tool.description ?? '';
672
+ // A lookup names toolsets without changing them (GitHub's get_toolset_tools,
673
+ // toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
674
+ const verb = verbOf(tool.name);
675
+ if (verb && LOOKUPS.has(verb) && !UNLOCK_SAYS.test(description))
676
+ continue;
677
+ const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
678
+ const required = tool.inputSchema?.required ?? [];
679
+ const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ??
680
+ (backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined) ??
681
+ // Named like an unlock, one required parameter: that's what it takes
682
+ // (toolception's enable_toolset { name }).
683
+ (UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
464
684
  let score = 0;
465
685
  if (param)
466
686
  score += enumOf(props[param]).length ? 3 : 2;
@@ -473,21 +693,105 @@ export function unlockers(tools) {
473
693
  }
474
694
  return found.sort((a, b) => b.score - a.score);
475
695
  }
696
+ /**
697
+ * Unlocks whose values the schema doesn't give (no enum), each with a read-only
698
+ * lister that should: no required arguments, a lookup verb, and a noun in common
699
+ * with the unlock (toolception: enable_toolset { name } ← list_toolsets).
700
+ */
701
+ export function unlockListers(tools) {
702
+ const out = [];
703
+ const nounSet = (name) => new Set(words(name).map(singular).filter((w) => !VERBS.has(w)));
704
+ for (const u of unlockers(tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)))) {
705
+ if (!u.param || u.values.length)
706
+ continue;
707
+ const mine = new Set([...nounSet(u.tool.name), ...nounSet(u.param)]);
708
+ const lister = tools.find((t) => {
709
+ if (t === u.tool || t.annotations?.readOnlyHint !== true || (t.inputSchema?.required ?? []).length)
710
+ return false;
711
+ const verb = verbOf(t.name);
712
+ return !!verb && LOOKUPS.has(verb) && [...nounSet(t.name)].some((n) => mine.has(n));
713
+ });
714
+ if (lister)
715
+ out.push({ unlock: u.tool.name, lister: lister.name });
716
+ }
717
+ return out;
718
+ }
719
+ /**
720
+ * The values a listing offers, from its JSON (structuredContent, or text that
721
+ * starts with JSON): the first list of strings, or of objects with a key, id,
722
+ * slug or name, each item's own field first (toolception's `key: "quotes"`, not
723
+ * the display name "Quotes" nested under it).
724
+ */
725
+ export function valuesFromListing(result) {
726
+ const roots = [];
727
+ if (result.structuredContent)
728
+ roots.push(result.structuredContent);
729
+ for (const part of Array.isArray(result.content) ? result.content : []) {
730
+ const text = part.text;
731
+ if (typeof text === 'string') {
732
+ const value = leadingJson(text);
733
+ if (value !== undefined)
734
+ roots.push(value);
735
+ }
736
+ }
737
+ const idOf = (item) => {
738
+ if (typeof item === 'string')
739
+ return item;
740
+ if (!item || typeof item !== 'object')
741
+ return undefined;
742
+ const o = item;
743
+ for (const k of ['key', 'id', 'slug', 'name'])
744
+ if (typeof o[k] === 'string' && o[k])
745
+ return o[k];
746
+ return undefined;
747
+ };
748
+ const walk = (v, depth) => {
749
+ if (depth > 5 || !v || typeof v !== 'object')
750
+ return undefined;
751
+ if (Array.isArray(v)) {
752
+ const ids = v.map(idOf).filter((x) => !!x);
753
+ if (ids.length && ids.length === v.length)
754
+ return [...new Set(ids)];
755
+ for (const x of v) {
756
+ const found = walk(x, depth + 1);
757
+ if (found)
758
+ return found;
759
+ }
760
+ return undefined;
761
+ }
762
+ for (const x of Object.values(v)) {
763
+ const found = walk(x, depth + 1);
764
+ if (found)
765
+ return found;
766
+ }
767
+ return undefined;
768
+ };
769
+ for (const r of roots) {
770
+ const found = walk(r, 0);
771
+ if (found)
772
+ return found;
773
+ }
774
+ return [];
775
+ }
476
776
  /** A YAML value for an unlock parameter: an array if the schema takes one. */
477
777
  function unlockValue(u, value) {
478
778
  const schema = u.tool.inputSchema?.properties?.[u.param];
479
779
  return schema?.type === 'array' || schema?.items ? `[${JSON.stringify(value)}]` : JSON.stringify(value);
480
780
  }
481
- export function starterScenario(menu) {
781
+ /** At most this many values of one unlock in a scenario; the rest are named in a comment. */
782
+ export const MAX_UNLOCKS = 50;
783
+ export function starterScenario(menu, options = {}) {
482
784
  const readOnly = menu.tools.filter((t) => t.annotations?.readOnlyHint === true);
483
785
  const required = (t) => t.inputSchema?.required ?? [];
484
- const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)));
786
+ const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t))).map((u) => u.values.length === 0 && u.param && options.values?.[u.tool.name]?.length ? { ...u, values: options.values[u.tool.name] } : u);
485
787
  const unlocking = new Set(found.filter((u) => u.param).map((u) => u.tool));
486
788
  const ready = readOnly.filter((t) => required(t).length === 0 && !unlocking.has(t)).slice(0, 12);
487
789
  const needsArgs = readOnly.filter((t) => required(t).length > 0);
488
- // The best candidate with a real value runs for real: two unlocks, then a repeat.
790
+ // The best candidate with real values runs for real: every value, so the tools
791
+ // behind each are seen (a session baseline holds only what was unlocked), then a repeat.
489
792
  const live = found.find((u) => u.values.length > 0 && u.tool.annotations?.readOnlyHint === true && required(u.tool).every((p) => p === u.param));
490
- const changers = found.filter((u) => u !== live).map((u) => u.tool);
793
+ const others = found.filter((u) => u !== live);
794
+ const changers = others.map((u) => u.tool);
491
795
  const example = (t, p) => {
492
796
  const schema = t.inputSchema?.properties?.[p];
493
797
  const u = found.find((x) => x.tool === t && x.param === p);
@@ -520,18 +824,28 @@ export function starterScenario(menu) {
520
824
  for (const t of ready)
521
825
  lines.push(` - call: ${t.name}`);
522
826
  if (live) {
523
- const [first, second] = live.values;
524
- lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock, list, and unlock again:`);
525
- for (const v of second === undefined ? [first] : [first, second]) {
827
+ const shown = live.values.slice(0, MAX_UNLOCKS);
828
+ lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock every ${live.param} (${live.values.length}), listing after each, then the first again:`);
829
+ for (const v of shown)
526
830
  lines.push(` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, v)} }`, ' - list');
831
+ if (live.values.length > shown.length) {
832
+ lines.push(` # …and ${live.values.length - shown.length} more: ${live.values.slice(shown.length).map((v) => JSON.stringify(v)).join(', ')}`);
527
833
  }
528
- lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, first)} }`, ' - list');
834
+ lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, shown[0])} }`, ' - list');
529
835
  }
530
836
  if (changers.length) {
531
837
  lines.push('', ` # ${live ? 'These also' : 'These'} look like they could change the menu. Try them:`);
532
- for (const t of changers.slice(0, 8)) {
838
+ for (const u of others.slice(0, 8)) {
839
+ const t = u.tool;
533
840
  if (t.annotations?.readOnlyHint !== true)
534
841
  lines.push(` # (not marked readOnlyHint: needs allow_writes: true)`);
842
+ // Every known value, as with a live unlock: each may bring different tools.
843
+ const values = u.param && required(t).every((p) => p === u.param) ? u.values.slice(0, MAX_UNLOCKS) : [];
844
+ if (values.length) {
845
+ for (const v of values)
846
+ lines.push(` # - call: ${t.name}`, ` # args: { ${u.param}: ${unlockValue(u, v)} }`, ' # - list');
847
+ continue;
848
+ }
535
849
  lines.push(` # - call: ${t.name}`);
536
850
  if (argNames(t).length)
537
851
  lines.push(` # args: ${placeholder(t)}`);
package/dist/snapshot.js CHANGED
@@ -1,17 +1,20 @@
1
- import { connect, listTools } from './connect.js';
1
+ import { listTools } from './connect.js';
2
+ import { patiently, tooMany } from './failures.js';
2
3
  import { buildMenu } from './menu.js';
3
- import { isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
4
+ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
4
5
  import { MENU_RULES, runRules } from './rules/index.js';
5
6
  import { readCatalog } from './catalog.js';
6
7
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
7
8
  export async function snapshot(target, options = {}) {
8
- const connection = await connect(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
9
+ // Connecting and listing are read-only: a refusal for being too many is waited
10
+ // out and they're sent again (CI behind a shared rate limit).
11
+ const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
9
12
  let menu;
10
13
  let secondList;
11
14
  let first;
12
15
  try {
13
- first = await listTools(connection, options);
14
- const second = await listTools(connection, options);
16
+ first = await patiently(() => listTools(connection, options), { error: tooMany });
17
+ const second = await patiently(() => listTools(connection, options), { error: tooMany });
15
18
  const server = {
16
19
  name: connection.server.name,
17
20
  version: connection.server.version,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.10.0",
3
+ "version": "0.12.0",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
5
  "author": "Niksa (https://niksa.me)",
6
6
  "main": "dist/index.js",