toolmenu 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/session.js CHANGED
@@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises';
2
2
  import { parse } from 'yaml';
3
3
  import { cacheBreak, canonical, compareMenus } from './compare.js';
4
4
  import { listTools } from './connect.js';
5
- import { callWithInput, MAX_ROUNDS } from './mrtr.js';
5
+ import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
6
6
  import { buildMenu, toolDefinition } from './menu.js';
7
7
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
8
8
  import { varianceFinding } from './rules/determinism.js';
@@ -78,8 +78,14 @@ export async function session(target, scenario, options = {}) {
78
78
  const raw = [];
79
79
  // 2026-07-28: every call that asked for input, for the input findings at the end.
80
80
  const inputs = [];
81
+ // --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
82
+ const handleErrors = [];
81
83
  // The main process runs with a pinned hash seed, and so does every scope probe:
82
84
  // a probe with a different seed would count ordering variance as a change.
85
+ // How every elicitation is answered: the same each run (--elicitation). Session
86
+ // connections declare elicitation; snapshot's don't, so its menu is the one 0.14 saw.
87
+ const answer = target.elicitation ?? 'decline';
88
+ target = { ...target, elicitation: answer };
83
89
  const mainTarget = seeded(target, MAIN_SEED);
84
90
  const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
85
91
  // Connecting and listing are read-only: a refusal for being too many is waited
@@ -169,7 +175,9 @@ export async function session(target, scenario, options = {}) {
169
175
  // this one when that happens.
170
176
  let probedAfter = probes ? 0 : undefined;
171
177
  const lost = (why, index) => {
172
- if (probedAfter === undefined || !SESSION_LOST.test(why))
178
+ // 2026-07-28 has no protocol sessions: "unknown session" from such a server is
179
+ // about a session it minted itself, a handle the model passes (SEP-2567).
180
+ if (probedAfter === undefined || conn.era === 'modern' || !SESSION_LOST.test(why))
173
181
  return false;
174
182
  // The server's own words, not the transport's wrapping of its JSON-RPC error.
175
183
  const said = serverWords(why);
@@ -277,10 +285,12 @@ export async function session(target, scenario, options = {}) {
277
285
  callWaited.ms = 0;
278
286
  // 2026-07-28: a tool that asks for input gets every elicitation declined
279
287
  // and is called again (mrtr.ts); on 2025 it's a plain call.
280
- const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: target.elicitation }), {
288
+ const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: answer }), {
281
289
  waits,
282
290
  waited: callWaited,
283
- error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
291
+ // A 429 means the server didn't run that request, but after an input round the
292
+ // tool's first leg did run: only a read-only tool is called again from scratch.
293
+ error: (e) => (httpStatus(e) === 429 && (readOnly || !(e instanceof AfterInputError))) || (readOnly && tooMany(e)),
284
294
  result: (c) => readOnly && c.result?.isError === true && RATE_LIMITED.test(errorText(c.result)),
285
295
  });
286
296
  if (call.input.outcome !== 'none' || call.input.missingResultType) {
@@ -288,62 +298,83 @@ export async function session(target, scenario, options = {}) {
288
298
  inputs.push({ step: index, tool: step.tool, input: call.input });
289
299
  }
290
300
  if (!call.result) {
291
- // It never answered: still asking, asking for what wasn't declared, or failing on the decline.
301
+ // It never answered: still asking, asking for what wasn't declared, or
302
+ // failing on the decline. Said by the input findings, not as a failed call;
303
+ // the menu is still listed after it, as after any step.
292
304
  record.outcome = 'needs-input';
293
- record.note = inputNote(call.input, target.elicitation ?? 'decline');
294
- throw new InputStopped();
305
+ record.note = inputNote(call.input, answer);
295
306
  }
296
- const result = call.result;
297
- record.outcome = result.isError ? 'tool-error' : 'answered';
298
- if (call.input.rounds && !result.isError)
299
- record.note = inputNote(call.input, target.elicitation ?? 'decline');
300
- if (result.isError) {
301
- const text = errorText(result);
302
- if (RATE_LIMITED.test(text)) {
303
- limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
304
- record.status = 'failed';
305
- record.note = 'rate-limited';
306
- done();
307
- break;
307
+ else {
308
+ const result = call.result;
309
+ record.outcome = result.isError ? 'tool-error' : 'answered';
310
+ if (call.input.rounds && !result.isError)
311
+ record.note = inputNote(call.input, answer);
312
+ if (step.probe) {
313
+ // A handle that can't exist: a tool error is the answer the spec asks for.
314
+ const text = result.isError ? errorText(result) : '';
315
+ record.note = result.isError
316
+ ? `called with a made-up ${step.probe.kind}; it answered with a tool error${text ? `: ${clip(text, 100)}` : ''}`
317
+ : `called with a made-up ${step.probe.kind}; it answered as if that ${step.probe.kind} existed`;
318
+ }
319
+ else if (result.isError) {
320
+ const text = errorText(result);
321
+ if (RATE_LIMITED.test(text)) {
322
+ limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
323
+ record.status = 'failed';
324
+ record.note = 'rate-limited';
325
+ done();
326
+ break;
327
+ }
328
+ if (call.input.rounds) {
329
+ // The tool's answer to the decline: how a tool should say no. Not a
330
+ // setup failure, whatever its words ("the user declined to connect GitHub").
331
+ record.note = `${inputNote(call.input, answer)}, with a tool error: ${clip(text, 100)}`;
332
+ }
333
+ else {
334
+ record.note = text ? `the tool returned an error: ${clip(text, 120)}` : 'the tool returned an error';
335
+ record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
336
+ // A setup failure is reported once, for the whole run (session/untested).
337
+ if (!SETUP_FAILURES.has(record.failure))
338
+ fail('session/tool-error', 'warn', index, step, text, record.failure);
339
+ }
308
340
  }
309
- const short = clip(text, 120);
310
- record.note = call.input.rounds
311
- ? `${inputNote(call.input, target.elicitation ?? 'decline')}, with a tool error: ${clip(text, 100)}`
312
- : text ? `the tool returned an error: ${short}` : 'the tool returned an error';
313
- record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
314
- // A setup failure is reported once, for the whole run (session/untested). A
315
- // tool error that answers a decline is how a tool should say no (input-required).
316
- if (!SETUP_FAILURES.has(record.failure) && !call.input.rounds)
317
- fail('session/tool-error', 'warn', index, step, text, record.failure);
318
341
  }
319
342
  }
320
343
  catch (error) {
321
- // A tool that never answered its input requests is said by the input findings,
322
- // not as a failed call. The menu is still listed after it, as after any step.
323
- if (error instanceof InputStopped) {
324
- // fall through to the listing below
344
+ // A call that asked for input before it failed still counts among those that asked.
345
+ if (error instanceof AfterInputError && (error.input.rounds || error.input.continuations)) {
346
+ record.input = error.input;
347
+ if (error.input.kinds.length)
348
+ inputs.push({ step: index, tool: step.tool, input: error.input });
349
+ }
350
+ record.status = 'failed';
351
+ record.outcome = 'failed';
352
+ record.note = serverWords(error, 200);
353
+ // A call with a made-up handle ("Unknown session: toolmenu-unknown-session") is about that handle.
354
+ if (!step.probe && lost(record.note, index)) {
355
+ done();
356
+ break;
357
+ }
358
+ if (tooMany(error)) {
359
+ limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
360
+ record.note = 'rate-limited';
361
+ done();
362
+ break;
363
+ }
364
+ const code = error.code;
365
+ // 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
366
+ if (code === -32021) {
367
+ const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
368
+ if (needed.length)
369
+ record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
370
+ }
371
+ if (step.probe && typeof code === 'number' && code !== -32021) {
372
+ // The server's protocol error to a handle that can't exist: the model gets no reason it can act on.
373
+ if (!handleErrors.some((e) => e.tool === step.tool))
374
+ handleErrors.push({ step: index, tool: step.tool, probe: step.probe, value: step.args[step.probe.param], message: record.note, code });
375
+ record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
325
376
  }
326
377
  else {
327
- record.status = 'failed';
328
- record.outcome = 'failed';
329
- record.note = serverWords(error, 200);
330
- if (lost(record.note, index)) {
331
- done();
332
- break;
333
- }
334
- if (tooMany(error)) {
335
- limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
336
- record.note = 'rate-limited';
337
- done();
338
- break;
339
- }
340
- const code = error.code;
341
- // 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
342
- if (code === -32021) {
343
- const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
344
- if (needed.length)
345
- record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
346
- }
347
378
  record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
348
379
  // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
349
380
  const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
@@ -455,12 +486,13 @@ export async function session(target, scenario, options = {}) {
455
486
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
456
487
  record.tools = current.tools.length;
457
488
  record.tokens = current.totalTokens;
458
- if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
489
+ if (firstTry && !record.changed && record.status === 'ok' && !record.failure && record.outcome !== 'needs-input')
459
490
  quiet.add(step.tool);
460
491
  done();
461
492
  }
462
493
  raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
463
- raw.push(...inputFindings(inputs, target.elicitation ?? 'decline'));
494
+ raw.push(...inputFindings(inputs, answer));
495
+ raw.push(...handleErrorFindings(handleErrors));
464
496
  if (explained.length)
465
497
  raw.push(knownVariance(explained));
466
498
  if (onWord.length)
@@ -547,7 +579,7 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
547
579
  // Never says it unlocks tools and the run didn't call it: nothing suggests tools
548
580
  // are missing (Garmin's garmin_sync { domains } downloads data). One it called
549
581
  // and stopped is covered by auto.stopped.
550
- if (!u.claims && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
582
+ if (!definite(u) && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
551
583
  continue;
552
584
  const reached = new Set();
553
585
  let last;
@@ -558,7 +590,8 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
558
590
  };
559
591
  for (const record of steps) {
560
592
  const step = scenario.steps[record.index - 1];
561
- if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure)
593
+ // A call that never finished its input rounds didn't unlock anything.
594
+ if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure || record.outcome === 'needs-input')
562
595
  continue;
563
596
  last = record.index;
564
597
  add(step.args[u.param]);
@@ -1071,6 +1104,28 @@ function settle(raw, options) {
1071
1104
  // description to back them up.
1072
1105
  // `namespace` is weak too: Kubernetes and Pinecone take one on every read.
1073
1106
  const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
1107
+ // Parameter names that only ever name sets of tools. Not `domains` (Garmin's
1108
+ // garmin_sync { domains: [sleep, hrv] } names data), nor `capabilities`, `packs` or
1109
+ // `bundles` (a device's capabilities, a price bundle): those need the tool to say so.
1110
+ const TOOLSET_PARAM = /^(?:toolsets?|tool_?sets?)$/i;
1111
+ /**
1112
+ * An unlock that surely unlocks tools: it says so (name or description), or its
1113
+ * parameter is named for tool sets outright (toolsets). One that only has an enum of domains, and
1114
+ * doesn't say it unlocks anything, may be data: session --auto still tries it once,
1115
+ * but menu/gated and session/unlock-coverage don't charge it.
1116
+ */
1117
+ export function definite(u) {
1118
+ return u.claims || (!!u.param && TOOLSET_PARAM.test(u.param) && !dispatches(u.tool));
1119
+ }
1120
+ /** What a dispatcher takes besides the set: the tool or operation to run in it, or that call's arguments. */
1121
+ const DISPATCH_PARAM = /^(?:tool(?:_?name)?|command|sub_?command|operation|method|action|parameters|params|arguments|args)$/i;
1122
+ /**
1123
+ * A dispatcher (`call_tool { toolset, tool, arguments }`) names a tool set to run
1124
+ * one of its tools, not to unlock it: its `toolset` alone doesn't make it an unlock.
1125
+ */
1126
+ function dispatches(tool) {
1127
+ return Object.keys(tool?.inputSchema?.properties ?? {}).some((p) => DISPATCH_PARAM.test(p));
1128
+ }
1074
1129
  const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
1075
1130
  const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
1076
1131
  // A description about loading tools: "Adds the audit tools", "enables more
@@ -1357,9 +1412,6 @@ export function starterScenario(menu, options = {}) {
1357
1412
  }
1358
1413
  return lines.join('\n') + '\n';
1359
1414
  }
1360
- /** Thrown inside a call step when the tool never completed its input rounds: not a failed request. */
1361
- class InputStopped extends Error {
1362
- }
1363
1415
  const PAST = { decline: 'declined', cancel: 'cancelled' };
1364
1416
  const KIND_WORDS = { form: 'a form', url: 'a URL to open', roots: 'roots', sampling: 'sampling', other: 'something else' };
1365
1417
  function kindList(kinds) {
@@ -1381,14 +1433,37 @@ function inputNote(input, answer) {
1381
1433
  return `${asked}; asked the same again after each ${answer}, stopped after ${rounds}`;
1382
1434
  case 'gave-up':
1383
1435
  return `${asked}; still asking after ${rounds}`;
1436
+ case 'unfinished': {
1437
+ const work = `kept answering input_required with only its requestState (${input.continuations ?? 0} rounds), never a result`;
1438
+ return input.kinds.length ? `${asked}; ${PAST[answer]} ${times(input.rounds)}, then ${work}` : `asked for nothing, but ${work}`;
1439
+ }
1384
1440
  case 'undeclared':
1385
1441
  return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
1386
1442
  case 'decline-failed':
1387
- return `${asked}; the retry with the ${answer} failed: ${serverWords(input.error, 160)}`;
1443
+ return `${asked}; the retry with the ${answer} failed: ${clip(input.error?.message ?? '', 160)}`;
1444
+ case 'bad-state':
1445
+ return `${input.kinds.length ? `${asked}, with` : 'answered input_required with'} a requestState that isn't a string, so it can't be sent back as it came`;
1388
1446
  default:
1389
1447
  return asked;
1390
1448
  }
1391
1449
  }
1450
+ /**
1451
+ * Handles that can't exist answered with a protocol error (2026-07-28, SEP-2567):
1452
+ * "a call against an expired or unknown handle should return a tool execution
1453
+ * error that says so, so the model can recover by creating a new one." One
1454
+ * finding per tool.
1455
+ */
1456
+ function handleErrorFindings(errors) {
1457
+ return errors.map((e) => ({
1458
+ rule: 'session/handle-error',
1459
+ severity: 'warn',
1460
+ confidence: 'unsure',
1461
+ step: e.step,
1462
+ tool: e.tool,
1463
+ message: `${e.tool} answers a ${e.probe.kind} that doesn't exist with a protocol error (${e.code}: “${clip(e.message, 120)}”), not a tool error. A model whose ${e.probe.kind} expired gets an error from its client instead of a result it can read, so it can't tell it should start a new one. toolmenu passed ${e.probe.param}: ${JSON.stringify(e.value)}, which no ${e.probe.kind} has.`,
1464
+ fix: `Return a tool result with isError: true that says the ${e.probe.kind} is unknown or expired and how to get a new one ("Session not found or expired: call start_session again").`,
1465
+ }));
1466
+ }
1392
1467
  /**
1393
1468
  * What tools that asked for input did with the answer (2026-07-28 multi
1394
1469
  * round-trip requests). One info finding for all of them, and a warning or error
@@ -1396,7 +1471,7 @@ function inputNote(input, answer) {
1396
1471
  */
1397
1472
  function inputFindings(inputs, answer) {
1398
1473
  const out = [];
1399
- const asked = inputs.filter((i) => i.input.outcome !== 'none');
1474
+ const asked = inputs.filter((i) => i.input.outcome !== 'none' && i.input.kinds.length);
1400
1475
  if (asked.length) {
1401
1476
  const finished = asked.filter((i) => i.input.outcome === 'completed').length;
1402
1477
  out.push({
@@ -1408,8 +1483,8 @@ function inputFindings(inputs, answer) {
1408
1483
  fix: 'Nothing to do for the ones that answered. The findings next to this one cover the rest.',
1409
1484
  });
1410
1485
  }
1411
- for (const { step, tool, input } of asked) {
1412
- if (input.outcome === 'loop' || input.outcome === 'gave-up') {
1486
+ for (const { step, tool, input } of inputs.filter((i) => i.input.outcome !== 'none')) {
1487
+ if (input.outcome === 'loop' || input.outcome === 'gave-up' || input.outcome === 'unfinished') {
1413
1488
  out.push({
1414
1489
  rule: 'session/input-loop',
1415
1490
  severity: 'warn',
@@ -1417,9 +1492,13 @@ function inputFindings(inputs, answer) {
1417
1492
  tool,
1418
1493
  message: input.outcome === 'loop'
1419
1494
  ? `${tool} asks the same thing again after a ${answer}, every time: an agent whose user says no gets the question back instead of an answer, and can't finish the call. toolmenu stopped after ${MAX_ROUNDS} rounds.`
1420
- : `${tool} was still asking for input after ${MAX_ROUNDS} rounds of ${answer}s, each time something else. An agent whose user says no can't finish the call.`,
1495
+ : input.outcome === 'gave-up'
1496
+ ? `${tool} was still asking for input after ${MAX_ROUNDS} rounds of ${answer}s, not always for the same thing. An agent whose user says no can't finish the call.`
1497
+ : `${tool} ${input.kinds.length ? `took the ${answer}, then ` : ''}kept answering input_required with only its requestState, asking for nothing more, and never returned a result (toolmenu stopped after ${input.continuations ?? 0} rounds, at most ${MAX_CONTINUATIONS} or 3 × the request timeout). The client keeps calling it again and the agent never gets an answer.`,
1421
1498
  detail: [inputNote(input, answer)],
1422
- fix: `Treat a ${answer} as an answer: return a result that says the user ${answer === 'decline' ? 'declined' : 'dismissed it'} (isError: true is fine), so the model can tell the user and move on.`,
1499
+ fix: input.outcome === 'unfinished'
1500
+ ? 'Return a result once the work is done, or a tool error if it can\'t finish; for long work, hand back a handle the agent can check with another tool.'
1501
+ : `Treat a ${answer} as an answer: return a result that says the user ${answer === 'decline' ? 'declined' : 'dismissed it'} (isError: true is fine), so the model can tell the user and move on.`,
1423
1502
  });
1424
1503
  }
1425
1504
  else if (input.outcome === 'decline-failed') {
@@ -1428,11 +1507,22 @@ function inputFindings(inputs, answer) {
1428
1507
  severity: 'warn',
1429
1508
  step,
1430
1509
  tool,
1431
- message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${serverWords(input.error, 120)}”). The model gets an error from the client instead of a result it can read and explain to the user.`,
1510
+ message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${clip(input.error?.message ?? '', 120)}”). The model gets an error from the client instead of a result it can read and explain to the user.`,
1432
1511
  detail: [inputNote(input, answer)],
1433
1512
  fix: `Return a tool result on a ${answer} (isError: true with a sentence like "the user declined"), not a JSON-RPC error.`,
1434
1513
  });
1435
1514
  }
1515
+ else if (input.outcome === 'bad-state') {
1516
+ out.push({
1517
+ rule: 'session/result-type',
1518
+ severity: 'error',
1519
+ step,
1520
+ tool,
1521
+ message: `${tool} answered input_required with a requestState that isn't a string. 2026-07-28 makes it a string the client sends back byte for byte; a client can't echo anything else as it came, so the call can't go on (calling again without it would run the tool from scratch).`,
1522
+ detail: [inputNote(input, answer)],
1523
+ fix: 'Send requestState as a string: serialize (and sign, if it matters) whatever state the tool needs, and parse it when the call comes back.',
1524
+ });
1525
+ }
1436
1526
  else if (input.outcome === 'undeclared') {
1437
1527
  out.push({
1438
1528
  rule: 'session/undeclared-input',
@@ -1,4 +1,6 @@
1
1
  import { type Target } from './connect.js';
2
+ import { type Identity } from './probe.js';
3
+ import type { IdentityMenu } from './rules/identities.js';
2
4
  import { type RuleSettings } from './rules/index.js';
3
5
  import { type CatalogOptions } from './catalog.js';
4
6
  import type { Routes } from './routes.js';
@@ -15,13 +17,20 @@ export interface SnapshotOptions extends RuleSettings {
15
17
  processes?: number;
16
18
  /** Also read the operations behind a catalog search tool or command routers (true, or which ones and how). */
17
19
  catalog?: boolean | CatalogOptions;
20
+ /**
21
+ * snapshot --as: list the menu once per identity. The first is the main one: its
22
+ * menu is the one saved and checked; the others are compared with it.
23
+ */
24
+ identities?: Identity[];
18
25
  }
19
26
  export interface SnapshotResult {
20
27
  menu: Menu;
21
28
  findings: Finding[];
29
+ /** snapshot --as: each identity's tools and tokens, or why it couldn't be listed. */
30
+ identities?: IdentityMenu[];
22
31
  }
23
32
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
24
- export declare function snapshot(target: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
33
+ export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
25
34
  /**
26
35
  * How this run reached the server, as the end of a command line, for next steps:
27
36
  * the options that got it past the door (names only; values are the user's) and
package/dist/snapshot.js CHANGED
@@ -2,13 +2,15 @@ import { listTools } from './connect.js';
2
2
  import { patiently, tooMany } from './failures.js';
3
3
  import { ConnectError } from './explain.js';
4
4
  import { buildMenu } from './menu.js';
5
- import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
5
+ import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
6
6
  import { MENU_RULES, runRules } from './rules/index.js';
7
7
  import { catalogFindings, readCatalog } from './catalog.js';
8
8
  import { SEVERITY_RANK } from './types.js';
9
9
  import { shellWord } from './init.js';
10
10
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
11
- export async function snapshot(target, options = {}) {
11
+ export async function snapshot(given, options = {}) {
12
+ const [mainIdentity, ...otherIdentities] = options.identities ?? [];
13
+ const target = asIdentity(given, mainIdentity);
12
14
  // Connecting and listing are read-only: a refusal for being too many is waited
13
15
  // out and they're sent again (CI behind a shared rate limit).
14
16
  const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
@@ -43,6 +45,19 @@ export async function snapshot(target, options = {}) {
43
45
  // After the main process has exited: a server that holds a file or a port
44
46
  // shouldn't have to share it with its own probe.
45
47
  const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000);
48
+ // Each other identity's menu, from its own process (stdio) or connection (HTTP).
49
+ const identities = mainIdentity
50
+ ? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
51
+ : undefined;
52
+ for (const identity of otherIdentities) {
53
+ try {
54
+ const m = await probeMenu(seeded(asIdentity(given, identity), MAIN_SEED), options.timeoutMs ?? 30_000);
55
+ identities.push({ name: identity.name, tools: m.tools, tokens: m.totalTokens, cacheScope: m.listMeta?.cacheScope });
56
+ }
57
+ catch (error) {
58
+ identities.push({ name: identity.name, error: firstLine(error) });
59
+ }
60
+ }
46
61
  const findings = runRules(MENU_RULES, {
47
62
  menu,
48
63
  secondList,
@@ -60,6 +75,7 @@ export async function snapshot(target, options = {}) {
60
75
  wrapper: isContainerWrapper(target),
61
76
  serverInfo: connection.server,
62
77
  server: serverArgs(target),
78
+ identities,
63
79
  }, options);
64
80
  // What --catalog called and what it got, under the same settings as the rules
65
81
  // (rules: { "catalog/read": "off" }, ignore).
@@ -73,8 +89,9 @@ export async function snapshot(target, options = {}) {
73
89
  }
74
90
  findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
75
91
  }
76
- return { menu, findings };
92
+ return { menu, findings, ...(identities ? { identities } : {}) };
77
93
  }
94
+ const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
78
95
  /**
79
96
  * How this run reached the server, as the end of a command line, for next steps:
80
97
  * the options that got it past the door (names only; values are the user's) and
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.16.0",
3
+ "version": "0.17.0",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
5
  "author": "Nikša Perović (https://niksa.me)",
6
6
  "main": "dist/index.js",