toolmenu 0.14.1 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/session.js CHANGED
@@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises';
2
2
  import { parse } from 'yaml';
3
3
  import { cacheBreak, canonical, compareMenus } from './compare.js';
4
4
  import { listTools } from './connect.js';
5
+ import { callWithInput, MAX_ROUNDS } from './mrtr.js';
5
6
  import { buildMenu, toolDefinition } from './menu.js';
6
7
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
8
  import { varianceFinding } from './rules/determinism.js';
@@ -75,6 +76,8 @@ export async function session(target, scenario, options = {}) {
75
76
  const timeoutMs = options.timeoutMs ?? 30_000;
76
77
  const grace = options.noticeGraceMs ?? 500;
77
78
  const raw = [];
79
+ // 2026-07-28: every call that asked for input, for the input findings at the end.
80
+ const inputs = [];
78
81
  // The main process runs with a pinned hash seed, and so does every scope probe:
79
82
  // a probe with a different seed would count ordering variance as a change.
80
83
  const mainTarget = seeded(target, MAIN_SEED);
@@ -272,13 +275,28 @@ export async function session(target, scenario, options = {}) {
272
275
  // read-only one is asked again. On the server's word, not the user's.
273
276
  const readOnly = tool?.annotations?.readOnlyHint === true;
274
277
  callWaited.ms = 0;
275
- const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
278
+ // 2026-07-28: a tool that asks for input gets every elicitation declined
279
+ // and is called again (mrtr.ts); on 2025 it's a plain call.
280
+ const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: target.elicitation }), {
276
281
  waits,
277
282
  waited: callWaited,
278
283
  error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
279
- result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
284
+ result: (c) => readOnly && c.result?.isError === true && RATE_LIMITED.test(errorText(c.result)),
280
285
  });
286
+ if (call.input.outcome !== 'none' || call.input.missingResultType) {
287
+ record.input = call.input;
288
+ inputs.push({ step: index, tool: step.tool, input: call.input });
289
+ }
290
+ if (!call.result) {
291
+ // It never answered: still asking, asking for what wasn't declared, or failing on the decline.
292
+ record.outcome = 'needs-input';
293
+ record.note = inputNote(call.input, target.elicitation ?? 'decline');
294
+ throw new InputStopped();
295
+ }
296
+ const result = call.result;
281
297
  record.outcome = result.isError ? 'tool-error' : 'answered';
298
+ if (call.input.rounds && !result.isError)
299
+ record.note = inputNote(call.input, target.elicitation ?? 'decline');
282
300
  if (result.isError) {
283
301
  const text = errorText(result);
284
302
  if (RATE_LIMITED.test(text)) {
@@ -289,33 +307,49 @@ export async function session(target, scenario, options = {}) {
289
307
  break;
290
308
  }
291
309
  const short = clip(text, 120);
292
- record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
310
+ record.note = call.input.rounds
311
+ ? `${inputNote(call.input, target.elicitation ?? 'decline')}, with a tool error: ${clip(text, 100)}`
312
+ : text ? `the tool returned an error: ${short}` : 'the tool returned an error';
293
313
  record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
294
- // A setup failure is reported once, for the whole run (session/untested).
295
- if (!SETUP_FAILURES.has(record.failure))
314
+ // A setup failure is reported once, for the whole run (session/untested). A
315
+ // tool error that answers a decline is how a tool should say no (input-required).
316
+ if (!SETUP_FAILURES.has(record.failure) && !call.input.rounds)
296
317
  fail('session/tool-error', 'warn', index, step, text, record.failure);
297
318
  }
298
319
  }
299
320
  catch (error) {
300
- record.status = 'failed';
301
- record.outcome = 'failed';
302
- record.note = serverWords(error, 200);
303
- if (lost(record.note, index)) {
304
- done();
305
- break;
321
+ // A tool that never answered its input requests is said by the input findings,
322
+ // not as a failed call. The menu is still listed after it, as after any step.
323
+ if (error instanceof InputStopped) {
324
+ // fall through to the listing below
306
325
  }
307
- if (tooMany(error)) {
308
- limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
309
- record.note = 'rate-limited';
310
- done();
311
- break;
326
+ else {
327
+ record.status = 'failed';
328
+ record.outcome = 'failed';
329
+ record.note = serverWords(error, 200);
330
+ if (lost(record.note, index)) {
331
+ done();
332
+ break;
333
+ }
334
+ if (tooMany(error)) {
335
+ limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
336
+ record.note = 'rate-limited';
337
+ done();
338
+ break;
339
+ }
340
+ const code = error.code;
341
+ // 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
342
+ if (code === -32021) {
343
+ const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
344
+ if (needed.length)
345
+ record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
346
+ }
347
+ record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
348
+ // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
349
+ const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
350
+ if (!SETUP_FAILURES.has(record.failure))
351
+ fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
312
352
  }
313
- const code = error.code;
314
- record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
315
- // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
316
- const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
317
- if (!SETUP_FAILURES.has(record.failure))
318
- fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
319
353
  }
320
354
  }
321
355
  else if (step.kind === 'wait_for') {
@@ -426,6 +460,7 @@ export async function session(target, scenario, options = {}) {
426
460
  done();
427
461
  }
428
462
  raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
463
+ raw.push(...inputFindings(inputs, target.elicitation ?? 'decline'));
429
464
  if (explained.length)
430
465
  raw.push(knownVariance(explained));
431
466
  if (onWord.length)
@@ -490,7 +525,7 @@ function untested(steps, target) {
490
525
  // Classified from the error's words: a guess, if a good one.
491
526
  confidence: 'unsure',
492
527
  message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
493
- detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
528
+ detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${clip((list[0].note ?? '').replace(/^the tool returned an error: /, ''), 100)}”)`),
494
529
  fix,
495
530
  },
496
531
  ];
@@ -509,6 +544,11 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
509
544
  for (const u of unlockers(menu)) {
510
545
  if (!u.param || u.values.length === 0)
511
546
  continue;
547
+ // Never says it unlocks tools and the run didn't call it: nothing suggests tools
548
+ // are missing (Garmin's garmin_sync { domains } downloads data). One it called
549
+ // and stopped is covered by auto.stopped.
550
+ if (!u.claims && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
551
+ continue;
512
552
  const reached = new Set();
513
553
  let last;
514
554
  const add = (arg) => {
@@ -1063,13 +1103,31 @@ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
1063
1103
  * character. Drops U+FFFD left over from a server that cut its own message by bytes.
1064
1104
  */
1065
1105
  export function clip(text, max) {
1066
- const chars = Array.from(text.replace(/\uFFFD+$/u, ''));
1106
+ let chars = Array.from(text.replace(/\uFFFD+$/u, ''));
1107
+ if (chars.length <= max)
1108
+ return chars.join('');
1109
+ // A long path loses its middle before the text loses its end: “spawn /tmp/…/x ENOENT”
1110
+ // cut at the end hides both the file and the reason.
1111
+ chars = Array.from(shortenPaths(chars.join('')));
1067
1112
  if (chars.length <= max)
1068
1113
  return chars.join('');
1069
1114
  const cut = chars.slice(0, max - 1).join('');
1070
1115
  const space = cut.lastIndexOf(' ');
1071
1116
  return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
1072
1117
  }
1118
+ /**
1119
+ * Absolute paths longer than 40 characters, kept to their last two parts:
1120
+ * /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
1121
+ */
1122
+ export function shortenPaths(text) {
1123
+ // Not a URL's path: a path starts after nothing that could be a scheme, host or word.
1124
+ return text.replace(/(?<![\w:/\\.])(?:[A-Za-z]:)?(?:[\\/][^\s\\/"'“”()]+){3,}/g, (path) => {
1125
+ if (path.length <= 40)
1126
+ return path;
1127
+ const sep = path.includes('\\') && !path.includes('/') ? '\\' : '/';
1128
+ return `…${sep}${path.split(/[\\/]/).slice(-2).join(sep)}`;
1129
+ });
1130
+ }
1073
1131
  function isWriteLike(t) {
1074
1132
  // Named as an unlock: changing the menu is its job, whatever its hints say
1075
1133
  // (toolception marks enable_toolset destructive). It's still only suggested,
@@ -1299,3 +1357,105 @@ export function starterScenario(menu, options = {}) {
1299
1357
  }
1300
1358
  return lines.join('\n') + '\n';
1301
1359
  }
1360
+ /** Thrown inside a call step when the tool never completed its input rounds: not a failed request. */
1361
+ class InputStopped extends Error {
1362
+ }
1363
+ const PAST = { decline: 'declined', cancel: 'cancelled' };
1364
+ const KIND_WORDS = { form: 'a form', url: 'a URL to open', roots: 'roots', sampling: 'sampling', other: 'something else' };
1365
+ function kindList(kinds) {
1366
+ const words = kinds.map((k) => KIND_WORDS[k]);
1367
+ return words.length <= 1 ? words.join('') : `${words.slice(0, -1).join(', ')} and ${words[words.length - 1]}`;
1368
+ }
1369
+ /** once, twice, 3 times. */
1370
+ function times(n) {
1371
+ return n === 1 ? 'once' : n === 2 ? 'twice' : `${n} times`;
1372
+ }
1373
+ /** The step's note for a call that asked for input. */
1374
+ function inputNote(input, answer) {
1375
+ const asked = `asked for ${kindList(input.kinds)}`;
1376
+ const rounds = `${input.rounds} round${input.rounds === 1 ? '' : 's'}`;
1377
+ switch (input.outcome) {
1378
+ case 'completed':
1379
+ return `${asked}; ${PAST[answer]} ${times(input.rounds)}, then answered`;
1380
+ case 'loop':
1381
+ return `${asked}; asked the same again after each ${answer}, stopped after ${rounds}`;
1382
+ case 'gave-up':
1383
+ return `${asked}; still asking after ${rounds}`;
1384
+ case 'undeclared':
1385
+ return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
1386
+ case 'decline-failed':
1387
+ return `${asked}; the retry with the ${answer} failed: ${serverWords(input.error, 160)}`;
1388
+ default:
1389
+ return asked;
1390
+ }
1391
+ }
1392
+ /**
1393
+ * What tools that asked for input did with the answer (2026-07-28 multi
1394
+ * round-trip requests). One info finding for all of them, and a warning or error
1395
+ * for each that an agent can't get past.
1396
+ */
1397
+ function inputFindings(inputs, answer) {
1398
+ const out = [];
1399
+ const asked = inputs.filter((i) => i.input.outcome !== 'none');
1400
+ if (asked.length) {
1401
+ const finished = asked.filter((i) => i.input.outcome === 'completed').length;
1402
+ out.push({
1403
+ rule: 'session/input-required',
1404
+ severity: 'info',
1405
+ step: asked[0].step,
1406
+ message: `${asked.length === 1 ? `${asked[0].tool} asked` : `${asked.length} tools asked`} for input mid-call (input_required, 2026-07-28). toolmenu ${PAST[answer]} every request, the same way every run, and called again: ${finished} of ${asked.length} then answered with a result the model can read.`,
1407
+ detail: asked.map((i) => `${i.tool} (step ${i.step}): ${inputNote(i.input, answer)}`),
1408
+ fix: 'Nothing to do for the ones that answered. The findings next to this one cover the rest.',
1409
+ });
1410
+ }
1411
+ for (const { step, tool, input } of asked) {
1412
+ if (input.outcome === 'loop' || input.outcome === 'gave-up') {
1413
+ out.push({
1414
+ rule: 'session/input-loop',
1415
+ severity: 'warn',
1416
+ step,
1417
+ tool,
1418
+ message: input.outcome === 'loop'
1419
+ ? `${tool} asks the same thing again after a ${answer}, every time: an agent whose user says no gets the question back instead of an answer, and can't finish the call. toolmenu stopped after ${MAX_ROUNDS} rounds.`
1420
+ : `${tool} was still asking for input after ${MAX_ROUNDS} rounds of ${answer}s, each time something else. An agent whose user says no can't finish the call.`,
1421
+ detail: [inputNote(input, answer)],
1422
+ fix: `Treat a ${answer} as an answer: return a result that says the user ${answer === 'decline' ? 'declined' : 'dismissed it'} (isError: true is fine), so the model can tell the user and move on.`,
1423
+ });
1424
+ }
1425
+ else if (input.outcome === 'decline-failed') {
1426
+ out.push({
1427
+ rule: 'session/decline-fails',
1428
+ severity: 'warn',
1429
+ step,
1430
+ tool,
1431
+ message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${serverWords(input.error, 120)}”). The model gets an error from the client instead of a result it can read and explain to the user.`,
1432
+ detail: [inputNote(input, answer)],
1433
+ fix: `Return a tool result on a ${answer} (isError: true with a sentence like "the user declined"), not a JSON-RPC error.`,
1434
+ });
1435
+ }
1436
+ else if (input.outcome === 'undeclared') {
1437
+ out.push({
1438
+ rule: 'session/undeclared-input',
1439
+ severity: 'warn',
1440
+ step,
1441
+ tool,
1442
+ message: `${tool} asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare among its client capabilities. Servers must not ask a client for what it doesn't support: they should return MissingRequiredClientCapabilityError (-32021) naming it, so the client can say why the call can't run.${(input.undeclared ?? []).some((k) => k === 'roots' || k === 'sampling') ? ' 2026-07-28 also deprecates roots and sampling (SEP-2577).' : ''}`,
1443
+ detail: [inputNote(input, answer)],
1444
+ fix: `Check the request's io.modelcontextprotocol/clientCapabilities before asking, and return -32021 with requiredCapabilities when it lacks ${kindList(input.undeclared ?? [])}.${(input.undeclared ?? []).includes('roots') ? ' For roots, take the directory as a tool parameter instead.' : ''}`,
1445
+ });
1446
+ }
1447
+ }
1448
+ const unmarked = inputs.filter((i) => i.input.missingResultType);
1449
+ if (unmarked.length) {
1450
+ out.push({
1451
+ rule: 'session/result-type',
1452
+ severity: 'error',
1453
+ step: unmarked[0].step,
1454
+ ...(unmarked.length === 1 ? { tool: unmarked[0].tool } : {}),
1455
+ message: `${unmarked.length === 1 ? `${unmarked[0].tool}'s tools/call result has` : `${unmarked.length} tools' tools/call results have`} no resultType. 2026-07-28 servers must send it ("complete", or "input_required" when asking for input), and the official SDK client rejects a result without it ("missing required resultType"), so agents on it get an error where the tool answered.`,
1456
+ detail: unmarked.map((i) => `${i.tool} (step ${i.step})`),
1457
+ fix: 'Add resultType: "complete" to every tools/call result (the official SDKs add it; a hand-written server has to).',
1458
+ });
1459
+ }
1460
+ return out;
1461
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.14.1",
3
+ "version": "0.16.0",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
5
  "author": "Nikša Perović (https://niksa.me)",
6
6
  "main": "dist/index.js",