toolmenu 0.14.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/breakdown.js +25 -7
- package/dist/catalog.js +4 -0
- package/dist/cli.js +10 -3
- package/dist/connect.d.ts +5 -5
- package/dist/connect.js +26 -1
- package/dist/diff.js +20 -0
- package/dist/explain.d.ts +8 -0
- package/dist/explain.js +51 -7
- package/dist/failures.d.ts +1 -1
- package/dist/failures.js +19 -6
- package/dist/help.js +2 -1
- package/dist/mrtr.d.ts +41 -0
- package/dist/mrtr.js +95 -0
- package/dist/options.d.ts +3 -0
- package/dist/options.js +1 -0
- package/dist/rules/gated.js +4 -1
- package/dist/rules/headers.d.ts +14 -0
- package/dist/rules/headers.js +116 -0
- package/dist/rules/ids.js +2 -1
- package/dist/rules/index.js +6 -0
- package/dist/rules/refs.d.ts +15 -0
- package/dist/rules/refs.js +125 -0
- package/dist/rules/search.d.ts +2 -0
- package/dist/rules/search.js +70 -0
- package/dist/search.d.ts +24 -0
- package/dist/search.js +86 -0
- package/dist/session.d.ts +9 -1
- package/dist/session.js +184 -24
- package/package.json +1 -1
package/dist/session.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
3
|
import { cacheBreak, canonical, compareMenus } from './compare.js';
|
|
4
4
|
import { listTools } from './connect.js';
|
|
5
|
+
import { callWithInput, MAX_ROUNDS } from './mrtr.js';
|
|
5
6
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
7
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
8
|
import { varianceFinding } from './rules/determinism.js';
|
|
@@ -75,6 +76,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
75
76
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
76
77
|
const grace = options.noticeGraceMs ?? 500;
|
|
77
78
|
const raw = [];
|
|
79
|
+
// 2026-07-28: every call that asked for input, for the input findings at the end.
|
|
80
|
+
const inputs = [];
|
|
78
81
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
79
82
|
// a probe with a different seed would count ordering variance as a change.
|
|
80
83
|
const mainTarget = seeded(target, MAIN_SEED);
|
|
@@ -272,13 +275,28 @@ export async function session(target, scenario, options = {}) {
|
|
|
272
275
|
// read-only one is asked again. On the server's word, not the user's.
|
|
273
276
|
const readOnly = tool?.annotations?.readOnlyHint === true;
|
|
274
277
|
callWaited.ms = 0;
|
|
275
|
-
|
|
278
|
+
// 2026-07-28: a tool that asks for input gets every elicitation declined
|
|
279
|
+
// and is called again (mrtr.ts); on 2025 it's a plain call.
|
|
280
|
+
const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: target.elicitation }), {
|
|
276
281
|
waits,
|
|
277
282
|
waited: callWaited,
|
|
278
283
|
error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
|
|
279
|
-
result: (
|
|
284
|
+
result: (c) => readOnly && c.result?.isError === true && RATE_LIMITED.test(errorText(c.result)),
|
|
280
285
|
});
|
|
286
|
+
if (call.input.outcome !== 'none' || call.input.missingResultType) {
|
|
287
|
+
record.input = call.input;
|
|
288
|
+
inputs.push({ step: index, tool: step.tool, input: call.input });
|
|
289
|
+
}
|
|
290
|
+
if (!call.result) {
|
|
291
|
+
// It never answered: still asking, asking for what wasn't declared, or failing on the decline.
|
|
292
|
+
record.outcome = 'needs-input';
|
|
293
|
+
record.note = inputNote(call.input, target.elicitation ?? 'decline');
|
|
294
|
+
throw new InputStopped();
|
|
295
|
+
}
|
|
296
|
+
const result = call.result;
|
|
281
297
|
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
298
|
+
if (call.input.rounds && !result.isError)
|
|
299
|
+
record.note = inputNote(call.input, target.elicitation ?? 'decline');
|
|
282
300
|
if (result.isError) {
|
|
283
301
|
const text = errorText(result);
|
|
284
302
|
if (RATE_LIMITED.test(text)) {
|
|
@@ -289,33 +307,49 @@ export async function session(target, scenario, options = {}) {
|
|
|
289
307
|
break;
|
|
290
308
|
}
|
|
291
309
|
const short = clip(text, 120);
|
|
292
|
-
record.note =
|
|
310
|
+
record.note = call.input.rounds
|
|
311
|
+
? `${inputNote(call.input, target.elicitation ?? 'decline')}, with a tool error: ${clip(text, 100)}`
|
|
312
|
+
: text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
293
313
|
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
294
|
-
// A setup failure is reported once, for the whole run (session/untested).
|
|
295
|
-
|
|
314
|
+
// A setup failure is reported once, for the whole run (session/untested). A
|
|
315
|
+
// tool error that answers a decline is how a tool should say no (input-required).
|
|
316
|
+
if (!SETUP_FAILURES.has(record.failure) && !call.input.rounds)
|
|
296
317
|
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
297
318
|
}
|
|
298
319
|
}
|
|
299
320
|
catch (error) {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
done();
|
|
305
|
-
break;
|
|
321
|
+
// A tool that never answered its input requests is said by the input findings,
|
|
322
|
+
// not as a failed call. The menu is still listed after it, as after any step.
|
|
323
|
+
if (error instanceof InputStopped) {
|
|
324
|
+
// fall through to the listing below
|
|
306
325
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
record.
|
|
310
|
-
|
|
311
|
-
|
|
326
|
+
else {
|
|
327
|
+
record.status = 'failed';
|
|
328
|
+
record.outcome = 'failed';
|
|
329
|
+
record.note = serverWords(error, 200);
|
|
330
|
+
if (lost(record.note, index)) {
|
|
331
|
+
done();
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
if (tooMany(error)) {
|
|
335
|
+
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
336
|
+
record.note = 'rate-limited';
|
|
337
|
+
done();
|
|
338
|
+
break;
|
|
339
|
+
}
|
|
340
|
+
const code = error.code;
|
|
341
|
+
// 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
|
|
342
|
+
if (code === -32021) {
|
|
343
|
+
const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
|
|
344
|
+
if (needed.length)
|
|
345
|
+
record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
|
|
346
|
+
}
|
|
347
|
+
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
348
|
+
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
349
|
+
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
350
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
351
|
+
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
312
352
|
}
|
|
313
|
-
const code = error.code;
|
|
314
|
-
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
315
|
-
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
316
|
-
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
317
|
-
if (!SETUP_FAILURES.has(record.failure))
|
|
318
|
-
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
319
353
|
}
|
|
320
354
|
}
|
|
321
355
|
else if (step.kind === 'wait_for') {
|
|
@@ -426,6 +460,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
426
460
|
done();
|
|
427
461
|
}
|
|
428
462
|
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
463
|
+
raw.push(...inputFindings(inputs, target.elicitation ?? 'decline'));
|
|
429
464
|
if (explained.length)
|
|
430
465
|
raw.push(knownVariance(explained));
|
|
431
466
|
if (onWord.length)
|
|
@@ -490,7 +525,7 @@ function untested(steps, target) {
|
|
|
490
525
|
// Classified from the error's words: a guess, if a good one.
|
|
491
526
|
confidence: 'unsure',
|
|
492
527
|
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
|
|
493
|
-
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '')
|
|
528
|
+
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${clip((list[0].note ?? '').replace(/^the tool returned an error: /, ''), 100)}”)`),
|
|
494
529
|
fix,
|
|
495
530
|
},
|
|
496
531
|
];
|
|
@@ -509,6 +544,11 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
509
544
|
for (const u of unlockers(menu)) {
|
|
510
545
|
if (!u.param || u.values.length === 0)
|
|
511
546
|
continue;
|
|
547
|
+
// Never says it unlocks tools and the run didn't call it: nothing suggests tools
|
|
548
|
+
// are missing (Garmin's garmin_sync { domains } downloads data). One it called
|
|
549
|
+
// and stopped is covered by auto.stopped.
|
|
550
|
+
if (!u.claims && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
|
|
551
|
+
continue;
|
|
512
552
|
const reached = new Set();
|
|
513
553
|
let last;
|
|
514
554
|
const add = (arg) => {
|
|
@@ -1063,13 +1103,31 @@ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
|
1063
1103
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
1064
1104
|
*/
|
|
1065
1105
|
export function clip(text, max) {
|
|
1066
|
-
|
|
1106
|
+
let chars = Array.from(text.replace(/\uFFFD+$/u, ''));
|
|
1107
|
+
if (chars.length <= max)
|
|
1108
|
+
return chars.join('');
|
|
1109
|
+
// A long path loses its middle before the text loses its end: “spawn /tmp/…/x ENOENT”
|
|
1110
|
+
// cut at the end hides both the file and the reason.
|
|
1111
|
+
chars = Array.from(shortenPaths(chars.join('')));
|
|
1067
1112
|
if (chars.length <= max)
|
|
1068
1113
|
return chars.join('');
|
|
1069
1114
|
const cut = chars.slice(0, max - 1).join('');
|
|
1070
1115
|
const space = cut.lastIndexOf(' ');
|
|
1071
1116
|
return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
|
|
1072
1117
|
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Absolute paths longer than 40 characters, kept to their last two parts:
|
|
1120
|
+
* /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
|
|
1121
|
+
*/
|
|
1122
|
+
export function shortenPaths(text) {
|
|
1123
|
+
// Not a URL's path: a path starts after nothing that could be a scheme, host or word.
|
|
1124
|
+
return text.replace(/(?<![\w:/\\.])(?:[A-Za-z]:)?(?:[\\/][^\s\\/"'“”()]+){3,}/g, (path) => {
|
|
1125
|
+
if (path.length <= 40)
|
|
1126
|
+
return path;
|
|
1127
|
+
const sep = path.includes('\\') && !path.includes('/') ? '\\' : '/';
|
|
1128
|
+
return `…${sep}${path.split(/[\\/]/).slice(-2).join(sep)}`;
|
|
1129
|
+
});
|
|
1130
|
+
}
|
|
1073
1131
|
function isWriteLike(t) {
|
|
1074
1132
|
// Named as an unlock: changing the menu is its job, whatever its hints say
|
|
1075
1133
|
// (toolception marks enable_toolset destructive). It's still only suggested,
|
|
@@ -1299,3 +1357,105 @@ export function starterScenario(menu, options = {}) {
|
|
|
1299
1357
|
}
|
|
1300
1358
|
return lines.join('\n') + '\n';
|
|
1301
1359
|
}
|
|
1360
|
+
/** Thrown inside a call step when the tool never completed its input rounds: not a failed request. */
|
|
1361
|
+
class InputStopped extends Error {
|
|
1362
|
+
}
|
|
1363
|
+
const PAST = { decline: 'declined', cancel: 'cancelled' };
|
|
1364
|
+
const KIND_WORDS = { form: 'a form', url: 'a URL to open', roots: 'roots', sampling: 'sampling', other: 'something else' };
|
|
1365
|
+
function kindList(kinds) {
|
|
1366
|
+
const words = kinds.map((k) => KIND_WORDS[k]);
|
|
1367
|
+
return words.length <= 1 ? words.join('') : `${words.slice(0, -1).join(', ')} and ${words[words.length - 1]}`;
|
|
1368
|
+
}
|
|
1369
|
+
/** once, twice, 3 times. */
|
|
1370
|
+
function times(n) {
|
|
1371
|
+
return n === 1 ? 'once' : n === 2 ? 'twice' : `${n} times`;
|
|
1372
|
+
}
|
|
1373
|
+
/** The step's note for a call that asked for input. */
|
|
1374
|
+
function inputNote(input, answer) {
|
|
1375
|
+
const asked = `asked for ${kindList(input.kinds)}`;
|
|
1376
|
+
const rounds = `${input.rounds} round${input.rounds === 1 ? '' : 's'}`;
|
|
1377
|
+
switch (input.outcome) {
|
|
1378
|
+
case 'completed':
|
|
1379
|
+
return `${asked}; ${PAST[answer]} ${times(input.rounds)}, then answered`;
|
|
1380
|
+
case 'loop':
|
|
1381
|
+
return `${asked}; asked the same again after each ${answer}, stopped after ${rounds}`;
|
|
1382
|
+
case 'gave-up':
|
|
1383
|
+
return `${asked}; still asking after ${rounds}`;
|
|
1384
|
+
case 'undeclared':
|
|
1385
|
+
return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
|
|
1386
|
+
case 'decline-failed':
|
|
1387
|
+
return `${asked}; the retry with the ${answer} failed: ${serverWords(input.error, 160)}`;
|
|
1388
|
+
default:
|
|
1389
|
+
return asked;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
/**
|
|
1393
|
+
* What tools that asked for input did with the answer (2026-07-28 multi
|
|
1394
|
+
* round-trip requests). One info finding for all of them, and a warning or error
|
|
1395
|
+
* for each that an agent can't get past.
|
|
1396
|
+
*/
|
|
1397
|
+
function inputFindings(inputs, answer) {
|
|
1398
|
+
const out = [];
|
|
1399
|
+
const asked = inputs.filter((i) => i.input.outcome !== 'none');
|
|
1400
|
+
if (asked.length) {
|
|
1401
|
+
const finished = asked.filter((i) => i.input.outcome === 'completed').length;
|
|
1402
|
+
out.push({
|
|
1403
|
+
rule: 'session/input-required',
|
|
1404
|
+
severity: 'info',
|
|
1405
|
+
step: asked[0].step,
|
|
1406
|
+
message: `${asked.length === 1 ? `${asked[0].tool} asked` : `${asked.length} tools asked`} for input mid-call (input_required, 2026-07-28). toolmenu ${PAST[answer]} every request, the same way every run, and called again: ${finished} of ${asked.length} then answered with a result the model can read.`,
|
|
1407
|
+
detail: asked.map((i) => `${i.tool} (step ${i.step}): ${inputNote(i.input, answer)}`),
|
|
1408
|
+
fix: 'Nothing to do for the ones that answered. The findings next to this one cover the rest.',
|
|
1409
|
+
});
|
|
1410
|
+
}
|
|
1411
|
+
for (const { step, tool, input } of asked) {
|
|
1412
|
+
if (input.outcome === 'loop' || input.outcome === 'gave-up') {
|
|
1413
|
+
out.push({
|
|
1414
|
+
rule: 'session/input-loop',
|
|
1415
|
+
severity: 'warn',
|
|
1416
|
+
step,
|
|
1417
|
+
tool,
|
|
1418
|
+
message: input.outcome === 'loop'
|
|
1419
|
+
? `${tool} asks the same thing again after a ${answer}, every time: an agent whose user says no gets the question back instead of an answer, and can't finish the call. toolmenu stopped after ${MAX_ROUNDS} rounds.`
|
|
1420
|
+
: `${tool} was still asking for input after ${MAX_ROUNDS} rounds of ${answer}s, each time something else. An agent whose user says no can't finish the call.`,
|
|
1421
|
+
detail: [inputNote(input, answer)],
|
|
1422
|
+
fix: `Treat a ${answer} as an answer: return a result that says the user ${answer === 'decline' ? 'declined' : 'dismissed it'} (isError: true is fine), so the model can tell the user and move on.`,
|
|
1423
|
+
});
|
|
1424
|
+
}
|
|
1425
|
+
else if (input.outcome === 'decline-failed') {
|
|
1426
|
+
out.push({
|
|
1427
|
+
rule: 'session/decline-fails',
|
|
1428
|
+
severity: 'warn',
|
|
1429
|
+
step,
|
|
1430
|
+
tool,
|
|
1431
|
+
message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${serverWords(input.error, 120)}”). The model gets an error from the client instead of a result it can read and explain to the user.`,
|
|
1432
|
+
detail: [inputNote(input, answer)],
|
|
1433
|
+
fix: `Return a tool result on a ${answer} (isError: true with a sentence like "the user declined"), not a JSON-RPC error.`,
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
else if (input.outcome === 'undeclared') {
|
|
1437
|
+
out.push({
|
|
1438
|
+
rule: 'session/undeclared-input',
|
|
1439
|
+
severity: 'warn',
|
|
1440
|
+
step,
|
|
1441
|
+
tool,
|
|
1442
|
+
message: `${tool} asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare among its client capabilities. Servers must not ask a client for what it doesn't support: they should return MissingRequiredClientCapabilityError (-32021) naming it, so the client can say why the call can't run.${(input.undeclared ?? []).some((k) => k === 'roots' || k === 'sampling') ? ' 2026-07-28 also deprecates roots and sampling (SEP-2577).' : ''}`,
|
|
1443
|
+
detail: [inputNote(input, answer)],
|
|
1444
|
+
fix: `Check the request's io.modelcontextprotocol/clientCapabilities before asking, and return -32021 with requiredCapabilities when it lacks ${kindList(input.undeclared ?? [])}.${(input.undeclared ?? []).includes('roots') ? ' For roots, take the directory as a tool parameter instead.' : ''}`,
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
const unmarked = inputs.filter((i) => i.input.missingResultType);
|
|
1449
|
+
if (unmarked.length) {
|
|
1450
|
+
out.push({
|
|
1451
|
+
rule: 'session/result-type',
|
|
1452
|
+
severity: 'error',
|
|
1453
|
+
step: unmarked[0].step,
|
|
1454
|
+
...(unmarked.length === 1 ? { tool: unmarked[0].tool } : {}),
|
|
1455
|
+
message: `${unmarked.length === 1 ? `${unmarked[0].tool}'s tools/call result has` : `${unmarked.length} tools' tools/call results have`} no resultType. 2026-07-28 servers must send it ("complete", or "input_required" when asking for input), and the official SDK client rejects a result without it ("missing required resultType"), so agents on it get an error where the tool answered.`,
|
|
1456
|
+
detail: unmarked.map((i) => `${i.tool} (step ${i.step})`),
|
|
1457
|
+
fix: 'Add resultType: "complete" to every tools/call result (the official SDKs add it; a hand-written server has to).',
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
return out;
|
|
1461
|
+
}
|
package/package.json
CHANGED