toolmenu 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/auth.js +2 -1
- package/dist/auto.js +33 -2
- package/dist/breakdown.js +3 -31
- package/dist/catalog.d.ts +1 -1
- package/dist/catalog.js +4 -3
- package/dist/cli.js +42 -6
- package/dist/config.d.ts +9 -0
- package/dist/config.js +11 -0
- package/dist/connect.d.ts +1 -0
- package/dist/connect.js +46 -16
- package/dist/diff.js +51 -13
- package/dist/explain.d.ts +9 -4
- package/dist/explain.js +90 -25
- package/dist/failures.js +5 -4
- package/dist/help.js +2 -1
- package/dist/mrtr.d.ts +27 -6
- package/dist/mrtr.js +116 -37
- package/dist/options.d.ts +4 -0
- package/dist/options.js +1 -0
- package/dist/probe.d.ts +8 -0
- package/dist/probe.js +9 -1
- package/dist/report.js +2 -1
- package/dist/rules/gated.js +5 -5
- package/dist/rules/handles.d.ts +12 -0
- package/dist/rules/handles.js +88 -0
- package/dist/rules/headers.js +4 -4
- package/dist/rules/identities.d.ts +23 -0
- package/dist/rules/identities.js +84 -0
- package/dist/rules/index.js +5 -0
- package/dist/rules/refs.js +10 -78
- package/dist/rules/rule.d.ts +2 -0
- package/dist/rules/search.js +3 -2
- package/dist/rules/spec.js +2 -1
- package/dist/schema-ref.d.ts +35 -0
- package/dist/schema-ref.js +178 -0
- package/dist/search.d.ts +8 -0
- package/dist/search.js +4 -1
- package/dist/session.d.ts +17 -0
- package/dist/session.js +156 -66
- package/dist/snapshot.d.ts +10 -1
- package/dist/snapshot.js +20 -3
- package/package.json +1 -1
package/dist/session.js
CHANGED
|
@@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
3
|
import { cacheBreak, canonical, compareMenus } from './compare.js';
|
|
4
4
|
import { listTools } from './connect.js';
|
|
5
|
-
import { callWithInput, MAX_ROUNDS } from './mrtr.js';
|
|
5
|
+
import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
|
|
6
6
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
7
7
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
8
8
|
import { varianceFinding } from './rules/determinism.js';
|
|
@@ -78,8 +78,14 @@ export async function session(target, scenario, options = {}) {
|
|
|
78
78
|
const raw = [];
|
|
79
79
|
// 2026-07-28: every call that asked for input, for the input findings at the end.
|
|
80
80
|
const inputs = [];
|
|
81
|
+
// --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
|
|
82
|
+
const handleErrors = [];
|
|
81
83
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
82
84
|
// a probe with a different seed would count ordering variance as a change.
|
|
85
|
+
// How every elicitation is answered: the same each run (--elicitation). Session
|
|
86
|
+
// connections declare elicitation; snapshot's don't, so its menu is the one 0.14 saw.
|
|
87
|
+
const answer = target.elicitation ?? 'decline';
|
|
88
|
+
target = { ...target, elicitation: answer };
|
|
83
89
|
const mainTarget = seeded(target, MAIN_SEED);
|
|
84
90
|
const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
|
|
85
91
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
@@ -169,7 +175,9 @@ export async function session(target, scenario, options = {}) {
|
|
|
169
175
|
// this one when that happens.
|
|
170
176
|
let probedAfter = probes ? 0 : undefined;
|
|
171
177
|
const lost = (why, index) => {
|
|
172
|
-
|
|
178
|
+
// 2026-07-28 has no protocol sessions: "unknown session" from such a server is
|
|
179
|
+
// about a session it minted itself, a handle the model passes (SEP-2567).
|
|
180
|
+
if (probedAfter === undefined || conn.era === 'modern' || !SESSION_LOST.test(why))
|
|
173
181
|
return false;
|
|
174
182
|
// The server's own words, not the transport's wrapping of its JSON-RPC error.
|
|
175
183
|
const said = serverWords(why);
|
|
@@ -277,10 +285,12 @@ export async function session(target, scenario, options = {}) {
|
|
|
277
285
|
callWaited.ms = 0;
|
|
278
286
|
// 2026-07-28: a tool that asks for input gets every elicitation declined
|
|
279
287
|
// and is called again (mrtr.ts); on 2025 it's a plain call.
|
|
280
|
-
const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation:
|
|
288
|
+
const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: answer }), {
|
|
281
289
|
waits,
|
|
282
290
|
waited: callWaited,
|
|
283
|
-
|
|
291
|
+
// A 429 means the server didn't run that request, but after an input round the
|
|
292
|
+
// tool's first leg did run: only a read-only tool is called again from scratch.
|
|
293
|
+
error: (e) => (httpStatus(e) === 429 && (readOnly || !(e instanceof AfterInputError))) || (readOnly && tooMany(e)),
|
|
284
294
|
result: (c) => readOnly && c.result?.isError === true && RATE_LIMITED.test(errorText(c.result)),
|
|
285
295
|
});
|
|
286
296
|
if (call.input.outcome !== 'none' || call.input.missingResultType) {
|
|
@@ -288,62 +298,83 @@ export async function session(target, scenario, options = {}) {
|
|
|
288
298
|
inputs.push({ step: index, tool: step.tool, input: call.input });
|
|
289
299
|
}
|
|
290
300
|
if (!call.result) {
|
|
291
|
-
// It never answered: still asking, asking for what wasn't declared, or
|
|
301
|
+
// It never answered: still asking, asking for what wasn't declared, or
|
|
302
|
+
// failing on the decline. Said by the input findings, not as a failed call;
|
|
303
|
+
// the menu is still listed after it, as after any step.
|
|
292
304
|
record.outcome = 'needs-input';
|
|
293
|
-
record.note = inputNote(call.input,
|
|
294
|
-
throw new InputStopped();
|
|
305
|
+
record.note = inputNote(call.input, answer);
|
|
295
306
|
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
record.
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
307
|
+
else {
|
|
308
|
+
const result = call.result;
|
|
309
|
+
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
310
|
+
if (call.input.rounds && !result.isError)
|
|
311
|
+
record.note = inputNote(call.input, answer);
|
|
312
|
+
if (step.probe) {
|
|
313
|
+
// A handle that can't exist: a tool error is the answer the spec asks for.
|
|
314
|
+
const text = result.isError ? errorText(result) : '';
|
|
315
|
+
record.note = result.isError
|
|
316
|
+
? `called with a made-up ${step.probe.kind}; it answered with a tool error${text ? `: ${clip(text, 100)}` : ''}`
|
|
317
|
+
: `called with a made-up ${step.probe.kind}; it answered as if that ${step.probe.kind} existed`;
|
|
318
|
+
}
|
|
319
|
+
else if (result.isError) {
|
|
320
|
+
const text = errorText(result);
|
|
321
|
+
if (RATE_LIMITED.test(text)) {
|
|
322
|
+
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
323
|
+
record.status = 'failed';
|
|
324
|
+
record.note = 'rate-limited';
|
|
325
|
+
done();
|
|
326
|
+
break;
|
|
327
|
+
}
|
|
328
|
+
if (call.input.rounds) {
|
|
329
|
+
// The tool's answer to the decline: how a tool should say no. Not a
|
|
330
|
+
// setup failure, whatever its words ("the user declined to connect GitHub").
|
|
331
|
+
record.note = `${inputNote(call.input, answer)}, with a tool error: ${clip(text, 100)}`;
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
record.note = text ? `the tool returned an error: ${clip(text, 120)}` : 'the tool returned an error';
|
|
335
|
+
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
336
|
+
// A setup failure is reported once, for the whole run (session/untested).
|
|
337
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
338
|
+
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
339
|
+
}
|
|
308
340
|
}
|
|
309
|
-
const short = clip(text, 120);
|
|
310
|
-
record.note = call.input.rounds
|
|
311
|
-
? `${inputNote(call.input, target.elicitation ?? 'decline')}, with a tool error: ${clip(text, 100)}`
|
|
312
|
-
: text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
313
|
-
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
314
|
-
// A setup failure is reported once, for the whole run (session/untested). A
|
|
315
|
-
// tool error that answers a decline is how a tool should say no (input-required).
|
|
316
|
-
if (!SETUP_FAILURES.has(record.failure) && !call.input.rounds)
|
|
317
|
-
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
318
341
|
}
|
|
319
342
|
}
|
|
320
343
|
catch (error) {
|
|
321
|
-
// A
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
344
|
+
// A call that asked for input before it failed still counts among those that asked.
|
|
345
|
+
if (error instanceof AfterInputError && (error.input.rounds || error.input.continuations)) {
|
|
346
|
+
record.input = error.input;
|
|
347
|
+
if (error.input.kinds.length)
|
|
348
|
+
inputs.push({ step: index, tool: step.tool, input: error.input });
|
|
349
|
+
}
|
|
350
|
+
record.status = 'failed';
|
|
351
|
+
record.outcome = 'failed';
|
|
352
|
+
record.note = serverWords(error, 200);
|
|
353
|
+
// A call with a made-up handle ("Unknown session: toolmenu-unknown-session") is about that handle.
|
|
354
|
+
if (!step.probe && lost(record.note, index)) {
|
|
355
|
+
done();
|
|
356
|
+
break;
|
|
357
|
+
}
|
|
358
|
+
if (tooMany(error)) {
|
|
359
|
+
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
360
|
+
record.note = 'rate-limited';
|
|
361
|
+
done();
|
|
362
|
+
break;
|
|
363
|
+
}
|
|
364
|
+
const code = error.code;
|
|
365
|
+
// 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
|
|
366
|
+
if (code === -32021) {
|
|
367
|
+
const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
|
|
368
|
+
if (needed.length)
|
|
369
|
+
record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
|
|
370
|
+
}
|
|
371
|
+
if (step.probe && typeof code === 'number' && code !== -32021) {
|
|
372
|
+
// The server's protocol error to a handle that can't exist: the model gets no reason it can act on.
|
|
373
|
+
if (!handleErrors.some((e) => e.tool === step.tool))
|
|
374
|
+
handleErrors.push({ step: index, tool: step.tool, probe: step.probe, value: step.args[step.probe.param], message: record.note, code });
|
|
375
|
+
record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
|
|
325
376
|
}
|
|
326
377
|
else {
|
|
327
|
-
record.status = 'failed';
|
|
328
|
-
record.outcome = 'failed';
|
|
329
|
-
record.note = serverWords(error, 200);
|
|
330
|
-
if (lost(record.note, index)) {
|
|
331
|
-
done();
|
|
332
|
-
break;
|
|
333
|
-
}
|
|
334
|
-
if (tooMany(error)) {
|
|
335
|
-
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
336
|
-
record.note = 'rate-limited';
|
|
337
|
-
done();
|
|
338
|
-
break;
|
|
339
|
-
}
|
|
340
|
-
const code = error.code;
|
|
341
|
-
// 2026-07-28's MissingRequiredClientCapabilityError says what it needed.
|
|
342
|
-
if (code === -32021) {
|
|
343
|
-
const needed = Object.keys((error.data?.requiredCapabilities ?? {}));
|
|
344
|
-
if (needed.length)
|
|
345
|
-
record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
|
|
346
|
-
}
|
|
347
378
|
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
348
379
|
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
349
380
|
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
@@ -455,12 +486,13 @@ export async function session(target, scenario, options = {}) {
|
|
|
455
486
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
456
487
|
record.tools = current.tools.length;
|
|
457
488
|
record.tokens = current.totalTokens;
|
|
458
|
-
if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
|
|
489
|
+
if (firstTry && !record.changed && record.status === 'ok' && !record.failure && record.outcome !== 'needs-input')
|
|
459
490
|
quiet.add(step.tool);
|
|
460
491
|
done();
|
|
461
492
|
}
|
|
462
493
|
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
463
|
-
raw.push(...inputFindings(inputs,
|
|
494
|
+
raw.push(...inputFindings(inputs, answer));
|
|
495
|
+
raw.push(...handleErrorFindings(handleErrors));
|
|
464
496
|
if (explained.length)
|
|
465
497
|
raw.push(knownVariance(explained));
|
|
466
498
|
if (onWord.length)
|
|
@@ -547,7 +579,7 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
547
579
|
// Never says it unlocks tools and the run didn't call it: nothing suggests tools
|
|
548
580
|
// are missing (Garmin's garmin_sync { domains } downloads data). One it called
|
|
549
581
|
// and stopped is covered by auto.stopped.
|
|
550
|
-
if (!u
|
|
582
|
+
if (!definite(u) && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
|
|
551
583
|
continue;
|
|
552
584
|
const reached = new Set();
|
|
553
585
|
let last;
|
|
@@ -558,7 +590,8 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
558
590
|
};
|
|
559
591
|
for (const record of steps) {
|
|
560
592
|
const step = scenario.steps[record.index - 1];
|
|
561
|
-
|
|
593
|
+
// A call that never finished its input rounds didn't unlock anything.
|
|
594
|
+
if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure || record.outcome === 'needs-input')
|
|
562
595
|
continue;
|
|
563
596
|
last = record.index;
|
|
564
597
|
add(step.args[u.param]);
|
|
@@ -1071,6 +1104,28 @@ function settle(raw, options) {
|
|
|
1071
1104
|
// description to back them up.
|
|
1072
1105
|
// `namespace` is weak too: Kubernetes and Pinecone take one on every read.
|
|
1073
1106
|
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
|
|
1107
|
+
// Parameter names that only ever name sets of tools. Not `domains` (Garmin's
|
|
1108
|
+
// garmin_sync { domains: [sleep, hrv] } names data), nor `capabilities`, `packs` or
|
|
1109
|
+
// `bundles` (a device's capabilities, a price bundle): those need the tool to say so.
|
|
1110
|
+
const TOOLSET_PARAM = /^(?:toolsets?|tool_?sets?)$/i;
|
|
1111
|
+
/**
|
|
1112
|
+
* An unlock that surely unlocks tools: it says so (name or description), or its
|
|
1113
|
+
* parameter is named for tool sets outright (toolsets). One that only has an enum of domains, and
|
|
1114
|
+
* doesn't say it unlocks anything, may be data: session --auto still tries it once,
|
|
1115
|
+
* but menu/gated and session/unlock-coverage don't charge it.
|
|
1116
|
+
*/
|
|
1117
|
+
export function definite(u) {
|
|
1118
|
+
return u.claims || (!!u.param && TOOLSET_PARAM.test(u.param) && !dispatches(u.tool));
|
|
1119
|
+
}
|
|
1120
|
+
/** What a dispatcher takes besides the set: the tool or operation to run in it, or that call's arguments. */
|
|
1121
|
+
const DISPATCH_PARAM = /^(?:tool(?:_?name)?|command|sub_?command|operation|method|action|parameters|params|arguments|args)$/i;
|
|
1122
|
+
/**
|
|
1123
|
+
* A dispatcher (`call_tool { toolset, tool, arguments }`) names a tool set to run
|
|
1124
|
+
* one of its tools, not to unlock it: its `toolset` alone doesn't make it an unlock.
|
|
1125
|
+
*/
|
|
1126
|
+
function dispatches(tool) {
|
|
1127
|
+
return Object.keys(tool?.inputSchema?.properties ?? {}).some((p) => DISPATCH_PARAM.test(p));
|
|
1128
|
+
}
|
|
1074
1129
|
const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
|
|
1075
1130
|
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
1076
1131
|
// A description about loading tools: "Adds the audit tools", "enables more
|
|
@@ -1357,9 +1412,6 @@ export function starterScenario(menu, options = {}) {
|
|
|
1357
1412
|
}
|
|
1358
1413
|
return lines.join('\n') + '\n';
|
|
1359
1414
|
}
|
|
1360
|
-
/** Thrown inside a call step when the tool never completed its input rounds: not a failed request. */
|
|
1361
|
-
class InputStopped extends Error {
|
|
1362
|
-
}
|
|
1363
1415
|
const PAST = { decline: 'declined', cancel: 'cancelled' };
|
|
1364
1416
|
const KIND_WORDS = { form: 'a form', url: 'a URL to open', roots: 'roots', sampling: 'sampling', other: 'something else' };
|
|
1365
1417
|
function kindList(kinds) {
|
|
@@ -1381,14 +1433,37 @@ function inputNote(input, answer) {
|
|
|
1381
1433
|
return `${asked}; asked the same again after each ${answer}, stopped after ${rounds}`;
|
|
1382
1434
|
case 'gave-up':
|
|
1383
1435
|
return `${asked}; still asking after ${rounds}`;
|
|
1436
|
+
case 'unfinished': {
|
|
1437
|
+
const work = `kept answering input_required with only its requestState (${input.continuations ?? 0} rounds), never a result`;
|
|
1438
|
+
return input.kinds.length ? `${asked}; ${PAST[answer]} ${times(input.rounds)}, then ${work}` : `asked for nothing, but ${work}`;
|
|
1439
|
+
}
|
|
1384
1440
|
case 'undeclared':
|
|
1385
1441
|
return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
|
|
1386
1442
|
case 'decline-failed':
|
|
1387
|
-
return `${asked}; the retry with the ${answer} failed: ${
|
|
1443
|
+
return `${asked}; the retry with the ${answer} failed: ${clip(input.error?.message ?? '', 160)}`;
|
|
1444
|
+
case 'bad-state':
|
|
1445
|
+
return `${input.kinds.length ? `${asked}, with` : 'answered input_required with'} a requestState that isn't a string, so it can't be sent back as it came`;
|
|
1388
1446
|
default:
|
|
1389
1447
|
return asked;
|
|
1390
1448
|
}
|
|
1391
1449
|
}
|
|
1450
|
+
/**
|
|
1451
|
+
* Handles that can't exist answered with a protocol error (2026-07-28, SEP-2567):
|
|
1452
|
+
* "a call against an expired or unknown handle should return a tool execution
|
|
1453
|
+
* error that says so, so the model can recover by creating a new one." One
|
|
1454
|
+
* finding per tool.
|
|
1455
|
+
*/
|
|
1456
|
+
function handleErrorFindings(errors) {
|
|
1457
|
+
return errors.map((e) => ({
|
|
1458
|
+
rule: 'session/handle-error',
|
|
1459
|
+
severity: 'warn',
|
|
1460
|
+
confidence: 'unsure',
|
|
1461
|
+
step: e.step,
|
|
1462
|
+
tool: e.tool,
|
|
1463
|
+
message: `${e.tool} answers a ${e.probe.kind} that doesn't exist with a protocol error (${e.code}: “${clip(e.message, 120)}”), not a tool error. A model whose ${e.probe.kind} expired gets an error from its client instead of a result it can read, so it can't tell it should start a new one. toolmenu passed ${e.probe.param}: ${JSON.stringify(e.value)}, which no ${e.probe.kind} has.`,
|
|
1464
|
+
fix: `Return a tool result with isError: true that says the ${e.probe.kind} is unknown or expired and how to get a new one ("Session not found or expired: call start_session again").`,
|
|
1465
|
+
}));
|
|
1466
|
+
}
|
|
1392
1467
|
/**
|
|
1393
1468
|
* What tools that asked for input did with the answer (2026-07-28 multi
|
|
1394
1469
|
* round-trip requests). One info finding for all of them, and a warning or error
|
|
@@ -1396,7 +1471,7 @@ function inputNote(input, answer) {
|
|
|
1396
1471
|
*/
|
|
1397
1472
|
function inputFindings(inputs, answer) {
|
|
1398
1473
|
const out = [];
|
|
1399
|
-
const asked = inputs.filter((i) => i.input.outcome !== 'none');
|
|
1474
|
+
const asked = inputs.filter((i) => i.input.outcome !== 'none' && i.input.kinds.length);
|
|
1400
1475
|
if (asked.length) {
|
|
1401
1476
|
const finished = asked.filter((i) => i.input.outcome === 'completed').length;
|
|
1402
1477
|
out.push({
|
|
@@ -1408,8 +1483,8 @@ function inputFindings(inputs, answer) {
|
|
|
1408
1483
|
fix: 'Nothing to do for the ones that answered. The findings next to this one cover the rest.',
|
|
1409
1484
|
});
|
|
1410
1485
|
}
|
|
1411
|
-
for (const { step, tool, input } of
|
|
1412
|
-
if (input.outcome === 'loop' || input.outcome === 'gave-up') {
|
|
1486
|
+
for (const { step, tool, input } of inputs.filter((i) => i.input.outcome !== 'none')) {
|
|
1487
|
+
if (input.outcome === 'loop' || input.outcome === 'gave-up' || input.outcome === 'unfinished') {
|
|
1413
1488
|
out.push({
|
|
1414
1489
|
rule: 'session/input-loop',
|
|
1415
1490
|
severity: 'warn',
|
|
@@ -1417,9 +1492,13 @@ function inputFindings(inputs, answer) {
|
|
|
1417
1492
|
tool,
|
|
1418
1493
|
message: input.outcome === 'loop'
|
|
1419
1494
|
? `${tool} asks the same thing again after a ${answer}, every time: an agent whose user says no gets the question back instead of an answer, and can't finish the call. toolmenu stopped after ${MAX_ROUNDS} rounds.`
|
|
1420
|
-
:
|
|
1495
|
+
: input.outcome === 'gave-up'
|
|
1496
|
+
? `${tool} was still asking for input after ${MAX_ROUNDS} rounds of ${answer}s, not always for the same thing. An agent whose user says no can't finish the call.`
|
|
1497
|
+
: `${tool} ${input.kinds.length ? `took the ${answer}, then ` : ''}kept answering input_required with only its requestState, asking for nothing more, and never returned a result (toolmenu stopped after ${input.continuations ?? 0} rounds, at most ${MAX_CONTINUATIONS} or 3 × the request timeout). The client keeps calling it again and the agent never gets an answer.`,
|
|
1421
1498
|
detail: [inputNote(input, answer)],
|
|
1422
|
-
fix:
|
|
1499
|
+
fix: input.outcome === 'unfinished'
|
|
1500
|
+
? 'Return a result once the work is done, or a tool error if it can\'t finish; for long work, hand back a handle the agent can check with another tool.'
|
|
1501
|
+
: `Treat a ${answer} as an answer: return a result that says the user ${answer === 'decline' ? 'declined' : 'dismissed it'} (isError: true is fine), so the model can tell the user and move on.`,
|
|
1423
1502
|
});
|
|
1424
1503
|
}
|
|
1425
1504
|
else if (input.outcome === 'decline-failed') {
|
|
@@ -1428,11 +1507,22 @@ function inputFindings(inputs, answer) {
|
|
|
1428
1507
|
severity: 'warn',
|
|
1429
1508
|
step,
|
|
1430
1509
|
tool,
|
|
1431
|
-
message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${
|
|
1510
|
+
message: `${tool} fails with a protocol error when the user ${answer === 'decline' ? 'declines' : 'dismisses'} its request (“${clip(input.error?.message ?? '', 120)}”). The model gets an error from the client instead of a result it can read and explain to the user.`,
|
|
1432
1511
|
detail: [inputNote(input, answer)],
|
|
1433
1512
|
fix: `Return a tool result on a ${answer} (isError: true with a sentence like "the user declined"), not a JSON-RPC error.`,
|
|
1434
1513
|
});
|
|
1435
1514
|
}
|
|
1515
|
+
else if (input.outcome === 'bad-state') {
|
|
1516
|
+
out.push({
|
|
1517
|
+
rule: 'session/result-type',
|
|
1518
|
+
severity: 'error',
|
|
1519
|
+
step,
|
|
1520
|
+
tool,
|
|
1521
|
+
message: `${tool} answered input_required with a requestState that isn't a string. 2026-07-28 makes it a string the client sends back byte for byte; a client can't echo anything else as it came, so the call can't go on (calling again without it would run the tool from scratch).`,
|
|
1522
|
+
detail: [inputNote(input, answer)],
|
|
1523
|
+
fix: 'Send requestState as a string: serialize (and sign, if it matters) whatever state the tool needs, and parse it when the call comes back.',
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1436
1526
|
else if (input.outcome === 'undeclared') {
|
|
1437
1527
|
out.push({
|
|
1438
1528
|
rule: 'session/undeclared-input',
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type Target } from './connect.js';
|
|
2
|
+
import { type Identity } from './probe.js';
|
|
3
|
+
import type { IdentityMenu } from './rules/identities.js';
|
|
2
4
|
import { type RuleSettings } from './rules/index.js';
|
|
3
5
|
import { type CatalogOptions } from './catalog.js';
|
|
4
6
|
import type { Routes } from './routes.js';
|
|
@@ -15,13 +17,20 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
15
17
|
processes?: number;
|
|
16
18
|
/** Also read the operations behind a catalog search tool or command routers (true, or which ones and how). */
|
|
17
19
|
catalog?: boolean | CatalogOptions;
|
|
20
|
+
/**
|
|
21
|
+
* snapshot --as: list the menu once per identity. The first is the main one: its
|
|
22
|
+
* menu is the one saved and checked; the others are compared with it.
|
|
23
|
+
*/
|
|
24
|
+
identities?: Identity[];
|
|
18
25
|
}
|
|
19
26
|
export interface SnapshotResult {
|
|
20
27
|
menu: Menu;
|
|
21
28
|
findings: Finding[];
|
|
29
|
+
/** snapshot --as: each identity's tools and tokens, or why it couldn't be listed. */
|
|
30
|
+
identities?: IdentityMenu[];
|
|
22
31
|
}
|
|
23
32
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
24
|
-
export declare function snapshot(
|
|
33
|
+
export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
|
25
34
|
/**
|
|
26
35
|
* How this run reached the server, as the end of a command line, for next steps:
|
|
27
36
|
* the options that got it past the door (names only; values are the user's) and
|
package/dist/snapshot.js
CHANGED
|
@@ -2,13 +2,15 @@ import { listTools } from './connect.js';
|
|
|
2
2
|
import { patiently, tooMany } from './failures.js';
|
|
3
3
|
import { ConnectError } from './explain.js';
|
|
4
4
|
import { buildMenu } from './menu.js';
|
|
5
|
-
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
5
|
+
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
6
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
7
7
|
import { catalogFindings, readCatalog } from './catalog.js';
|
|
8
8
|
import { SEVERITY_RANK } from './types.js';
|
|
9
9
|
import { shellWord } from './init.js';
|
|
10
10
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
11
|
-
export async function snapshot(
|
|
11
|
+
export async function snapshot(given, options = {}) {
|
|
12
|
+
const [mainIdentity, ...otherIdentities] = options.identities ?? [];
|
|
13
|
+
const target = asIdentity(given, mainIdentity);
|
|
12
14
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
13
15
|
// out and they're sent again (CI behind a shared rate limit).
|
|
14
16
|
const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
@@ -43,6 +45,19 @@ export async function snapshot(target, options = {}) {
|
|
|
43
45
|
// After the main process has exited: a server that holds a file or a port
|
|
44
46
|
// shouldn't have to share it with its own probe.
|
|
45
47
|
const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000);
|
|
48
|
+
// Each other identity's menu, from its own process (stdio) or connection (HTTP).
|
|
49
|
+
const identities = mainIdentity
|
|
50
|
+
? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
|
|
51
|
+
: undefined;
|
|
52
|
+
for (const identity of otherIdentities) {
|
|
53
|
+
try {
|
|
54
|
+
const m = await probeMenu(seeded(asIdentity(given, identity), MAIN_SEED), options.timeoutMs ?? 30_000);
|
|
55
|
+
identities.push({ name: identity.name, tools: m.tools, tokens: m.totalTokens, cacheScope: m.listMeta?.cacheScope });
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
identities.push({ name: identity.name, error: firstLine(error) });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
46
61
|
const findings = runRules(MENU_RULES, {
|
|
47
62
|
menu,
|
|
48
63
|
secondList,
|
|
@@ -60,6 +75,7 @@ export async function snapshot(target, options = {}) {
|
|
|
60
75
|
wrapper: isContainerWrapper(target),
|
|
61
76
|
serverInfo: connection.server,
|
|
62
77
|
server: serverArgs(target),
|
|
78
|
+
identities,
|
|
63
79
|
}, options);
|
|
64
80
|
// What --catalog called and what it got, under the same settings as the rules
|
|
65
81
|
// (rules: { "catalog/read": "off" }, ignore).
|
|
@@ -73,8 +89,9 @@ export async function snapshot(target, options = {}) {
|
|
|
73
89
|
}
|
|
74
90
|
findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
75
91
|
}
|
|
76
|
-
return { menu, findings };
|
|
92
|
+
return { menu, findings, ...(identities ? { identities } : {}) };
|
|
77
93
|
}
|
|
94
|
+
const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
|
|
78
95
|
/**
|
|
79
96
|
* How this run reached the server, as the end of a command line, for next steps:
|
|
80
97
|
* the options that got it past the door (names only; values are the user's) and
|
package/package.json
CHANGED