toolmenu 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -6
- package/dist/auto.js +74 -2
- package/dist/cli.js +42 -1
- package/dist/config.d.ts +11 -1
- package/dist/config.js +11 -0
- package/dist/failures.js +1 -1
- package/dist/help.js +2 -1
- package/dist/options.d.ts +4 -0
- package/dist/options.js +1 -0
- package/dist/probe.d.ts +8 -0
- package/dist/probe.js +13 -1
- package/dist/rules/description.d.ts +3 -0
- package/dist/rules/description.js +32 -11
- package/dist/rules/handles.d.ts +12 -0
- package/dist/rules/handles.js +88 -0
- package/dist/rules/identities.d.ts +23 -0
- package/dist/rules/identities.js +84 -0
- package/dist/rules/index.js +5 -0
- package/dist/rules/rule.d.ts +2 -0
- package/dist/rules/spec.js +2 -1
- package/dist/session.d.ts +6 -0
- package/dist/session.js +55 -8
- package/dist/snapshot.d.ts +10 -1
- package/dist/snapshot.js +20 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -188,7 +188,8 @@ serves, and 23 issue and pull-request tools that only the dynamic mode has.
|
|
|
188
188
|
| `session/assumed-read-only` | info | Tools called on your word (`--assume-read-only`, `assume_read_only`) although the server doesn't mark them readOnlyHint |
|
|
189
189
|
| `session/unlock-coverage` | warn | An unlock whose enum lists its values, called with only some of them: the tools behind the rest were never seen, so a baseline from the session misses them. With `--union-out` or `--auto`, an unlock never called at all too, with why `--auto` skipped it |
|
|
190
190
|
| `session/nothing-called` | warn | `--auto` called no tools (all open-world, missing values, or not read-only): the run only listed the menu |
|
|
191
|
-
| `session/session-lost` | error | The server ended the session after toolmenu opened a second connection with the same credentials (servers with one session per client). The run stops there; rerun with `--processes 1` |
|
|
191
|
+
| `session/session-lost` | error | The server ended the session after toolmenu opened a second connection with the same credentials (servers with one session per client; 2025-era servers only). The run stops there; rerun with `--processes 1` |
|
|
192
|
+
| `session/handle-error` | warn | 2026-07-28: `--auto` called a read-only tool with a handle that can't exist (`toolmenu-unknown-session`), and it answered with a protocol error instead of a tool error the model can read |
|
|
192
193
|
| `session/rate-limited` | error | The server kept refusing requests for being too many (a 429) after toolmenu waited about a minute, or a tool not marked `readOnlyHint` said it was rate-limited (it isn't called again). The run stops there, with the steps it didn't check; give the run its own server instance or raise the limit for it |
|
|
193
194
|
| `session/scope-unchecked` | info | A second server process or connection couldn't start mid-session (a server that holds a file or a port), so whether a change was global wasn't checked. Said once for the run |
|
|
194
195
|
|
|
@@ -382,16 +383,19 @@ server, or from the spec itself. If a rule can't point to one, it doesn't ship.
|
|
|
382
383
|
| `spec/tools-capability` | error | The server doesn't declare the `tools` capability, so clients (the official SDK included) never ask for its tools |
|
|
383
384
|
| `menu/duplicate-name` | error | Two tools with the same name: only one can be called, and some clients reject the list |
|
|
384
385
|
| `naming/route` | error | A `routes.yml` expectation broke: a keyword now matches the wrong tool at least as well as the right one |
|
|
385
|
-
| `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions:
|
|
386
|
+
| `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions: in Claude Code 16,384 characters for a tool loaded through tool search (its default) and 4,096 for one that loads up front, or your `descriptionLimit`. From a real failure: a client cut at 280 characters, and the line that decided routing was at 1,222 |
|
|
386
387
|
| `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary that shows where each one is cut. The model gets a prefix that can read as complete |
|
|
387
388
|
| `description/late-instruction` | info | With the default cut: instructions after character 280, which a client that cuts descriptions short would hide, as one summary. Set `descriptionLimit` and `description/buried` checks your real cut instead. (Before 0.13 this was part of `description/cut`; that id's setting still applies.) |
|
|
388
389
|
| `naming/vague-id` | warn | Parameters called just `id` that don't say *which* thing, as one summary with a suggested name |
|
|
389
390
|
| `ids/authored` | info | An ID the agent must supply that no tool appears to return, so the agent may invent it, one finding per kind of ID (heuristic: IDs from a URL or another tool's text output look the same) |
|
|
391
|
+
| `ids/handle-lifetime` | info | 2026-07-28: a tool that creates a handle another tool takes back (`start_session` → `session_id`) doesn't say how long it lasts (heuristic) |
|
|
390
392
|
| `write/unannotated` | warn | Tools named like writes (`delete_`, `send_`…) with no annotations, as one summary grouped by the annotation each name suggests |
|
|
391
393
|
| `naming/shared-word` | info | Nouns that name different things across tools (`list_team_audits` vs `get_audit_trail`), as one summary |
|
|
392
394
|
| `write/no-dry-run` | info | Destructive tools with no dry-run parameter or preview tool, as one summary |
|
|
393
395
|
| `spec/discover` | info | The server only speaks the 2025 protocol, so 2026-07-28 rules are skipped |
|
|
394
396
|
| `spec/deprecated`, `spec/cache-hints` | info | 2026-07-28: deprecated features still advertised; `ttlMs: 0`, or `cacheScope: "public"` on a server that took credentials |
|
|
397
|
+
| `menu/auth-variance` | info | With `--as` (or `identities` in the config): the tool set differs by identity, what each adds, lacks or defines differently against the first, and its tokens. Allowed by the spec; a baseline of one identity's menu doesn't cover the others. An identity that can't be listed is said here too |
|
|
398
|
+
| `spec/cache-scope-public` | warn | 2026-07-28, with `--as`: the tool set differs by identity, and `tools/list` says `cacheScope: "public"`, which lets shared caches serve one caller's list to another |
|
|
395
399
|
|
|
396
400
|
The naming and ID rules are heuristics, tuned on real servers (FINDINGS F7 and
|
|
397
401
|
F10) and a private 115-tool one. They're the extras; the change checks are the point.
|
|
@@ -427,10 +431,12 @@ meant to steer away from, and toolmenu says so.
|
|
|
427
431
|
```
|
|
428
432
|
|
|
429
433
|
`descriptionLimit` is where your client cuts tool descriptions: a number, or a client
|
|
430
|
-
whose cut has been reported: `"claude-code"` (
|
|
431
|
-
[
|
|
432
|
-
|
|
433
|
-
|
|
434
|
+
whose cut has been reported: `"claude-code"` (the default: Claude Code's own docs,
|
|
435
|
+
[code.claude.com/docs/en/mcp](https://code.claude.com/docs/en/mcp), read 2026-11-02:
|
|
436
|
+
16,384 characters for a tool loaded through tool search, which is on by default, and
|
|
437
|
+
4,096 for a tool that loads up front, marked `_meta["anthropic/alwaysLoad"]: true`;
|
|
438
|
+
the cut was 2,048 before v2.1.296), `"claude-code-upfront"` (4,096 for every tool:
|
|
439
|
+
Claude Code with tool search off) or
|
|
434
440
|
`"amazon-q"` (10,024, reported in
|
|
435
441
|
[makenotion/notion-mcp-server#145](https://github.com/makenotion/notion-mcp-server/issues/145)).
|
|
436
442
|
Clients that send descriptions in full don't need a setting beyond that. If your own
|
|
@@ -469,6 +475,9 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
469
475
|
(2026-07-28 only; an error if the server lacks it)
|
|
470
476
|
--processes <n> server processes or connections to compare (default: 2);
|
|
471
477
|
1 opens no second one, session's scope check included
|
|
478
|
+
--as <name>:<cred> snapshot: list the menu as this identity too, repeatable:
|
|
479
|
+
"reader:Authorization: Bearer …" (HTTP) or "admin:ROLE=admin"
|
|
480
|
+
(stdio). The first is the menu saved; the others are compared
|
|
472
481
|
```
|
|
473
482
|
|
|
474
483
|
Exit codes: `0` clean · `1` findings at or above `--fail-on` · `2` couldn't connect
|
package/dist/auto.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { isMap, isNode, isScalar, parseDocument } from 'yaml';
|
|
3
3
|
import { synthesizeArgs } from './args.js';
|
|
4
|
+
import { createdKinds, handleParams } from './rules/handles.js';
|
|
4
5
|
import { MAX_UNLOCKS, unlockers } from './session.js';
|
|
5
6
|
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf, WRITE_VERBS } from './words.js';
|
|
6
7
|
/** Marked or named as a write: never called on the user's word. Says why, or undefined. */
|
|
@@ -45,6 +46,7 @@ export function autoScenario(menu, options = {}) {
|
|
|
45
46
|
const { perTool, ignored: unusedValues } = resolveValues(menu, options.values ?? {});
|
|
46
47
|
ignored.push(...unusedValues);
|
|
47
48
|
const candidates = [];
|
|
49
|
+
const modern = (menu.server.protocolVersion ?? '') >= '2026-07-28';
|
|
48
50
|
// Read-only unlocks with every value in an enum: each value, outside the call
|
|
49
51
|
// budget, so the tools behind every toolset are seen (GitHub: 19 toolsets).
|
|
50
52
|
// Called even when marked openWorldHint, without --open-world: an unlock changes
|
|
@@ -76,10 +78,23 @@ export function autoScenario(menu, options = {}) {
|
|
|
76
78
|
}
|
|
77
79
|
const filled = synthesizeArgs(tool.inputSchema, perTool.get(tool.name));
|
|
78
80
|
if (!filled.ok) {
|
|
81
|
+
// 2026-07-28: a handle the server mints (a session another tool starts) is
|
|
82
|
+
// called once with one that can't exist, to see the model gets a readable error.
|
|
83
|
+
const probe = modern ? handleProbe(tool, menu.tools, filled.missing, perTool.get(tool.name)) : undefined;
|
|
84
|
+
if (probe) {
|
|
85
|
+
candidates.push({ tool, args: probe.args, rank: 4, valued: false, probe: probe.probe });
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
79
88
|
skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
|
|
80
89
|
continue;
|
|
81
90
|
}
|
|
82
91
|
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args), valued: Object.values(filled.sources).includes('value') });
|
|
92
|
+
// An optional handle (inspect_session { view, session_id? }): the call above
|
|
93
|
+
// leaves it out; one more call passes one that can't exist. Not when the user
|
|
94
|
+
// gave a value for it (it's in args then).
|
|
95
|
+
const optional = modern ? optionalHandleProbe(tool, menu.tools, filled.args) : undefined;
|
|
96
|
+
if (optional)
|
|
97
|
+
candidates.push({ tool, args: optional.args, rank: 4, valued: false, probe: optional.probe });
|
|
83
98
|
}
|
|
84
99
|
candidates.sort((x, y) => x.rank - y.rank);
|
|
85
100
|
const chosen = candidates.slice(0, maxCalls);
|
|
@@ -87,7 +102,7 @@ export function autoScenario(menu, options = {}) {
|
|
|
87
102
|
skipped.push({ tool: c.tool.name, reason: 'over the call budget' });
|
|
88
103
|
const steps = options.callsOnly ? [] : [{ kind: 'list' }];
|
|
89
104
|
for (const c of chosen)
|
|
90
|
-
steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
|
|
105
|
+
steps.push({ kind: 'call', tool: c.tool.name, args: c.args, ...(c.probe ? { probe: c.probe } : {}) });
|
|
91
106
|
for (const u of unlocks) {
|
|
92
107
|
const array = u.tool.inputSchema?.properties?.[u.param]?.type === 'array';
|
|
93
108
|
const values = u.values.slice(0, MAX_UNLOCKS);
|
|
@@ -100,7 +115,7 @@ export function autoScenario(menu, options = {}) {
|
|
|
100
115
|
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [values[0]] : values[0] }, ...tentative });
|
|
101
116
|
}
|
|
102
117
|
if (chosen.length && !options.callsOnly)
|
|
103
|
-
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
118
|
+
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args, ...(chosen[0].probe ? { probe: chosen[0].probe } : {}) });
|
|
104
119
|
const called = [...chosen.map((c) => c.tool.name), ...unlocks.map((u) => u.tool.name)];
|
|
105
120
|
return {
|
|
106
121
|
scenario: { allowWrites: false, steps, ...(assumed.size ? { assumeReadOnly: [...assumed] } : {}) },
|
|
@@ -234,6 +249,59 @@ function rank(tool, args) {
|
|
|
234
249
|
return 2;
|
|
235
250
|
return 3;
|
|
236
251
|
}
|
|
252
|
+
/**
|
|
253
|
+
* A required handle filled with one that can't exist: `toolmenu-unknown-session`,
|
|
254
|
+
* or the nil-version UUID for a uuid format. Only for a read-only tool (the caller
|
|
255
|
+
* checks), only when every other required value is filled, and only for a kind
|
|
256
|
+
* some tool in the menu creates, so it is a handle the server mints.
|
|
257
|
+
*/
|
|
258
|
+
/** The value a probe passes for a handle: one no server minted. Undefined when the schema rules a guess out (an enum, not a string, a pattern neither made-up value passes). */
|
|
259
|
+
function madeUpHandle(schema, kind) {
|
|
260
|
+
if (!schema || (schema.type !== undefined && schema.type !== 'string') || schema.enum || 'const' in schema)
|
|
261
|
+
return undefined;
|
|
262
|
+
const value = schema.format === 'uuid' || /uuid/i.test(String(schema.pattern ?? '')) ? '00000000-0000-4000-8000-000000000000' : `toolmenu-unknown-${kind}`;
|
|
263
|
+
if (typeof schema.pattern !== 'string')
|
|
264
|
+
return value;
|
|
265
|
+
// A pattern is the server's own check: the made-up handle must pass it (a nil
|
|
266
|
+
// UUID passes most UUID patterns), or the error would be about the format.
|
|
267
|
+
const uuid = '00000000-0000-4000-8000-000000000000';
|
|
268
|
+
for (const candidate of [value, uuid]) {
|
|
269
|
+
try {
|
|
270
|
+
if (new RegExp(schema.pattern, 'u').test(candidate))
|
|
271
|
+
return candidate;
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
return undefined;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
/** The one optional handle parameter of a tool, of a kind the menu creates, filled with a made-up handle on top of `args`. */
|
|
280
|
+
function optionalHandleProbe(tool, tools, args) {
|
|
281
|
+
const created = createdKinds(tools);
|
|
282
|
+
const required = new Set(tool.inputSchema?.required ?? []);
|
|
283
|
+
const handles = [...created].flatMap((kind) => handleParams(tool, kind).filter((p) => !required.has(p) && !(p in args)).map((param) => ({ param, kind })));
|
|
284
|
+
if (handles.length !== 1)
|
|
285
|
+
return undefined;
|
|
286
|
+
const [{ param, kind }] = handles;
|
|
287
|
+
const value = madeUpHandle(tool.inputSchema?.properties?.[param], kind);
|
|
288
|
+
return value === undefined ? undefined : { args: { ...args, [param]: value }, probe: { param, kind } };
|
|
289
|
+
}
|
|
290
|
+
function handleProbe(tool, tools, missing, given) {
|
|
291
|
+
const created = createdKinds(tools);
|
|
292
|
+
const handles = missing.flatMap((param) => {
|
|
293
|
+
const kind = [...created].find((k) => handleParams(tool, k).includes(param));
|
|
294
|
+
return kind ? [{ param, kind }] : [];
|
|
295
|
+
});
|
|
296
|
+
if (handles.length !== 1 || missing.length !== 1)
|
|
297
|
+
return undefined;
|
|
298
|
+
const [{ param, kind }] = handles;
|
|
299
|
+
const value = madeUpHandle(tool.inputSchema?.properties?.[param], kind);
|
|
300
|
+
if (value === undefined)
|
|
301
|
+
return undefined;
|
|
302
|
+
const rest = synthesizeArgs(tool.inputSchema, { ...given, [param]: { value, raw: value } });
|
|
303
|
+
return rest.ok ? { args: rest.args, probe: { param, kind } } : undefined;
|
|
304
|
+
}
|
|
237
305
|
/** Parameters the skipped tools need, most needed first, with how many tools each fills. */
|
|
238
306
|
export function neededValues(skipped) {
|
|
239
307
|
const by = new Map();
|
|
@@ -322,6 +390,10 @@ export function scenarioYaml(plan, serverName) {
|
|
|
322
390
|
for (const step of plan.scenario.steps) {
|
|
323
391
|
if (step.kind === 'list')
|
|
324
392
|
lines.push(' - list');
|
|
393
|
+
else if (step.kind === 'call' && step.probe) {
|
|
394
|
+
// A made-up handle is --auto's check, not a step to replay: as written it would fail as a step.
|
|
395
|
+
lines.push(` # --auto called ${step.tool} with a made-up ${step.probe.kind} (session/handle-error); to replay it, put a real one in:`, ` # - call: ${step.tool}`, ` # args: ${JSON.stringify(step.args)}`);
|
|
396
|
+
}
|
|
325
397
|
else if (step.kind === 'call') {
|
|
326
398
|
lines.push(` - call: ${step.tool}`);
|
|
327
399
|
if (Object.keys(step.args).length)
|
package/dist/cli.js
CHANGED
|
@@ -82,6 +82,8 @@ export async function main(argv) {
|
|
|
82
82
|
const elicitation = (values.elicitation ?? 'decline');
|
|
83
83
|
if (!ELICITATIONS.includes(elicitation))
|
|
84
84
|
throw new UsageError('--elicitation must be decline or cancel');
|
|
85
|
+
if (values.as !== undefined && sub !== 'snapshot')
|
|
86
|
+
throw new UsageError('--as goes with snapshot: it lists the menu once per identity.');
|
|
85
87
|
if (values.elicitation !== undefined && sub !== 'session')
|
|
86
88
|
throw new UsageError('--elicitation goes with session: only session calls tools.');
|
|
87
89
|
// Only session calls tools, so only its targets declare elicitation and say how to answer.
|
|
@@ -295,9 +297,10 @@ export async function main(argv) {
|
|
|
295
297
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
296
298
|
}
|
|
297
299
|
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
300
|
+
const identities = values.as?.length ? parseIdentities(values.as, target.kind) : configIdentities(config.identities, target.kind);
|
|
298
301
|
const routesPath = values.routes ?? config.routes;
|
|
299
302
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
300
|
-
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
303
|
+
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, identities, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
301
304
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
302
305
|
if (outPath)
|
|
303
306
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
|
@@ -352,6 +355,44 @@ async function authCommand(args, values) {
|
|
|
352
355
|
process.stdout.write(`Logged in to ${result.name ?? url} (${url}): it lists ${result.tools} tool${result.tools === 1 ? '' : 's'} with this login.\nsnapshot and session use it from now on (--no-auth to skip it); stored in ${authDir()}.\n`);
|
|
353
356
|
return 0;
|
|
354
357
|
}
|
|
358
|
+
/**
|
|
359
|
+
* --as name:credential, repeatable, in order of first mention: `reader:Authorization:
|
|
360
|
+
* Bearer x` (HTTP header) or `admin:ROLE=admin` (stdio env var). A name alone is an
|
|
361
|
+
* identity with only the --header/--env given to every run.
|
|
362
|
+
*/
|
|
363
|
+
function parseIdentities(items, kind) {
|
|
364
|
+
const byName = new Map();
|
|
365
|
+
for (const item of items) {
|
|
366
|
+
const colon = item.indexOf(':');
|
|
367
|
+
const name = (colon < 0 ? item : item.slice(0, colon)).trim();
|
|
368
|
+
if (!/^[\w.-]+$/.test(name))
|
|
369
|
+
throw new UsageError(`--as "${item}" should start with a name (letters, digits, _ . -): --as reader:${kind === 'http' ? 'Authorization: Bearer …' : 'TOKEN=…'}`);
|
|
370
|
+
const identity = byName.get(name) ?? { name };
|
|
371
|
+
byName.set(name, identity);
|
|
372
|
+
if (colon < 0)
|
|
373
|
+
continue;
|
|
374
|
+
const credential = item.slice(colon + 1);
|
|
375
|
+
if (kind === 'http')
|
|
376
|
+
identity.headers = { ...identity.headers, ...parsePairs([credential], ':', `--as ${name}:`) };
|
|
377
|
+
else
|
|
378
|
+
identity.env = { ...identity.env, ...parsePairs([credential], '=', `--as ${name}:`) };
|
|
379
|
+
}
|
|
380
|
+
if (byName.size < 2)
|
|
381
|
+
throw new UsageError('--as needs two identities or more to compare: --as reader:… --as admin:…');
|
|
382
|
+
return [...byName.values()];
|
|
383
|
+
}
|
|
384
|
+
/** The config file's identities, for this kind of server: headers for HTTP, env for stdio. */
|
|
385
|
+
function configIdentities(ids, kind) {
|
|
386
|
+
if (!ids)
|
|
387
|
+
return undefined;
|
|
388
|
+
// Credentials for the other kind of server only (env for an HTTP server): that identity would list as nobody in particular.
|
|
389
|
+
const wrong = Object.entries(ids).filter(([, id]) => (kind === 'http' ? !id.headers && id.env : !id.env && id.headers)).map(([name]) => name);
|
|
390
|
+
if (wrong.length) {
|
|
391
|
+
throw new UsageError(`identities ${wrong.join(', ')} in the config give ${kind === 'http' ? 'env vars' : 'headers'} only, and this is ${kind === 'http' ? 'an HTTP' : 'a stdio'} server: give each ${kind === 'http' ? '"headers"' : '"env"'}.`);
|
|
392
|
+
}
|
|
393
|
+
const list = Object.entries(ids).map(([name, id]) => ({ name, ...(kind === 'http' ? (id.headers ? { headers: id.headers } : {}) : id.env ? { env: id.env } : {}) }));
|
|
394
|
+
return list.length >= 2 ? list : undefined;
|
|
395
|
+
}
|
|
355
396
|
function parsePairs(items, separator, flag) {
|
|
356
397
|
const pairs = {};
|
|
357
398
|
for (const item of items) {
|
package/dist/config.d.ts
CHANGED
|
@@ -11,11 +11,21 @@ export interface Config {
|
|
|
11
11
|
serverVersionIsRelease?: boolean;
|
|
12
12
|
/**
|
|
13
13
|
* Where your client cuts tool descriptions: a number of characters, or a
|
|
14
|
-
* known client ("claude-code", "amazon-q"). Default "claude-code"
|
|
14
|
+
* known client ("claude-code", "claude-code-upfront", "amazon-q"). Default "claude-code"
|
|
15
|
+
* (16,384 through tool search, 4,096 for a tool that loads up front).
|
|
15
16
|
*/
|
|
16
17
|
descriptionLimit?: number | string;
|
|
17
18
|
/** Tools (names or globs) your client sends with the full description. */
|
|
18
19
|
fullDescriptions?: string[];
|
|
20
|
+
/**
|
|
21
|
+
* snapshot: list the menu once per identity, the first being the main one:
|
|
22
|
+
* { "reader": { "headers": { "Authorization": "Bearer …" } }, "admin": { "env": { "ROLE": "admin" } } }.
|
|
23
|
+
* --as replaces them.
|
|
24
|
+
*/
|
|
25
|
+
identities?: Record<string, {
|
|
26
|
+
headers?: Record<string, string>;
|
|
27
|
+
env?: Record<string, string>;
|
|
28
|
+
}>;
|
|
19
29
|
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
20
30
|
processes?: number;
|
|
21
31
|
/**
|
package/dist/config.js
CHANGED
|
@@ -29,6 +29,17 @@ export async function loadConfig(path) {
|
|
|
29
29
|
if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
|
|
30
30
|
throw new Error(`${file}: processes must be a whole number, 1 or more`);
|
|
31
31
|
}
|
|
32
|
+
if (config.identities !== undefined) {
|
|
33
|
+
const strings = (v) => v === undefined || (typeof v === 'object' && v !== null && !Array.isArray(v) && Object.values(v).every((x) => typeof x === 'string'));
|
|
34
|
+
const ids = config.identities;
|
|
35
|
+
if (typeof ids !== 'object' || ids === null || Array.isArray(ids))
|
|
36
|
+
throw new Error(`${file}: identities must be an object, like { "reader": { "headers": { "Authorization": "Bearer …" } } }`);
|
|
37
|
+
for (const [name, id] of Object.entries(ids)) {
|
|
38
|
+
if (typeof id !== 'object' || id === null || Array.isArray(id) || !strings(id.headers) || !strings(id.env)) {
|
|
39
|
+
throw new Error(`${file}: identities.${name} must be { "headers": { … } } or { "env": { … } }, with string values`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
32
43
|
const catalog = config.catalog;
|
|
33
44
|
if (catalog !== undefined) {
|
|
34
45
|
const names = (v) => Array.isArray(v) && v.length > 0 && v.every((x) => typeof x === 'string' && x.length > 0);
|
package/dist/failures.js
CHANGED
|
@@ -22,7 +22,7 @@ const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bno
|
|
|
22
22
|
// Netlify" (Netlify), "The API key you provided was rejected" (Pinecone), "Admin
|
|
23
23
|
// auth is missing", "Runtime credentials are missing" (securenow). Not a bare
|
|
24
24
|
// "token": plenty of errors mention one.
|
|
25
|
-
const AUTH = /\b(?:auth|authentication|credentials?|api[ _-]?key|access token)\b (?:is |are )?missing\b|\b(401|403)\b|unauthori[sz]ed|unauth(enticated|error)|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|not logged (?:in|into)\b(?! to the browser)|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|(api[ _-]?key|access token|credentials)\b[^.]{0,30}\b(was|were|is|are) (rejected|refused|revoked)|valid (access )?token/i;
|
|
25
|
+
const AUTH = /\b(?:auth|authentication|credentials?|api[ _-]?key|access token)\b (?:is |are )?missing\b|\bno (?:authentication|auth|credentials?|api[ _-]?keys?|access tokens?|auth tokens?) (?:was |were )?(?:found|provided|configured|set|given)\b|\b(401|403)\b|unauthori[sz]ed|unauth(enticated|error)|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|not logged (?:in|into)\b(?! to the browser)|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|(api[ _-]?key|access token|credentials)\b[^.]{0,30}\b(was|were|is|are) (rejected|refused|revoked)|valid (access )?token/i;
|
|
26
26
|
const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
|
|
27
27
|
// "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
|
|
28
28
|
// "**Input Error**" (Sentry), JSON-RPC's invalid params.
|
package/dist/help.js
CHANGED
|
@@ -18,6 +18,7 @@ const OPTIONS = {
|
|
|
18
18
|
header: ['--header "<k>: <v>"', ['HTTP header, repeatable (e.g. "Authorization: Bearer ...")']],
|
|
19
19
|
env: ['--env <K=V>', ['environment variable for a stdio server, repeatable. The server only', 'gets a minimal environment (PATH, HOME, ...) plus these']],
|
|
20
20
|
protocol: ['--protocol <auto|legacy|modern>', ['which handshake to make (default: auto)', 'legacy: the 2025 handshake only; modern: 2026-07-28 only, an error if the server lacks it']],
|
|
21
|
+
as: ['--as <name>[:<credential>]', ['list the menu as this identity too, repeatable: a header ("reader:Authorization:', 'Bearer ...") or env var ("admin:ROLE=admin") on top of --header/--env. The first', 'is the main one (its menu is saved); the others are compared with it']],
|
|
21
22
|
'no-auth': ['--no-auth', ["don't use a stored OAuth login (toolmenu auth login) for this server"]],
|
|
22
23
|
timeout: ['--timeout <ms>', ['per-request timeout (default: 30000)']],
|
|
23
24
|
processes: ['--processes <n>', ['server processes (stdio) or connections (HTTP) to compare, the main', 'one included (default: 2). 1 opens no second one: for servers with one', 'session per client']],
|
|
@@ -77,7 +78,7 @@ const HELP = {
|
|
|
77
78
|
'reports what in the menu confuses agents or breaks prompt caches.',
|
|
78
79
|
SERVER_NOTE,
|
|
79
80
|
],
|
|
80
|
-
own: ['out', 'no-write', 'routes', 'catalog'],
|
|
81
|
+
own: ['out', 'no-write', 'routes', 'catalog', 'as'],
|
|
81
82
|
shared: ['config', 'format', 'json', 'fail-on', 'header', 'env', 'no-auth', 'protocol', 'timeout', 'processes'],
|
|
82
83
|
examples: [
|
|
83
84
|
'toolmenu snapshot -- npx -y @modelcontextprotocol/server-memory',
|
package/dist/options.d.ts
CHANGED
package/dist/options.js
CHANGED
|
@@ -12,6 +12,7 @@ export const CLI_OPTIONS = {
|
|
|
12
12
|
'fail-on': { type: 'string' },
|
|
13
13
|
header: { type: 'string', multiple: true },
|
|
14
14
|
env: { type: 'string', multiple: true },
|
|
15
|
+
as: { type: 'string', multiple: true },
|
|
15
16
|
timeout: { type: 'string' },
|
|
16
17
|
protocol: { type: 'string' },
|
|
17
18
|
elicitation: { type: 'string' },
|
package/dist/probe.d.ts
CHANGED
|
@@ -14,6 +14,14 @@ export declare const PROBE_SEED = "1";
|
|
|
14
14
|
* a stdio server. Other runtimes ignore it.
|
|
15
15
|
*/
|
|
16
16
|
export declare function seeded(target: Target, seed: string): Target;
|
|
17
|
+
/** Credentials for one identity (snapshot --as, config identities): HTTP headers or stdio env vars. */
|
|
18
|
+
export interface Identity {
|
|
19
|
+
name: string;
|
|
20
|
+
headers?: Record<string, string>;
|
|
21
|
+
env?: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
/** The target as this identity: its headers (HTTP) or env vars (stdio) over the ones given. */
|
|
24
|
+
export declare function asIdentity(target: Target, identity: Identity | undefined): Target;
|
|
17
25
|
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
18
26
|
export declare function isContainerWrapper(target: Target): boolean;
|
|
19
27
|
/**
|
package/dist/probe.js
CHANGED
|
@@ -20,6 +20,18 @@ export function seeded(target, seed) {
|
|
|
20
20
|
return target;
|
|
21
21
|
return { ...target, env: { ...target.env, PYTHONHASHSEED: seed } };
|
|
22
22
|
}
|
|
23
|
+
/** The target as this identity: its headers (HTTP) or env vars (stdio) over the ones given. */
|
|
24
|
+
export function asIdentity(target, identity) {
|
|
25
|
+
if (!identity)
|
|
26
|
+
return target;
|
|
27
|
+
if (target.kind === 'http') {
|
|
28
|
+
// Header names match whatever their case: the identity's authorization replaces --header's Authorization.
|
|
29
|
+
const mine = new Set(Object.keys(identity.headers ?? {}).map((h) => h.toLowerCase()));
|
|
30
|
+
const kept = Object.fromEntries(Object.entries(target.headers ?? {}).filter(([h]) => !mine.has(h.toLowerCase())));
|
|
31
|
+
return { ...target, headers: { ...kept, ...identity.headers } };
|
|
32
|
+
}
|
|
33
|
+
return { ...target, env: { ...target.env, ...identity.env } };
|
|
34
|
+
}
|
|
23
35
|
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
24
36
|
export function isContainerWrapper(target) {
|
|
25
37
|
return target.kind === 'stdio' && /^(docker|podman|nerdctl)(\.exe)?$/.test(basename(target.command));
|
|
@@ -50,7 +62,7 @@ export async function probeMenu(target, timeoutMs, waits, waited) {
|
|
|
50
62
|
const c = await connect(target, { timeoutMs });
|
|
51
63
|
try {
|
|
52
64
|
const list = await listTools(c, { timeoutMs });
|
|
53
|
-
return buildMenu(list.tools, { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era });
|
|
65
|
+
return buildMenu(list.tools, { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era }, list.listMeta);
|
|
54
66
|
}
|
|
55
67
|
finally {
|
|
56
68
|
await c.close().catch(() => { });
|
|
@@ -5,11 +5,14 @@ export declare const DEFAULT_CLIENT = "claude-code";
|
|
|
5
5
|
/**
|
|
6
6
|
* The cut in characters, the client to name in a message ("Claude Code", "your
|
|
7
7
|
* client"), and whether it's toolmenu's assumption rather than the user's setting.
|
|
8
|
+
* `perTool`: the cut for one tool, which differs under Claude Code's default for a
|
|
9
|
+
* tool marked to load up front.
|
|
8
10
|
*/
|
|
9
11
|
export declare function resolveLimit(setting: number | string | undefined): {
|
|
10
12
|
limit: number;
|
|
11
13
|
who: string;
|
|
12
14
|
assumed: boolean;
|
|
15
|
+
perTool: (tool: MenuTool) => number;
|
|
13
16
|
};
|
|
14
17
|
interface Instruction {
|
|
15
18
|
at: number;
|
|
@@ -1,24 +1,39 @@
|
|
|
1
1
|
import { clip, kept, num } from './rule.js';
|
|
2
2
|
/*
|
|
3
3
|
* Clients that cut tool descriptions, and where. Only reported cuts, with sources:
|
|
4
|
-
* - claude-code:
|
|
5
|
-
*
|
|
4
|
+
* - claude-code: Claude Code's default, where MCP tools are deferred behind tool
|
|
5
|
+
* search: 16,384 characters for a tool loaded through tool search, 4,096 for one
|
|
6
|
+
* that loads up front (marked `_meta["anthropic/alwaysLoad"]: true`). Read
|
|
7
|
+
* 2026-11-02 from https://code.claude.com/docs/en/mcp ("Tool search for MCP server
|
|
8
|
+
* authors"). Before v2.1.296 the cut was 2,048 for every tool.
|
|
9
|
+
* - claude-code-upfront: Claude Code with tool search off (ENABLE_TOOL_SEARCH=false,
|
|
10
|
+
* or a server set to alwaysLoad): 4,096 for every tool, same source.
|
|
6
11
|
* - amazon-q: 10,024 characters, with a warning (Amazon Q CLI 1.19; makenotion/notion-mcp-server#145).
|
|
7
12
|
* Other clients send the description in full or don't document a cut. Your own
|
|
8
13
|
* client may cut elsewhere: set descriptionLimit to its number.
|
|
9
14
|
*/
|
|
10
|
-
export const CLIENT_LIMITS = { 'claude-code':
|
|
15
|
+
export const CLIENT_LIMITS = { 'claude-code': 16384, 'claude-code-upfront': 4096, 'amazon-q': 10024 };
|
|
11
16
|
export const DEFAULT_CLIENT = 'claude-code';
|
|
17
|
+
/** Claude Code's cut for a tool that loads up front, not through tool search. */
|
|
18
|
+
const CLAUDE_CODE_UPFRONT = 4096;
|
|
12
19
|
/**
|
|
13
20
|
* The cut in characters, the client to name in a message ("Claude Code", "your
|
|
14
21
|
* client"), and whether it's toolmenu's assumption rather than the user's setting.
|
|
22
|
+
* `perTool`: the cut for one tool, which differs under Claude Code's default for a
|
|
23
|
+
* tool marked to load up front.
|
|
15
24
|
*/
|
|
16
25
|
export function resolveLimit(setting) {
|
|
17
26
|
if (typeof setting === 'number')
|
|
18
|
-
return { limit: setting, who: 'your client', assumed: false };
|
|
19
|
-
const client = setting
|
|
20
|
-
const limit = CLIENT_LIMITS[client]
|
|
21
|
-
|
|
27
|
+
return { limit: setting, who: 'your client', assumed: false, perTool: () => setting };
|
|
28
|
+
const client = setting !== undefined && setting in CLIENT_LIMITS ? setting : DEFAULT_CLIENT;
|
|
29
|
+
const limit = CLIENT_LIMITS[client];
|
|
30
|
+
const upfront = (tool) => tool._meta?.['anthropic/alwaysLoad'] === true;
|
|
31
|
+
return {
|
|
32
|
+
limit,
|
|
33
|
+
who: client === 'amazon-q' ? 'Amazon Q CLI' : 'Claude Code',
|
|
34
|
+
assumed: setting === undefined,
|
|
35
|
+
perTool: client === 'claude-code' ? (tool) => (upfront(tool) ? CLAUDE_CODE_UPFRONT : limit) : () => limit,
|
|
36
|
+
};
|
|
22
37
|
}
|
|
23
38
|
/** The fix's last words when the cut is assumed, not set. */
|
|
24
39
|
function unlessOtherClient(assumed) {
|
|
@@ -100,10 +115,11 @@ export const buried = {
|
|
|
100
115
|
lesson: 'truncated description',
|
|
101
116
|
summary: 'Instructions to the agent sit past the point where your client cuts descriptions',
|
|
102
117
|
run(ctx) {
|
|
103
|
-
const {
|
|
118
|
+
const { perTool, who, assumed } = resolveLimit(ctx.descriptionLimit);
|
|
104
119
|
const all = ctx.menu.tools.map((t) => t.name);
|
|
105
120
|
const findings = [];
|
|
106
121
|
for (const tool of ctx.menu.tools) {
|
|
122
|
+
const limit = perTool(tool);
|
|
107
123
|
const text = tool.description ?? '';
|
|
108
124
|
if (text.length <= limit || sentInFull(tool, ctx.fullDescriptions))
|
|
109
125
|
continue;
|
|
@@ -141,12 +157,16 @@ export const cut = {
|
|
|
141
157
|
lesson: 'truncated description',
|
|
142
158
|
summary: 'Descriptions longer than your client sends',
|
|
143
159
|
run(ctx) {
|
|
144
|
-
const {
|
|
160
|
+
const { perTool, who, assumed } = resolveLimit(ctx.descriptionLimit);
|
|
145
161
|
const all = ctx.menu.tools.map((t) => t.name);
|
|
146
162
|
const long = kept(ctx.menu.tools, ctx).filter((tool) => {
|
|
147
163
|
const text = tool.description ?? '';
|
|
164
|
+
const limit = perTool(tool);
|
|
148
165
|
return !sentInFull(tool, ctx.fullDescriptions) && text.length > limit && pastTheCut(text, all.filter((n) => n !== tool.name), limit).length === 0;
|
|
149
166
|
});
|
|
167
|
+
// One number when every cut tool has the same cut (Claude Code cuts up-front tools shorter).
|
|
168
|
+
const limits = [...new Set(long.map(perTool))];
|
|
169
|
+
const limit = limits.length === 1 ? limits[0] : Math.min(...limits);
|
|
150
170
|
if (long.length === 0)
|
|
151
171
|
return [];
|
|
152
172
|
const one = long.length === 1;
|
|
@@ -154,10 +174,11 @@ export const cut = {
|
|
|
154
174
|
return [
|
|
155
175
|
{
|
|
156
176
|
...(one ? { tool: long[0].name } : {}),
|
|
157
|
-
message: `${one ? `The ${long[0].name} description is` : `${long.length} descriptions are`} longer than the ${num(limit)} characters ${who} sends. The model gets the start, with no sign that anything is missing, so a cut list or example can read as complete. Nothing past the cut reads as an instruction to the agent, so this is a check, not a bug.`,
|
|
177
|
+
message: `${one ? `The ${long[0].name} description is` : `${long.length} descriptions are`} longer than ${limits.length === 1 ? `the ${num(limit)} characters ${who} sends` : `${who} sends (${limits.map(num).join(' or ')} characters)`}. The model gets the start, with no sign that anything is missing, so a cut list or example can read as complete. Nothing past the cut reads as an instruction to the agent, so this is a check, not a bug.`,
|
|
158
178
|
detail: long.map((t) => {
|
|
159
179
|
const text = t.description ?? '';
|
|
160
|
-
|
|
180
|
+
const at = perTool(t);
|
|
181
|
+
return `${t.name}: ${num(text.length)} characters, ${num(text.length - at)} past the cut, which falls at ${aroundTheCut(text, at)}`;
|
|
161
182
|
}),
|
|
162
183
|
fix: `Check that what's past the cut ${one ? 'in' : 'in each of'} ${one ? long[0].name : 'these'} can be lost, and move anything the agent needs into the first ${num(limit)} characters.${unlessOtherClient(assumed)}`,
|
|
163
184
|
},
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { MenuTool } from '../types.js';
|
|
2
|
+
import { type Rule } from './rule.js';
|
|
3
|
+
/** A call made with a handle that can't exist (session --auto): the parameter, and the kind of handle. */
|
|
4
|
+
export interface HandleProbe {
|
|
5
|
+
param: string;
|
|
6
|
+
kind: string;
|
|
7
|
+
}
|
|
8
|
+
/** The handle kinds some tool in the menu creates. */
|
|
9
|
+
export declare function createdKinds(tools: MenuTool[]): Set<string>;
|
|
10
|
+
/** Parameters of a tool that take a handle of this kind: session_id, cartToken, `session`. */
|
|
11
|
+
export declare function handleParams(tool: MenuTool, kind: string): string[];
|
|
12
|
+
export declare const handleLifetime: Rule;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { singular, words } from '../words.js';
|
|
2
|
+
import { kept, names } from './rule.js';
|
|
3
|
+
/*
|
|
4
|
+
* Server-minted handles (2026-07-28, SEP-2567). With protocol sessions gone, a
|
|
5
|
+
* server that keeps state across calls hands out a handle as an ordinary tool
|
|
6
|
+
* result (`basket_id`) and takes it back as an argument. The spec asks for the
|
|
7
|
+
* handle's retention "in the creation tool's description" ("baskets expire after
|
|
8
|
+
* 24 hours of inactivity"), so a model knows when to make a new one.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Kinds that are server state for a while, never a lasting record: a session, a
|
|
12
|
+
* basket or cart, a checkout, a lease or lock, or something called a handle.
|
|
13
|
+
* Not drafts, runs, jobs or tasks: in real menus those are as often records that
|
|
14
|
+
* stay (an email draft, a CI run, a planner task).
|
|
15
|
+
*/
|
|
16
|
+
const HANDLE_KINDS = new Set(['session', 'basket', 'cart', 'checkout', 'handle', 'lease', 'lock']);
|
|
17
|
+
const CREATE = new Set(['create', 'start', 'open', 'new', 'begin', 'init', 'initialize', 'acquire']);
|
|
18
|
+
/** How a handle parameter ends: session_id, sessionHandle, cart_token, or the kind alone. */
|
|
19
|
+
const HANDLE_SUFFIX = new Set(['id', 'ids', 'uuid', 'handle', 'token', 'key', 'ref']);
|
|
20
|
+
/**
|
|
21
|
+
* Words that say how long a handle lasts, or that it lasts: "expires", "TTL",
|
|
22
|
+
* "valid for 24 hours", "after 30 minutes of inactivity", "kept until closed",
|
|
23
|
+
* "never expires".
|
|
24
|
+
*/
|
|
25
|
+
const LIFETIME = /\bexpir(?:e|es|ed|y|ing|ation)\b|\bttl\b|\btime[- ]to[- ]live\b|\bvalid (?:for|until)\b|\blasts?\b|\binactiv|\bidle\b|\btime(?:s|d)? ?out\b|\bretain|\bretention\b|\bkept (?:for|until)\b|\b(?:deleted|discarded|cleared|purged|removed) (?:after|when|on)\b|\bpersist|\bindefinite|\bnever expire|\buntil (?:it is |you )?(?:closed?|ended?|deleted?|released?)\b|\b\d+\s*(?:s|sec|seconds?|m|min|minutes?|h|hrs?|hours?|d|days?)\b/i;
|
|
26
|
+
/** The handle kind a tool creates, from its name: start_session → session, create-shopping-cart → cart. */
|
|
27
|
+
function createdKind(tool) {
|
|
28
|
+
const w = words(tool.name).map(singular);
|
|
29
|
+
if (!w.length || !CREATE.has(w[0]))
|
|
30
|
+
return undefined;
|
|
31
|
+
return w.slice(1).find((x) => HANDLE_KINDS.has(x));
|
|
32
|
+
}
|
|
33
|
+
/** The handle kinds some tool in the menu creates. */
|
|
34
|
+
export function createdKinds(tools) {
|
|
35
|
+
return new Set(tools.map(createdKind).filter((k) => !!k));
|
|
36
|
+
}
|
|
37
|
+
/** Parameters of a tool that take a handle of this kind: session_id, cartToken, `session`. */
|
|
38
|
+
export function handleParams(tool, kind) {
|
|
39
|
+
return Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => {
|
|
40
|
+
const w = words(p).map(singular);
|
|
41
|
+
if (w.length === 1)
|
|
42
|
+
return w[0] === kind;
|
|
43
|
+
return w.at(-2) === kind && HANDLE_SUFFIX.has(w.at(-1));
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
/** Text a creation tool gives about what it returns: its description, and its output schema's descriptions. */
|
|
47
|
+
function ownText(tool) {
|
|
48
|
+
const out = [tool.description ?? ''];
|
|
49
|
+
const walk = (s, depth) => {
|
|
50
|
+
if (!s || depth > 4)
|
|
51
|
+
return;
|
|
52
|
+
if (typeof s.description === 'string')
|
|
53
|
+
out.push(s.description);
|
|
54
|
+
for (const child of Object.values(s.properties ?? {}))
|
|
55
|
+
walk(child, depth + 1);
|
|
56
|
+
};
|
|
57
|
+
walk(tool.outputSchema, 0);
|
|
58
|
+
return out.join('\n');
|
|
59
|
+
}
|
|
60
|
+
export const handleLifetime = {
|
|
61
|
+
id: 'ids/handle-lifetime',
|
|
62
|
+
severity: 'info',
|
|
63
|
+
since: '2026-07-28',
|
|
64
|
+
summary: "A tool that creates a handle should say how long it lasts",
|
|
65
|
+
run(ctx) {
|
|
66
|
+
const findings = [];
|
|
67
|
+
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
68
|
+
const kind = createdKind(tool);
|
|
69
|
+
if (!kind)
|
|
70
|
+
continue;
|
|
71
|
+
// A handle is passed back: some other tool takes it. Otherwise it's not a handle the model keeps.
|
|
72
|
+
const users = ctx.menu.tools.filter((t) => t.name !== tool.name && handleParams(t, kind).length);
|
|
73
|
+
if (!users.length)
|
|
74
|
+
continue;
|
|
75
|
+
if (LIFETIME.test(ownText(tool)))
|
|
76
|
+
continue;
|
|
77
|
+
const takers = users.map((t) => `${t.name}.${handleParams(t, kind)[0]}`);
|
|
78
|
+
findings.push({
|
|
79
|
+
tool: tool.name,
|
|
80
|
+
confidence: 'unsure',
|
|
81
|
+
message: `${tool.name} creates a ${kind} that ${takers.length === 1 ? takers[0] : `${takers.length} tools`} take${takers.length === 1 ? 's' : ''} back, but its description doesn't say how long the ${kind} lasts. 2026-07-28 asks for a handle's retention in the creation tool's description, so a model can tell an expired ${kind} from a wrong one and knows when to make a new one.`,
|
|
82
|
+
...(takers.length > 1 ? { detail: [`Taken by ${names(takers, 8)}.`] } : {}),
|
|
83
|
+
fix: `Say in ${tool.name}'s description how long a ${kind} lasts ("${kind}s expire after 30 minutes of inactivity", or "a ${kind} lasts until it is closed"), and return a tool error that says so for an expired or unknown one.`,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return findings;
|
|
87
|
+
},
|
|
88
|
+
};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { MenuTool } from '../types.js';
|
|
2
|
+
import { type Rule } from './rule.js';
|
|
3
|
+
/** One identity's menu, as snapshot listed it: the first is the main one, whose menu is saved. */
|
|
4
|
+
export interface IdentityMenu {
|
|
5
|
+
name: string;
|
|
6
|
+
tools?: MenuTool[];
|
|
7
|
+
tokens?: number;
|
|
8
|
+
cacheScope?: unknown;
|
|
9
|
+
/** Why it couldn't be listed (bad credentials, a refused connection). */
|
|
10
|
+
error?: string;
|
|
11
|
+
}
|
|
12
|
+
/** What one identity's menu adds and lacks against the main one, and its token total. */
|
|
13
|
+
export interface IdentityDifference {
|
|
14
|
+
name: string;
|
|
15
|
+
adds: string[];
|
|
16
|
+
lacks: string[];
|
|
17
|
+
/** Tools both list whose definitions differ (a description or schema per role). */
|
|
18
|
+
changed: string[];
|
|
19
|
+
tokens: number;
|
|
20
|
+
}
|
|
21
|
+
export declare function identityDifferences(identities: IdentityMenu[]): IdentityDifference[];
|
|
22
|
+
export declare const authVariance: Rule;
|
|
23
|
+
export declare const cacheScopePublic: Rule;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { count, names, num } from './rule.js';
|
|
2
|
+
export function identityDifferences(identities) {
|
|
3
|
+
const [main, ...others] = identities;
|
|
4
|
+
if (!main?.tools)
|
|
5
|
+
return [];
|
|
6
|
+
const mine = new Map(main.tools.map((t) => [t.name, JSON.stringify({ d: t.description, i: t.inputSchema, o: t.outputSchema, a: t.annotations })]));
|
|
7
|
+
return others
|
|
8
|
+
.filter((o) => o.tools)
|
|
9
|
+
.map((o) => {
|
|
10
|
+
const theirs = new Map(o.tools.map((t) => [t.name, JSON.stringify({ d: t.description, i: t.inputSchema, o: t.outputSchema, a: t.annotations })]));
|
|
11
|
+
return {
|
|
12
|
+
name: o.name,
|
|
13
|
+
adds: [...theirs.keys()].filter((n) => !mine.has(n)),
|
|
14
|
+
lacks: [...mine.keys()].filter((n) => !theirs.has(n)),
|
|
15
|
+
changed: [...theirs].filter(([n, def]) => mine.has(n) && mine.get(n) !== def).map(([n]) => n),
|
|
16
|
+
tokens: o.tokens ?? 0,
|
|
17
|
+
};
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
const differs = (d) => d.adds.length + d.lacks.length + d.changed.length > 0;
|
|
21
|
+
function describe(d, main) {
|
|
22
|
+
const parts = [
|
|
23
|
+
d.adds.length ? `adds ${names(d.adds, 6)}` : '',
|
|
24
|
+
d.lacks.length ? `lacks ${names(d.lacks, 6)}` : '',
|
|
25
|
+
d.changed.length ? `${count(d.changed.length, 'tool')} defined differently (${names(d.changed, 4)})` : '',
|
|
26
|
+
].filter(Boolean);
|
|
27
|
+
const delta = d.tokens - (main.tokens ?? 0);
|
|
28
|
+
return `${d.name}: ${parts.join('; ')} (${num(d.tokens)} tokens, ${delta >= 0 ? '+' : ''}${num(delta)} against ${main.name})`;
|
|
29
|
+
}
|
|
30
|
+
export const authVariance = {
|
|
31
|
+
id: 'menu/auth-variance',
|
|
32
|
+
severity: 'info',
|
|
33
|
+
summary: 'The menu differs between identities (allowed; says what differs)',
|
|
34
|
+
run(ctx) {
|
|
35
|
+
const identities = ctx.identities ?? [];
|
|
36
|
+
if (identities.length < 2)
|
|
37
|
+
return [];
|
|
38
|
+
const [main] = identities;
|
|
39
|
+
const findings = [];
|
|
40
|
+
const diffs = identityDifferences(identities);
|
|
41
|
+
const varying = diffs.filter(differs);
|
|
42
|
+
if (varying.length) {
|
|
43
|
+
findings.push({
|
|
44
|
+
message: `The menu differs by identity: ${varying.length === diffs.length && diffs.length === 1 ? `${varying[0].name} sees a different tool set than ${main.name}` : `${count(varying.length, 'identity', 'identities')} of ${num(diffs.length)} see a different tool set than ${main.name}`}. Allowed (the tool set may vary by authorization); each sees only its own, so a baseline of ${main.name}'s menu doesn't cover the others.`,
|
|
45
|
+
detail: varying.map((d) => describe(d, main)),
|
|
46
|
+
fix: `Keep a baseline per identity you care about: the first --as is the menu saved, so snapshot --as ${varying[0].name}:… --as ${main.name}:… --out menu.${varying[0].name}.json, and diff each one.`,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
const failed = identities.filter((i) => i.error);
|
|
50
|
+
if (failed.length) {
|
|
51
|
+
findings.push({
|
|
52
|
+
message: failed.length === 1
|
|
53
|
+
? `${failed[0].name} couldn't be listed, so its menu is not compared (${failed[0].error.replace(/[.\s]+$/, '')}).`
|
|
54
|
+
: `${names(failed.map((i) => i.name))} couldn't be listed, so their menus are not compared.`,
|
|
55
|
+
...(failed.length > 1 ? { detail: failed.map((i) => `${i.name}: ${i.error.replace(/[.\s]+$/, '')}`) } : {}),
|
|
56
|
+
fix: 'Check the credentials given with --as for that identity.',
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return findings;
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
export const cacheScopePublic = {
|
|
63
|
+
id: 'spec/cache-scope-public',
|
|
64
|
+
severity: 'warn',
|
|
65
|
+
since: '2026-07-28',
|
|
66
|
+
summary: 'A list marked cacheScope "public" that differs by identity',
|
|
67
|
+
run(ctx) {
|
|
68
|
+
const identities = ctx.identities ?? [];
|
|
69
|
+
const varying = identityDifferences(identities).filter(differs);
|
|
70
|
+
if (!varying.length)
|
|
71
|
+
return [];
|
|
72
|
+
const listed = identities.filter((i) => i.tools);
|
|
73
|
+
const publicOnes = listed.filter((i) => i.cacheScope === 'public').map((i) => i.name);
|
|
74
|
+
if (!publicOnes.length)
|
|
75
|
+
return [];
|
|
76
|
+
return [
|
|
77
|
+
{
|
|
78
|
+
message: `tools/list says cacheScope: "public"${publicOnes.length < listed.length ? ` (for ${names(publicOnes)})` : ''}, but the tool set differs by identity. Public lets shared caches serve one caller's list to another, across authorization contexts: a client can be shown tools its credentials can't use, or miss ones they can.`,
|
|
79
|
+
detail: varying.map((d) => describe(d, identities[0])),
|
|
80
|
+
fix: 'Set cacheScope: "private" on tools/list when the tools depend on who is asking.',
|
|
81
|
+
},
|
|
82
|
+
];
|
|
83
|
+
},
|
|
84
|
+
};
|
package/dist/rules/index.js
CHANGED
|
@@ -2,6 +2,8 @@ import { SEVERITY_RANK } from '../types.js';
|
|
|
2
2
|
import { buried, cut, lateInstruction } from './description.js';
|
|
3
3
|
import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
|
+
import { handleLifetime } from './handles.js';
|
|
6
|
+
import { authVariance, cacheScopePublic } from './identities.js';
|
|
5
7
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
6
8
|
import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
|
|
7
9
|
import { noDryRun, unannotated } from './write.js';
|
|
@@ -21,6 +23,7 @@ export const MENU_RULES = [
|
|
|
21
23
|
route,
|
|
22
24
|
findable,
|
|
23
25
|
authoredIds,
|
|
26
|
+
handleLifetime,
|
|
24
27
|
buried,
|
|
25
28
|
cut,
|
|
26
29
|
lateInstruction,
|
|
@@ -33,6 +36,8 @@ export const MENU_RULES = [
|
|
|
33
36
|
discover,
|
|
34
37
|
deprecated,
|
|
35
38
|
cacheHints,
|
|
39
|
+
authVariance,
|
|
40
|
+
cacheScopePublic,
|
|
36
41
|
serverInfo,
|
|
37
42
|
];
|
|
38
43
|
/**
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -26,6 +26,8 @@ export interface RuleContext {
|
|
|
26
26
|
};
|
|
27
27
|
/** How this run reached the server, as the command line ends: `-- node server.js` or the URL. For next steps. */
|
|
28
28
|
server?: string;
|
|
29
|
+
/** snapshot --as: each identity's menu, the main one first (menu/auth-variance, spec/cache-scope-public). */
|
|
30
|
+
identities?: import('./identities.js').IdentityMenu[];
|
|
29
31
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
|
30
32
|
descriptionLimit?: number | string;
|
|
31
33
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
package/dist/rules/spec.js
CHANGED
|
@@ -167,7 +167,8 @@ export const cacheHints = {
|
|
|
167
167
|
fix: 'If the tool set is stable, set a positive ttlMs on tools/list (3600000 is an hour).',
|
|
168
168
|
});
|
|
169
169
|
}
|
|
170
|
-
|
|
170
|
+
// With --as identities, spec/cache-scope-public answers this with evidence instead.
|
|
171
|
+
if (meta.cacheScope === 'public' && ctx.usedAuth && (ctx.identities?.length ?? 0) < 2) {
|
|
171
172
|
findings.push({
|
|
172
173
|
message: 'tools/list says cacheScope: "public" on a server that took credentials. Public lets shared caches serve the list to other callers, across authorization contexts. Fine only if the tool set never depends on who is asking.',
|
|
173
174
|
fix: 'If tools depend on the caller\'s account or scopes, set cacheScope: "private" on tools/list.',
|
package/dist/session.d.ts
CHANGED
|
@@ -15,6 +15,12 @@ export type Step = {
|
|
|
15
15
|
* alone. If its first such call changes nothing, the rest aren't made.
|
|
16
16
|
*/
|
|
17
17
|
tentative?: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* --auto only (2026-07-28): called with a handle that can't exist, to see what
|
|
20
|
+
* the model gets back. A tool error is the right answer; a JSON-RPC error is
|
|
21
|
+
* session/handle-error.
|
|
22
|
+
*/
|
|
23
|
+
probe?: import('./rules/handles.js').HandleProbe;
|
|
18
24
|
} | {
|
|
19
25
|
kind: 'wait_for';
|
|
20
26
|
event: 'tools_list_changed';
|
package/dist/session.js
CHANGED
|
@@ -78,6 +78,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
78
78
|
const raw = [];
|
|
79
79
|
// 2026-07-28: every call that asked for input, for the input findings at the end.
|
|
80
80
|
const inputs = [];
|
|
81
|
+
// --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
|
|
82
|
+
const handleErrors = [];
|
|
81
83
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
82
84
|
// a probe with a different seed would count ordering variance as a change.
|
|
83
85
|
// How every elicitation is answered: the same each run (--elicitation). Session
|
|
@@ -173,7 +175,9 @@ export async function session(target, scenario, options = {}) {
|
|
|
173
175
|
// this one when that happens.
|
|
174
176
|
let probedAfter = probes ? 0 : undefined;
|
|
175
177
|
const lost = (why, index) => {
|
|
176
|
-
|
|
178
|
+
// 2026-07-28 has no protocol sessions: "unknown session" from such a server is
|
|
179
|
+
// about a session it minted itself, a handle the model passes (SEP-2567).
|
|
180
|
+
if (probedAfter === undefined || conn.era === 'modern' || !SESSION_LOST.test(why))
|
|
177
181
|
return false;
|
|
178
182
|
// The server's own words, not the transport's wrapping of its JSON-RPC error.
|
|
179
183
|
const said = serverWords(why);
|
|
@@ -305,7 +309,21 @@ export async function session(target, scenario, options = {}) {
|
|
|
305
309
|
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
306
310
|
if (call.input.rounds && !result.isError)
|
|
307
311
|
record.note = inputNote(call.input, answer);
|
|
308
|
-
|
|
312
|
+
const setup = step.probe && result.isError ? classifyFailure(errorText(result), { hadArguments: true }) : undefined;
|
|
313
|
+
if (step.probe && setup && SETUP_FAILURES.has(setup)) {
|
|
314
|
+
// Missing credentials, this machine: the call never got as far as the
|
|
315
|
+
// handle, so it says nothing about it. Counted with the other setup failures.
|
|
316
|
+
record.failure = setup;
|
|
317
|
+
record.note = `called with a made-up ${step.probe.kind}, but it failed before the ${step.probe.kind} mattered: ${clip(errorText(result), 100)}`;
|
|
318
|
+
}
|
|
319
|
+
else if (step.probe) {
|
|
320
|
+
// A handle that can't exist: a tool error is the answer the spec asks for.
|
|
321
|
+
const text = result.isError ? errorText(result) : '';
|
|
322
|
+
record.note = result.isError
|
|
323
|
+
? `called with a made-up ${step.probe.kind}; it answered with a tool error${text ? `: ${clip(text, 100)}` : ''}`
|
|
324
|
+
: `called with a made-up ${step.probe.kind}; it answered as if that ${step.probe.kind} existed`;
|
|
325
|
+
}
|
|
326
|
+
else if (result.isError) {
|
|
309
327
|
const text = errorText(result);
|
|
310
328
|
if (RATE_LIMITED.test(text)) {
|
|
311
329
|
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
@@ -339,7 +357,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
339
357
|
record.status = 'failed';
|
|
340
358
|
record.outcome = 'failed';
|
|
341
359
|
record.note = serverWords(error, 200);
|
|
342
|
-
|
|
360
|
+
// A call with a made-up handle ("Unknown session: toolmenu-unknown-session") is about that handle.
|
|
361
|
+
if (!step.probe && lost(record.note, index)) {
|
|
343
362
|
done();
|
|
344
363
|
break;
|
|
345
364
|
}
|
|
@@ -356,11 +375,21 @@ export async function session(target, scenario, options = {}) {
|
|
|
356
375
|
if (needed.length)
|
|
357
376
|
record.note = `${record.note} (needs the client to support ${needed.join(', ')})`;
|
|
358
377
|
}
|
|
359
|
-
|
|
360
|
-
//
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
378
|
+
const probeFailure = step.probe ? classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: true }) : undefined;
|
|
379
|
+
// Not a setup failure (credentials, this machine, -32021): those never got as far as the handle.
|
|
380
|
+
if (step.probe && typeof code === 'number' && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
|
|
381
|
+
// The server's protocol error to a handle that can't exist: the model gets no reason it can act on.
|
|
382
|
+
if (!handleErrors.some((e) => e.tool === step.tool))
|
|
383
|
+
handleErrors.push({ step: index, tool: step.tool, probe: step.probe, value: step.args[step.probe.param], message: record.note, code });
|
|
384
|
+
record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
388
|
+
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
389
|
+
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
390
|
+
if (!SETUP_FAILURES.has(record.failure))
|
|
391
|
+
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
392
|
+
}
|
|
364
393
|
}
|
|
365
394
|
}
|
|
366
395
|
else if (step.kind === 'wait_for') {
|
|
@@ -472,6 +501,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
472
501
|
}
|
|
473
502
|
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
474
503
|
raw.push(...inputFindings(inputs, answer));
|
|
504
|
+
raw.push(...handleErrorFindings(handleErrors));
|
|
475
505
|
if (explained.length)
|
|
476
506
|
raw.push(knownVariance(explained));
|
|
477
507
|
if (onWord.length)
|
|
@@ -1426,6 +1456,23 @@ function inputNote(input, answer) {
|
|
|
1426
1456
|
return asked;
|
|
1427
1457
|
}
|
|
1428
1458
|
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Handles that can't exist answered with a protocol error (2026-07-28, SEP-2567):
|
|
1461
|
+
* "a call against an expired or unknown handle should return a tool execution
|
|
1462
|
+
* error that says so, so the model can recover by creating a new one." One
|
|
1463
|
+
* finding per tool.
|
|
1464
|
+
*/
|
|
1465
|
+
function handleErrorFindings(errors) {
|
|
1466
|
+
return errors.map((e) => ({
|
|
1467
|
+
rule: 'session/handle-error',
|
|
1468
|
+
severity: 'warn',
|
|
1469
|
+
confidence: 'unsure',
|
|
1470
|
+
step: e.step,
|
|
1471
|
+
tool: e.tool,
|
|
1472
|
+
message: `${e.tool} answers a ${e.probe.kind} that doesn't exist with a protocol error (${e.code}: “${clip(e.message, 120)}”), not a tool error. A model whose ${e.probe.kind} expired gets an error from its client instead of a result it can read, so it can't tell it should start a new one. toolmenu passed ${e.probe.param}: ${JSON.stringify(e.value)}, which no ${e.probe.kind} has.`,
|
|
1473
|
+
fix: `Return a tool result with isError: true that says the ${e.probe.kind} is unknown or expired and how to get a new one ("Session not found or expired: call start_session again").`,
|
|
1474
|
+
}));
|
|
1475
|
+
}
|
|
1429
1476
|
/**
|
|
1430
1477
|
* What tools that asked for input did with the answer (2026-07-28 multi
|
|
1431
1478
|
* round-trip requests). One info finding for all of them, and a warning or error
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type Target } from './connect.js';
|
|
2
|
+
import { type Identity } from './probe.js';
|
|
3
|
+
import type { IdentityMenu } from './rules/identities.js';
|
|
2
4
|
import { type RuleSettings } from './rules/index.js';
|
|
3
5
|
import { type CatalogOptions } from './catalog.js';
|
|
4
6
|
import type { Routes } from './routes.js';
|
|
@@ -15,13 +17,20 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
15
17
|
processes?: number;
|
|
16
18
|
/** Also read the operations behind a catalog search tool or command routers (true, or which ones and how). */
|
|
17
19
|
catalog?: boolean | CatalogOptions;
|
|
20
|
+
/**
|
|
21
|
+
* snapshot --as: list the menu once per identity. The first is the main one: its
|
|
22
|
+
* menu is the one saved and checked; the others are compared with it.
|
|
23
|
+
*/
|
|
24
|
+
identities?: Identity[];
|
|
18
25
|
}
|
|
19
26
|
export interface SnapshotResult {
|
|
20
27
|
menu: Menu;
|
|
21
28
|
findings: Finding[];
|
|
29
|
+
/** snapshot --as: each identity's tools and tokens, or why it couldn't be listed. */
|
|
30
|
+
identities?: IdentityMenu[];
|
|
22
31
|
}
|
|
23
32
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
24
|
-
export declare function snapshot(
|
|
33
|
+
export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
|
25
34
|
/**
|
|
26
35
|
* How this run reached the server, as the end of a command line, for next steps:
|
|
27
36
|
* the options that got it past the door (names only; values are the user's) and
|
package/dist/snapshot.js
CHANGED
|
@@ -2,13 +2,15 @@ import { listTools } from './connect.js';
|
|
|
2
2
|
import { patiently, tooMany } from './failures.js';
|
|
3
3
|
import { ConnectError } from './explain.js';
|
|
4
4
|
import { buildMenu } from './menu.js';
|
|
5
|
-
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
5
|
+
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
6
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
7
7
|
import { catalogFindings, readCatalog } from './catalog.js';
|
|
8
8
|
import { SEVERITY_RANK } from './types.js';
|
|
9
9
|
import { shellWord } from './init.js';
|
|
10
10
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
11
|
-
export async function snapshot(
|
|
11
|
+
export async function snapshot(given, options = {}) {
|
|
12
|
+
const [mainIdentity, ...otherIdentities] = options.identities ?? [];
|
|
13
|
+
const target = asIdentity(given, mainIdentity);
|
|
12
14
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
13
15
|
// out and they're sent again (CI behind a shared rate limit).
|
|
14
16
|
const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
@@ -43,6 +45,19 @@ export async function snapshot(target, options = {}) {
|
|
|
43
45
|
// After the main process has exited: a server that holds a file or a port
|
|
44
46
|
// shouldn't have to share it with its own probe.
|
|
45
47
|
const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000);
|
|
48
|
+
// Each other identity's menu, from its own process (stdio) or connection (HTTP).
|
|
49
|
+
const identities = mainIdentity
|
|
50
|
+
? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
|
|
51
|
+
: undefined;
|
|
52
|
+
for (const identity of otherIdentities) {
|
|
53
|
+
try {
|
|
54
|
+
const m = await probeMenu(seeded(asIdentity(given, identity), MAIN_SEED), options.timeoutMs ?? 30_000);
|
|
55
|
+
identities.push({ name: identity.name, tools: m.tools, tokens: m.totalTokens, cacheScope: m.listMeta?.cacheScope });
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
identities.push({ name: identity.name, error: firstLine(error) });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
46
61
|
const findings = runRules(MENU_RULES, {
|
|
47
62
|
menu,
|
|
48
63
|
secondList,
|
|
@@ -60,6 +75,7 @@ export async function snapshot(target, options = {}) {
|
|
|
60
75
|
wrapper: isContainerWrapper(target),
|
|
61
76
|
serverInfo: connection.server,
|
|
62
77
|
server: serverArgs(target),
|
|
78
|
+
identities,
|
|
63
79
|
}, options);
|
|
64
80
|
// What --catalog called and what it got, under the same settings as the rules
|
|
65
81
|
// (rules: { "catalog/read": "off" }, ignore).
|
|
@@ -73,8 +89,9 @@ export async function snapshot(target, options = {}) {
|
|
|
73
89
|
}
|
|
74
90
|
findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
75
91
|
}
|
|
76
|
-
return { menu, findings };
|
|
92
|
+
return { menu, findings, ...(identities ? { identities } : {}) };
|
|
77
93
|
}
|
|
94
|
+
const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
|
|
78
95
|
/**
|
|
79
96
|
* How this run reached the server, as the end of a command line, for next steps:
|
|
80
97
|
* the options that got it past the door (names only; values are the user's) and
|
package/package.json
CHANGED