toolmenu 0.20.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +119 -41
  2. package/dist/apps.d.ts +56 -0
  3. package/dist/apps.js +99 -0
  4. package/dist/auth.d.ts +14 -0
  5. package/dist/auth.js +75 -2
  6. package/dist/auto.d.ts +6 -0
  7. package/dist/auto.js +70 -7
  8. package/dist/card.d.ts +18 -0
  9. package/dist/card.js +39 -0
  10. package/dist/cli.js +108 -24
  11. package/dist/colour.d.ts +8 -0
  12. package/dist/colour.js +70 -0
  13. package/dist/config.d.ts +2 -0
  14. package/dist/config.js +2 -0
  15. package/dist/connect.d.ts +14 -0
  16. package/dist/connect.js +45 -7
  17. package/dist/diff.js +112 -13
  18. package/dist/explain.d.ts +1 -0
  19. package/dist/explain.js +11 -0
  20. package/dist/failures.d.ts +24 -0
  21. package/dist/failures.js +58 -1
  22. package/dist/help.d.ts +1 -1
  23. package/dist/help.js +36 -4
  24. package/dist/mrtr.d.ts +9 -1
  25. package/dist/mrtr.js +17 -0
  26. package/dist/options.d.ts +15 -0
  27. package/dist/options.js +5 -0
  28. package/dist/probe.d.ts +11 -1
  29. package/dist/probe.js +18 -3
  30. package/dist/report.d.ts +3 -0
  31. package/dist/report.js +15 -8
  32. package/dist/rule-docs.d.ts +26 -0
  33. package/dist/rule-docs.js +88 -0
  34. package/dist/rule-docs.json +597 -0
  35. package/dist/rules/apps.d.ts +11 -0
  36. package/dist/rules/apps.js +156 -0
  37. package/dist/rules/auth.d.ts +18 -0
  38. package/dist/rules/auth.js +29 -0
  39. package/dist/rules/capabilities.d.ts +21 -0
  40. package/dist/rules/capabilities.js +103 -0
  41. package/dist/rules/card.d.ts +7 -0
  42. package/dist/rules/card.js +65 -0
  43. package/dist/rules/clients.js +1 -0
  44. package/dist/rules/description.js +3 -0
  45. package/dist/rules/handles.d.ts +13 -0
  46. package/dist/rules/handles.js +78 -0
  47. package/dist/rules/ids.js +35 -1
  48. package/dist/rules/index.js +20 -2
  49. package/dist/rules/naming.js +3 -0
  50. package/dist/rules/network.d.ts +21 -0
  51. package/dist/rules/network.js +63 -0
  52. package/dist/rules/rule.d.ts +20 -0
  53. package/dist/rules/search.js +1 -0
  54. package/dist/rules/spec.d.ts +9 -1
  55. package/dist/rules/spec.js +117 -3
  56. package/dist/rules/write.d.ts +0 -6
  57. package/dist/rules/write.js +11 -5
  58. package/dist/scan.d.ts +47 -0
  59. package/dist/scan.js +334 -0
  60. package/dist/session.d.ts +2 -0
  61. package/dist/session.js +149 -9
  62. package/dist/snapshot.d.ts +15 -0
  63. package/dist/snapshot.js +130 -5
  64. package/dist/types.d.ts +5 -0
  65. package/dist/words.d.ts +10 -0
  66. package/dist/words.js +44 -2
  67. package/package.json +2 -2
  68. package/schemas/server-card.json +260 -0
package/dist/scan.js ADDED
@@ -0,0 +1,334 @@
1
+ /*
2
+ * `toolmenu scan <list>`: snapshot many servers in one run, a few at a time, and
3
+ * report them side by side. The list is an MCP client config (Claude Desktop,
4
+ * Claude Code's .mcp.json, Cursor, VS Code: `mcpServers` or `servers`) or a text
5
+ * file with one server per line. A server that couldn't be read is a row that
6
+ * says why, never a pass.
7
+ */
8
+ import { readFile } from 'node:fs/promises';
9
+ import { ConnectError, clean, commandLine } from './explain.js';
10
+ import { counts, findingLines, githubLines, mdFindings } from './report.js';
11
+ import { snapshot } from './snapshot.js';
12
+ import { SEVERITY_RANK } from './types.js';
13
+ export class ListError extends Error {
14
+ }
15
+ /** `${VAR}`, `${VAR:-default}` (Claude Code) and `${env:VAR}` (VS Code), from the environment. */
16
+ function expand(value, env, missing) {
17
+ return value.replace(/\$\{(?:env:)?([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g, (_, name, fallback) => {
18
+ const v = env[name];
19
+ if (v !== undefined && v !== '')
20
+ return v;
21
+ if (fallback !== undefined)
22
+ return fallback;
23
+ missing.add(name);
24
+ return '';
25
+ });
26
+ }
27
+ function strings(v) {
28
+ return Array.isArray(v) && v.every((x) => typeof x === 'string');
29
+ }
30
+ function record(v) {
31
+ if (!v || typeof v !== 'object' || Array.isArray(v))
32
+ return undefined;
33
+ return Object.fromEntries(Object.entries(v).filter(([, x]) => typeof x === 'string'));
34
+ }
35
+ /** One server from a client config: `{ command, args, env, cwd }` or `{ url, headers }`. */
36
+ function fromConfig(name, raw, env) {
37
+ if (!raw || typeof raw !== 'object')
38
+ return { name, problem: 'not an object' };
39
+ const s = raw;
40
+ if (s.disabled === true || s.enabled === false)
41
+ return { name, disabled: true };
42
+ const missing = new Set();
43
+ const x = (v) => expand(v, env, missing);
44
+ const map = (r) => (r ? Object.fromEntries(Object.entries(r).map(([k, v]) => [k, x(v)])) : undefined);
45
+ const type = typeof s.type === 'string' ? s.type : typeof s.transport === 'string' ? s.transport : undefined;
46
+ let target;
47
+ if (typeof s.url === 'string' || typeof s.serverUrl === 'string' || typeof s.httpUrl === 'string') {
48
+ if (type === 'sse')
49
+ return { name, problem: 'HTTP+SSE transport, which toolmenu doesn\'t speak (Streamable HTTP only)' };
50
+ const url = x((s.url ?? s.serverUrl ?? s.httpUrl));
51
+ const headers = map(record(s.headers));
52
+ target = { kind: 'http', url, ...(headers && Object.keys(headers).length ? { headers } : {}) };
53
+ }
54
+ else if (typeof s.command === 'string') {
55
+ if (s.args !== undefined && !strings(s.args))
56
+ return { name, problem: 'args is not a list of strings' };
57
+ const envs = map(record(s.env));
58
+ target = {
59
+ kind: 'stdio',
60
+ command: x(s.command),
61
+ args: (s.args ?? []).map(x),
62
+ ...(envs && Object.keys(envs).length ? { env: envs } : {}),
63
+ ...(typeof s.cwd === 'string' ? { cwd: x(s.cwd) } : {}),
64
+ };
65
+ }
66
+ else
67
+ return { name, problem: 'neither a command nor a url' };
68
+ // A missing variable would send an empty credential: say so instead of checking a server set up wrong.
69
+ if (missing.size)
70
+ return { name, problem: `${[...missing].join(', ')} ${missing.size === 1 ? 'is' : 'are'} not set (the config refers to ${missing.size === 1 ? 'it' : 'them'})` };
71
+ if (target.kind === 'http' && !/^https?:\/\//.test(target.url))
72
+ return { name, problem: `"${clean(target.url, 60)}" isn't an http(s) URL` };
73
+ const shown = target.kind === 'http'
74
+ ? { kind: 'http', url: (s.url ?? s.serverUrl ?? s.httpUrl) }
75
+ : { kind: 'stdio', command: s.command, args: s.args ?? [] };
76
+ return { name, target, shown };
77
+ }
78
+ /** Words of a command line: whitespace-separated, with '…' and "…" kept together. */
79
+ export function splitCommand(line) {
80
+ const out = [];
81
+ let word = '';
82
+ let quote;
83
+ let started = false;
84
+ for (let i = 0; i < line.length; i++) {
85
+ const c = line[i];
86
+ if (quote) {
87
+ if (c === quote)
88
+ quote = undefined;
89
+ else if (c === '\\' && quote === '"' && i + 1 < line.length)
90
+ word += line[++i];
91
+ else
92
+ word += c;
93
+ }
94
+ else if (c === '"' || c === "'") {
95
+ quote = c;
96
+ started = true;
97
+ }
98
+ else if (/\s/.test(c)) {
99
+ if (started)
100
+ out.push(word);
101
+ word = '';
102
+ started = false;
103
+ }
104
+ else {
105
+ word += c;
106
+ started = true;
107
+ }
108
+ }
109
+ if (quote)
110
+ throw new ListError(`an unclosed ${quote} in: ${clean(line, 80)}`);
111
+ if (started)
112
+ out.push(word);
113
+ return out;
114
+ }
115
+ /** The name a line gets when it gives none: the URL's host, or the package or command. */
116
+ function defaultName(target) {
117
+ if (target.kind === 'http')
118
+ return new URL(target.url).host;
119
+ const pkg = target.args.find((a) => !a.startsWith('-') && /^(?:@[\w.-]+\/)?[\w.-]+(?:[@=]=?[\w.^~<>=*-]+)?$/.test(a) && !/^(?:-y|--yes|run|exec|x)$/.test(a));
120
+ const launcher = /(?:^|\/)(?:npx|bunx|pnpx|uvx|pipx|dlx)$/.test(target.command) || /(?:^|\/)(?:pnpm|yarn|bun|npm)$/.test(target.command);
121
+ if (launcher && pkg)
122
+ return pkg;
123
+ // node server.js, python -m x, deno run main.ts: the script, not the interpreter.
124
+ const script = target.args.find((a) => !a.startsWith('-') && /\.(?:[cm]?[jt]s|py)$/.test(a));
125
+ if (script)
126
+ return script.split('/').pop().replace(/\.[^.]+$/, '');
127
+ return target.command.split('/').pop() || target.command;
128
+ }
129
+ /**
130
+ * A text list: one server per line, a URL or the command that starts it, optionally
131
+ * named first ("github<TAB>npx -y @x/github-mcp", or "github = https://…").
132
+ * Blank lines and # comments are skipped.
133
+ */
134
+ function fromLines(text) {
135
+ const entries = [];
136
+ for (const [i, raw] of text.split('\n').entries()) {
137
+ const line = raw.replace(/^\s*#.*$/, '').trim();
138
+ if (!line)
139
+ continue;
140
+ let name;
141
+ let rest = line;
142
+ const named = /^([\w.@/-]+)(?:\t+|\s+=\s+)(.+)$/.exec(line);
143
+ if (named)
144
+ [, name, rest] = named;
145
+ let target;
146
+ if (/^https?:\/\//.test(rest)) {
147
+ if (/\s/.test(rest.trim()))
148
+ throw new ListError(`line ${i + 1}: a URL and something after it; one server per line`);
149
+ target = { kind: 'http', url: rest.trim() };
150
+ if (!URL.canParse(target.url)) {
151
+ entries.push({ name: name ?? `line ${i + 1}`, problem: `"${clean(target.url, 60)}" isn't a valid URL` });
152
+ continue;
153
+ }
154
+ }
155
+ else {
156
+ const words = splitCommand(rest);
157
+ target = { kind: 'stdio', command: words[0], args: words.slice(1) };
158
+ }
159
+ entries.push({ name: name ?? defaultName(target), target });
160
+ }
161
+ return entries;
162
+ }
163
+ /** The servers in a list file, unique names (a repeat gets #2, #3). */
164
+ export function parseList(text, file, env = process.env) {
165
+ let entries;
166
+ if (/\.jsonc?$/i.test(file) || /^\s*\{/.test(text)) {
167
+ let json;
168
+ try {
169
+ // .jsonc and VS Code's settings allow comments; strip whole-line ones.
170
+ json = JSON.parse(text.replace(/^\s*\/\/.*$/gm, ''));
171
+ }
172
+ catch (error) {
173
+ throw new ListError(`${file} isn't valid JSON: ${error.message}`);
174
+ }
175
+ const j = json;
176
+ const servers = j.mcpServers ?? j.servers ?? j.mcp?.servers ?? j['mcp.servers'];
177
+ if (!servers || typeof servers !== 'object' || Array.isArray(servers))
178
+ throw new ListError(`${file} has no mcpServers or servers object, the key MCP clients keep their servers under`);
179
+ entries = Object.entries(servers).map(([name, raw]) => fromConfig(name, raw, env));
180
+ }
181
+ else
182
+ entries = fromLines(text);
183
+ if (!entries.length)
184
+ throw new ListError(`${file} lists no servers`);
185
+ const seen = new Map();
186
+ return entries.map((e) => {
187
+ const n = (seen.get(e.name) ?? 0) + 1;
188
+ seen.set(e.name, n);
189
+ return n === 1 ? e : { ...e, name: `${e.name}#${n}` };
190
+ });
191
+ }
192
+ export async function loadList(file) {
193
+ let text;
194
+ try {
195
+ text = await readFile(file, 'utf8');
196
+ }
197
+ catch (error) {
198
+ throw new ListError(`couldn't read ${file}: ${error.code ?? error.message}`);
199
+ }
200
+ return parseList(text, file);
201
+ }
202
+ /** Printable location. A URL's query and credentials can hold keys, so they're left out. */
203
+ export function where(target) {
204
+ if (!target)
205
+ return '';
206
+ if (target.kind === 'stdio')
207
+ return commandLine(target.command, target.args);
208
+ try {
209
+ const u = new URL(target.url);
210
+ return `${u.origin}${u.pathname}${u.search ? '?…' : ''}`;
211
+ }
212
+ catch {
213
+ return clean(target.url, 80);
214
+ }
215
+ }
216
+ /** A one-line reason a server wasn't read: a ConnectError's headline, else the first line of the error. */
217
+ function reason(error, target, shown) {
218
+ const text = error instanceof ConnectError ? error.headline : clean((error instanceof Error ? error.message : String(error)).split('\n')[0], 200);
219
+ // The URL can carry a key a ${VAR} filled in: print it as where() does.
220
+ return target.kind === 'http' ? text.split(target.url).join(where(shown)) : text;
221
+ }
222
+ export async function scan(entries, options = {}) {
223
+ const rows = new Array(entries.length);
224
+ let next = 0;
225
+ let done = 0;
226
+ const worker = async () => {
227
+ while (next < entries.length) {
228
+ const i = next++;
229
+ const e = entries[i];
230
+ let row;
231
+ if (e.disabled)
232
+ row = { name: e.name, where: '', findings: [], disabled: true };
233
+ else if (!e.target)
234
+ row = { name: e.name, where: '', findings: [], failed: e.problem };
235
+ else {
236
+ try {
237
+ const { menu, findings } = await snapshot(e.target, options);
238
+ row = { name: e.name, where: where(e.shown ?? e.target), menu, findings };
239
+ }
240
+ catch (error) {
241
+ const status = error instanceof ConnectError ? error.status : undefined;
242
+ row = { name: e.name, where: where(e.shown ?? e.target), findings: [], failed: reason(error, e.target, e.shown ?? e.target), ...(status === 401 || status === 403 ? { wantsCredentials: true } : {}) };
243
+ }
244
+ }
245
+ rows[i] = row;
246
+ options.onDone?.(row, ++done, entries.length);
247
+ }
248
+ };
249
+ await Promise.all(Array.from({ length: Math.min(options.concurrency ?? 4, entries.length) }, worker));
250
+ return rows;
251
+ }
252
+ /** 0 every server read and nothing at --fail-on; 1 findings at --fail-on; 2 a server wasn't read. */
253
+ export function scanExit(rows, failOn) {
254
+ if (rows.some((r) => r.failed))
255
+ return 2;
256
+ return rows.some((r) => r.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn])) ? 1 : 0;
257
+ }
258
+ function tokens(n) {
259
+ return n >= 10_000 ? `${Math.round(n / 1000)}K` : n >= 1000 ? `${(n / 1000).toFixed(1)}K` : String(n);
260
+ }
261
+ function countText(findings) {
262
+ const c = counts(findings);
263
+ const parts = [c.error ? `${c.error} error` : '', c.warn ? `${c.warn} warn` : '', c.info ? `${c.info} info` : ''].filter(Boolean);
264
+ return parts.length ? parts.join(' · ') : 'no findings';
265
+ }
266
+ function status(r) {
267
+ if (r.disabled)
268
+ return 'disabled in the config';
269
+ if (r.failed)
270
+ return `not checked: ${r.failed}`;
271
+ return countText(r.findings);
272
+ }
273
+ function totals(rows) {
274
+ const read = rows.filter((r) => r.menu);
275
+ const off = rows.filter((r) => r.disabled).length;
276
+ const notRead = rows.length - read.length - off;
277
+ const all = read.flatMap((r) => r.findings);
278
+ const c = counts(all);
279
+ return `${read.length} of ${rows.length} server${rows.length === 1 ? '' : 's'} checked${notRead ? `, ${notRead} not` : ''}${off ? `, ${off} disabled` : ''} · ${c.error} error · ${c.warn} warn · ${c.info} info`;
280
+ }
281
+ export function formatScan(rows, format, file) {
282
+ if (format === 'json') {
283
+ return JSON.stringify({
284
+ command: 'scan',
285
+ list: file,
286
+ servers: rows.map((r) => ({
287
+ name: r.name,
288
+ where: r.where,
289
+ checked: !!r.menu,
290
+ ...(r.menu ? { server: r.menu.server, tools: r.menu.tools.length, totalTokens: r.menu.totalTokens, counts: counts(r.findings) } : {}),
291
+ ...(r.failed ? { failed: r.failed } : {}),
292
+ ...(r.disabled ? { disabled: true } : {}),
293
+ findings: r.findings,
294
+ })),
295
+ }, null, 2);
296
+ }
297
+ if (format === 'github') {
298
+ const lines = [];
299
+ for (const r of rows) {
300
+ if (r.failed)
301
+ lines.push(`::error title=toolmenu scan ${r.name}::${status(r)}`);
302
+ lines.push(...githubLines(r.findings.map((f) => ({ ...f, message: `${r.name}: ${f.message}` }))));
303
+ }
304
+ return lines.join('\n');
305
+ }
306
+ if (format === 'markdown') {
307
+ const cell = (t) => t.replace(/\|/g, '\\|').replace(/\n/g, ' ');
308
+ const lines = [`### toolmenu scan: ${cell(file)}`, '', '| Server | Tools | Tokens | Findings |', '|---|--:|--:|---|'];
309
+ for (const r of rows)
310
+ lines.push(`| \`${cell(r.name)}\` | ${r.menu ? r.menu.tools.length : '–'} | ${r.menu ? `~${tokens(r.menu.totalTokens)}` : '–'} | ${cell(status(r))} |`);
311
+ lines.push('', totals(rows));
312
+ for (const r of rows.filter((x) => x.findings.length))
313
+ lines.push('', `#### \`${cell(r.name)}\``, '', ...mdFindings(r.findings));
314
+ return lines.join('\n');
315
+ }
316
+ const nameWidth = Math.min(32, Math.max(6, ...rows.map((r) => r.name.length)));
317
+ const lines = [`toolmenu scan ${file}`, ''];
318
+ lines.push(` ${'server'.padEnd(nameWidth)} ${'tools'.padStart(5)} ${'tokens'.padStart(6)} findings`);
319
+ for (const r of rows) {
320
+ const name = r.name.length > nameWidth ? `${r.name.slice(0, nameWidth - 1)}…` : r.name;
321
+ lines.push(` ${name.padEnd(nameWidth)} ${(r.menu ? String(r.menu.tools.length) : '–').padStart(5)} ${(r.menu ? `~${tokens(r.menu.totalTokens)}` : '–').padStart(6)} ${status(r)}`);
322
+ }
323
+ lines.push('', totals(rows));
324
+ for (const r of rows.filter((x) => x.findings.length)) {
325
+ const s = r.menu.server;
326
+ lines.push('', `── ${r.name} ${s.name ?? ''} ${s.version ?? ''} · protocol ${s.protocolVersion ?? 'unknown'}`.trimEnd(), '');
327
+ lines.push(...findingLines(r.findings).filter((l, i, all) => !(i === all.length - 1 && l === '')));
328
+ }
329
+ if (rows.some((r) => r.wantsCredentials))
330
+ lines.push('', 'A server that wants credentials: toolmenu auth login <url> once (OAuth), or its headers in the config.');
331
+ if (rows.some((r) => r.failed && !r.wantsCredentials))
332
+ lines.push('', 'toolmenu snapshot on one server says more about why it couldn\'t be read.');
333
+ return lines.join('\n');
334
+ }
package/dist/session.d.ts CHANGED
@@ -21,6 +21,8 @@ export type Step = {
21
21
  * session/handle-error.
22
22
  */
23
23
  probe?: import('./rules/handles.js').HandleProbe;
24
+ /** --create-handles: this call creates a handle of this kind, for the probes after it. A write, allowed for this step only. */
25
+ creates?: string;
24
26
  } | {
25
27
  kind: 'wait_for';
26
28
  event: 'tools_list_changed';
package/dist/session.js CHANGED
@@ -4,14 +4,15 @@ import { cacheBreak, canonical, compareMenus } from './compare.js';
4
4
  import { listTools } from './connect.js';
5
5
  import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
6
6
  import { buildMenu, toolDefinition } from './menu.js';
7
- import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
+ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeSameEra, probeVariance, seeded } from './probe.js';
8
8
  import { varianceFinding } from './rules/determinism.js';
9
- import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
+ import { classifyFailure, errorWords, structuredError, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
10
10
  import { SEVERITY_RANK } from './types.js';
11
11
  import { leadingJson } from './catalog.js';
12
12
  import { allDifferences, describeToolDifference } from './difference.js';
13
13
  import { autoNextStep, neededValues, writeSign } from './auto.js';
14
14
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
15
+ import { handleFromResult } from './rules/handles.js';
15
16
  const LIST_CHANGED = 'notifications/tools/list_changed';
16
17
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
17
18
  const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
@@ -78,6 +79,11 @@ export async function session(target, scenario, options = {}) {
78
79
  const raw = [];
79
80
  // 2026-07-28: every call that asked for input, for the input findings at the end.
80
81
  const inputs = [];
82
+ // --create-handles: the handles creating tools returned, by kind, and what went wrong on the way.
83
+ const created = new Map();
84
+ const roundtrips = [];
85
+ // HTTP errors passed on as plain answers, not isError (session/error-not-flagged).
86
+ const unflagged = [];
81
87
  // --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
82
88
  const handleErrors = [];
83
89
  // The main process runs with a pinned hash seed, and so does every scope probe:
@@ -85,7 +91,8 @@ export async function session(target, scenario, options = {}) {
85
91
  // How every elicitation is answered: the same each run (--elicitation). Session
86
92
  // connections declare elicitation; snapshot's don't, so its menu is the one 0.14 saw.
87
93
  const answer = target.elicitation ?? 'decline';
88
- target = { ...target, elicitation: answer };
94
+ // And MCP Apps, as snapshot's baseline is listed: the same tools, the same variants.
95
+ target = { ...target, elicitation: answer, apps: true };
89
96
  const mainTarget = seeded(target, MAIN_SEED);
90
97
  const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
91
98
  // Connecting and listing are read-only: a refusal for being too many is waited
@@ -146,7 +153,7 @@ export async function session(target, scenario, options = {}) {
146
153
  };
147
154
  // Fresh processes or connections, same credentials, must see the same menu.
148
155
  let connectionCheck;
149
- for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
156
+ for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits, era)) {
150
157
  if (probe.error) {
151
158
  raw.push({
152
159
  rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
@@ -255,7 +262,7 @@ export async function session(target, scenario, options = {}) {
255
262
  fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
256
263
  });
257
264
  }
258
- else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
265
+ else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word && !(step.creates && tool.annotations?.destructiveHint !== true)) {
259
266
  record.status = 'refused';
260
267
  record.reason = 'refused';
261
268
  record.outcome = 'not-sent';
@@ -275,6 +282,11 @@ export async function session(target, scenario, options = {}) {
275
282
  }
276
283
  if (word)
277
284
  onWord.push({ tool: step.tool, step: index });
285
+ // --create-handles: the handle the creating tool returned, in place of a made-up one.
286
+ const real = step.probe?.fromCreated ? created.get(step.probe.kind) : undefined;
287
+ const args = real && step.probe ? { ...step.args, [step.probe.param]: real.value } : step.args;
288
+ if (real)
289
+ record.label = stepLabel({ ...step, args });
278
290
  if (record.status === 'ok')
279
291
  try {
280
292
  // Sent again after a refusal only when that repeats nothing: the transport's
@@ -285,7 +297,7 @@ export async function session(target, scenario, options = {}) {
285
297
  callWaited.ms = 0;
286
298
  // 2026-07-28: a tool that asks for input gets every elicitation declined
287
299
  // and is called again (mrtr.ts); on 2025 it's a plain call.
288
- const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: answer }), {
300
+ const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: args }, { timeoutMs, elicitation: answer }), {
289
301
  waits,
290
302
  waited: callWaited,
291
303
  // A 429 means the server didn't run that request, but after an input round the
@@ -309,12 +321,29 @@ export async function session(target, scenario, options = {}) {
309
321
  record.outcome = result.isError ? 'tool-error' : 'answered';
310
322
  if (call.input.rounds && !result.isError)
311
323
  record.note = inputNote(call.input, answer);
324
+ if (step.creates) {
325
+ // The handle this result carries, for the probes after it.
326
+ const value = result.isError ? undefined : handleFromResult(result, step.creates);
327
+ if (value)
328
+ created.set(step.creates, { tool: step.tool, value });
329
+ else
330
+ roundtrips.push({ step: index, tool: step.tool, kind: step.creates, problem: result.isError ? `returned an error: ${clip(errorText(result), 120)}` : 'returned no handle toolmenu could find' });
331
+ record.note = value ? `created a ${step.creates} (${clip(value, 40)}) for the calls after it` : `--create-handles: ${roundtrips.at(-1).problem}`;
332
+ }
312
333
  const setup = step.probe && result.isError ? classifyFailure(errorText(result), { hadArguments: true }) : undefined;
313
334
  if (step.probe && setup && SETUP_FAILURES.has(setup)) {
314
335
  // Missing credentials, this machine: the call never got as far as the
315
336
  // handle, so it says nothing about it. Counted with the other setup failures.
316
337
  record.failure = setup;
317
- record.note = `called with a made-up ${step.probe.kind}, but it failed before the ${step.probe.kind} mattered: ${clip(errorText(result), 100)}`;
338
+ record.note = `called with ${real ? `the ${step.probe.kind} ${real.tool} just created` : `a made-up ${step.probe.kind}`}, but it failed before the ${step.probe.kind} mattered: ${clip(errorText(result), 100)}`;
339
+ }
340
+ else if (step.probe && real) {
341
+ // The handle the creating tool just returned: an error here is the round trip broken.
342
+ record.note = result.isError
343
+ ? `called with the ${step.probe.kind} ${real.tool} just created; it answered with an error: ${clip(errorText(result), 100)}`
344
+ : `called with the ${step.probe.kind} ${real.tool} just created; it answered`;
345
+ if (result.isError)
346
+ roundtrips.push({ step: index, tool: step.tool, kind: step.probe.kind, creator: real.tool, problem: `answered with an error: ${clip(errorText(result), 160)}` });
318
347
  }
319
348
  else if (step.probe) {
320
349
  // A handle that can't exist: a tool error is the answer the spec asks for.
@@ -345,6 +374,20 @@ export async function session(target, scenario, options = {}) {
345
374
  fail('session/tool-error', 'warn', index, step, text, record.failure);
346
375
  }
347
376
  }
377
+ else if (!call.input.rounds && !step.creates) {
378
+ // An HTTP error passed on as an answer (not isError): a client that reads
379
+ // isError takes it for a success. An auth one also means the call reached the
380
+ // server but not the API behind it, so it tested no more than a refused one.
381
+ // A tool that takes a URL fetches it: what its answer says (a status, "Forbidden") is the page's.
382
+ const fetches = Object.keys(tool?.inputSchema?.properties ?? {}).some((p) => /^(?:url|uri|href|endpoint|link)s?$/i.test(p) || /_url$|Url$/.test(p));
383
+ const passedOn = fetches ? undefined : structuredError(result);
384
+ if (passedOn) {
385
+ unflagged.push({ step: index, tool: step.tool, said: passedOn.said });
386
+ record.note = `the tool answered, not as an error, with the API's ${clip(passedOn.said, 120)}`;
387
+ if (passedOn.auth)
388
+ record.failure = 'auth';
389
+ }
390
+ }
348
391
  }
349
392
  }
350
393
  catch (error) {
@@ -377,7 +420,11 @@ export async function session(target, scenario, options = {}) {
377
420
  }
378
421
  const probeFailure = step.probe ? classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: true }) : undefined;
379
422
  // Not a setup failure (credentials, this machine, -32021): those never got as far as the handle.
380
- if (step.probe && typeof code === 'number' && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
423
+ if (step.probe && real && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
424
+ roundtrips.push({ step: index, tool: step.tool, kind: step.probe.kind, creator: real.tool, problem: `failed: ${clip(record.note ?? '', 160)}` });
425
+ record.note = `called with the ${step.probe.kind} ${real.tool} just created; it failed: ${clip(record.note ?? '', 120)}`;
426
+ }
427
+ else if (step.probe && typeof code === 'number' && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
381
428
  // The server's protocol error to a handle that can't exist: the model gets no reason it can act on.
382
429
  if (!handleErrors.some((e) => e.tool === step.tool))
383
430
  handleErrors.push({ step: index, tool: step.tool, probe: step.probe, value: step.args[step.probe.param], message: record.note, code });
@@ -420,6 +467,31 @@ export async function session(target, scenario, options = {}) {
420
467
  done();
421
468
  break;
422
469
  }
470
+ if (/\btimed out after\b.*\bnever answered\b/.test(why)) {
471
+ // The server is running but answers nothing, tools/list included: every later
472
+ // step would wait out its timeout too (fulcra-context-mcp polls a device-code
473
+ // sign-in inside the call). One finding, and the run stops, as for a rate limit.
474
+ limitedAt = index;
475
+ // The call's own timeout at this step is the same cause: said by this finding.
476
+ for (const [key, f] of failures) {
477
+ if (!f.timedOutMs || f.steps.at(-1) !== index)
478
+ continue;
479
+ f.steps.pop();
480
+ if (!f.steps.length)
481
+ failures.delete(key);
482
+ }
483
+ const last = scenario.steps.length;
484
+ raw.push({
485
+ rule: 'session/server-unresponsive',
486
+ severity: 'error',
487
+ step: index,
488
+ ...(step.kind === 'call' ? { tool: step.tool } : {}),
489
+ message: `The server stopped answering at step ${index} (${label}): after it, ${why.replace(/\.$/, '')}. The run stopped there${index < last ? `: steps ${index + 1}–${last} weren't run` : ''}.`,
490
+ fix: 'A tool that waits for something outside the call (a sign-in in a browser, a device code, input on a terminal) blocks every request after it. The server\'s stderr may say what it waits for: sign in before the run, or pass credentials (--env), and rerun.',
491
+ });
492
+ done();
493
+ break;
494
+ }
423
495
  raw.push({
424
496
  rule: 'session/step-failed',
425
497
  severity: 'error',
@@ -475,7 +547,7 @@ export async function session(target, scenario, options = {}) {
475
547
  const probeWaited = { ms: 0 };
476
548
  try {
477
549
  probedAfter = index;
478
- const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
550
+ const probe = await probeSameEra(mainTarget, era, timeoutMs, waits, probeWaited);
479
551
  record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
480
552
  raw.push(...scopeFindings(record.scope, modern, index));
481
553
  }
@@ -502,6 +574,10 @@ export async function session(target, scenario, options = {}) {
502
574
  raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
503
575
  raw.push(...inputFindings(inputs, answer));
504
576
  raw.push(...handleErrorFindings(handleErrors));
577
+ raw.push(...roundtripFindings(roundtrips));
578
+ // `ignore` is applied by tool name; this finding sums several tools up, so it's applied here.
579
+ const ignored = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
580
+ raw.push(...unflaggedFindings(unflagged.filter((u) => !ignored.some((re) => re.test(u.tool)))));
505
581
  if (explained.length)
506
582
  raw.push(knownVariance(explained));
507
583
  if (onWord.length)
@@ -1450,12 +1526,61 @@ function inputNote(input, answer) {
1450
1526
  return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
1451
1527
  case 'decline-failed':
1452
1528
  return `${asked}; the retry with the ${answer} failed: ${clip(input.error?.message ?? '', 160)}`;
1529
+ case 'task':
1530
+ return `answered with a task to poll (resultType "task"${input.task?.taskId ? `, taskId ${clip(input.task.taskId, 40)}` : ''}${input.task?.status ? `, ${input.task.status}` : ''}), which the request didn't declare it could take`;
1453
1531
  case 'bad-state':
1454
1532
  return `${input.kinds.length ? `${asked}, with` : 'answered input_required with'} a requestState that isn't a string, so it can't be sent back as it came`;
1455
1533
  default:
1456
1534
  return asked;
1457
1535
  }
1458
1536
  }
1537
+ /**
1538
+ * Tools that passed an API's HTTP error on as a plain answer, not isError: one finding
1539
+ * for the run. The spec reports tool execution errors, API failures among them, in
1540
+ * the result with isError: true; a client that reads it (retry logic, its UI, a check
1541
+ * like this one) takes the call for a success.
1542
+ */
1543
+ function unflaggedFindings(list) {
1544
+ if (!list.length)
1545
+ return [];
1546
+ const tools = [...new Set(list.map((u) => u.tool))];
1547
+ return [
1548
+ {
1549
+ rule: 'session/error-not-flagged',
1550
+ severity: 'warn',
1551
+ // Read from the answer's shape: a tool may return an error object as its data.
1552
+ confidence: 'unsure',
1553
+ ...(tools.length === 1 ? { tool: tools[0] } : {}),
1554
+ message: `${tools.length === 1 ? tools[0] : `${tools.length} tools`} answered with an HTTP error from the API behind ${tools.length === 1 ? 'it' : 'them'}, passed on as a plain result, not marked isError. The spec reports tool errors, API failures among them, with isError: true; a client that reads it (retries, its UI, a check like this one) takes the call for a success.`,
1555
+ detail: list.slice(0, 8).map((u) => `step ${u.step}, ${u.tool}: ${clip(u.said, 120)}`).concat(list.length > 8 ? [`… and ${list.length - 8} more`] : []),
1556
+ fix: 'Return the API\'s error with isError: true (in the official SDKs: throw from the handler, or return { isError: true, content }), keeping its message for the model.',
1557
+ },
1558
+ ];
1559
+ }
1560
+ /**
1561
+ * --create-handles: a handle a creating tool returned that its own tools then refuse, or
1562
+ * a creating tool whose result carries no handle to find.
1563
+ */
1564
+ function roundtripFindings(problems) {
1565
+ return problems.map((p) => p.creator
1566
+ ? {
1567
+ rule: 'session/handle-roundtrip',
1568
+ severity: 'warn',
1569
+ confidence: 'unsure',
1570
+ step: p.step,
1571
+ tool: p.tool,
1572
+ message: `${p.tool} was given the ${p.kind} ${p.creator} had just created, and ${p.problem}. A model that starts a ${p.kind} and uses it right away hits the same error.`,
1573
+ fix: `Check that ${p.tool} accepts the ${p.kind} handle ${p.creator} returns, in the same form (the same field, no extra prefix or quoting).`,
1574
+ }
1575
+ : {
1576
+ rule: 'session/handle-roundtrip',
1577
+ severity: 'info',
1578
+ step: p.step,
1579
+ tool: p.tool,
1580
+ message: `--create-handles called ${p.tool}, which ${p.problem}, so the tools that take a ${p.kind} were called with a made-up one instead.`,
1581
+ fix: `Return the new ${p.kind}'s handle where a model finds it: a ${p.kind}_id field in structuredContent (declared in outputSchema), or at least "${p.kind}_id: …" in the text.`,
1582
+ });
1583
+ }
1459
1584
  /**
1460
1585
  * Handles that can't exist answered with a protocol error (2026-07-28, SEP-2567):
1461
1586
  * "a call against an expired or unknown handle should return a tool execution
@@ -1544,6 +1669,21 @@ function inputFindings(inputs, answer) {
1544
1669
  });
1545
1670
  }
1546
1671
  }
1672
+ // ext-tasks: "A server MUST NOT return CreateTaskResult to a client that did not
1673
+ // include the extension capability on its request". session declares no Tasks support.
1674
+ const tasks = inputs.filter((i) => i.input.outcome === 'task');
1675
+ if (tasks.length) {
1676
+ const one = tasks.length === 1;
1677
+ out.push({
1678
+ rule: 'session/undeclared-task',
1679
+ severity: 'error',
1680
+ step: tasks[0].step,
1681
+ ...(one ? { tool: tasks[0].tool } : {}),
1682
+ message: `${one ? `${tasks[0].tool} answered` : `${tasks.length} calls answered`} with a task to poll (resultType "task", the Tasks extension), though the request didn't declare io.modelcontextprotocol/tasks. Servers must not send a task to such a client: it has no tasks/get to poll with, so the agent gets no result at all.`,
1683
+ detail: tasks.map((i) => `${i.tool} (step ${i.step}): ${inputNote(i.input, answer)}`),
1684
+ fix: `Check the request's io.modelcontextprotocol/clientCapabilities for the io.modelcontextprotocol/tasks extension before creating a task. Without it, answer the call in one go, or return -32021 (Missing Required Client Capability) naming the extension.`,
1685
+ });
1686
+ }
1547
1687
  const unmarked = inputs.filter((i) => i.input.missingResultType);
1548
1688
  if (unmarked.length) {
1549
1689
  out.push({
@@ -28,6 +28,8 @@ export interface SnapshotOptions extends RuleSettings {
28
28
  lists?: string[];
29
29
  /** --client: check the menu for this client (src/clients.ts): its description cut, its search, its name format. */
30
30
  client?: string;
31
+ /** --offline-check (stdio): list the menu once more with the network cut off (menu/network-dependent). */
32
+ offlineCheck?: boolean;
31
33
  }
32
34
  export interface SnapshotResult {
33
35
  menu: Menu;
@@ -41,6 +43,19 @@ export interface SnapshotResult {
41
43
  }
42
44
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
43
45
  export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
46
+ /**
47
+ * The target with every credential it carries swapped for one that can't be valid,
48
+ * same scheme: Bearer stays Bearer, Basic gets a made-up user and password, an API-key
49
+ * header a made-up key. A stored login (OAuth) becomes an invalid Bearer token.
50
+ */
51
+ export declare function withInvalidCredential(target: Extract<Target, {
52
+ kind: 'http';
53
+ }>): Extract<Target, {
54
+ kind: 'http';
55
+ }>;
56
+ export declare function offlineEnvironment(): Record<string, string>;
57
+ /** capabilities.extensions as the server sent it: identifier → settings, {} for none. */
58
+ export declare function declaredExtensions(capabilities: Record<string, unknown>): Record<string, unknown>;
44
59
  /**
45
60
  * How this run reached the server, as the end of a command line, for next steps:
46
61
  * the options that got it past the door (names only; values are the user's) and