toolmenu 0.25.2 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +170 -683
  2. package/dist/args.d.ts +14 -2
  3. package/dist/args.js +44 -1
  4. package/dist/auth-check.d.ts +27 -0
  5. package/dist/auth-check.js +177 -0
  6. package/dist/auth.d.ts +7 -0
  7. package/dist/auth.js +29 -1
  8. package/dist/auto.d.ts +8 -0
  9. package/dist/auto.js +9 -2
  10. package/dist/breakdown.js +14 -10
  11. package/dist/cli.js +151 -49
  12. package/dist/clients.d.ts +13 -2
  13. package/dist/clients.js +65 -11
  14. package/dist/config.js +6 -0
  15. package/dist/diff.d.ts +35 -1
  16. package/dist/diff.js +57 -12
  17. package/dist/failures.js +4 -3
  18. package/dist/help.d.ts +1 -1
  19. package/dist/help.js +45 -18
  20. package/dist/history.js +5 -0
  21. package/dist/html.d.ts +29 -0
  22. package/dist/html.js +360 -0
  23. package/dist/index.d.ts +3 -2
  24. package/dist/index.js +3 -2
  25. package/dist/init.d.ts +14 -0
  26. package/dist/init.js +20 -6
  27. package/dist/junit.d.ts +6 -1
  28. package/dist/junit.js +12 -3
  29. package/dist/mcp-env.d.ts +6 -0
  30. package/dist/mcp-env.js +6 -0
  31. package/dist/mcp.d.ts +11 -1
  32. package/dist/mcp.js +123 -22
  33. package/dist/menu.d.ts +6 -0
  34. package/dist/menu.js +12 -1
  35. package/dist/options.d.ts +10 -0
  36. package/dist/options.js +3 -0
  37. package/dist/report.d.ts +14 -0
  38. package/dist/report.js +57 -51
  39. package/dist/review.d.ts +48 -0
  40. package/dist/review.js +212 -0
  41. package/dist/rule-docs.json +49 -4
  42. package/dist/rules/clients.d.ts +11 -0
  43. package/dist/rules/clients.js +65 -2
  44. package/dist/rules/determinism.d.ts +3 -0
  45. package/dist/rules/determinism.js +44 -7
  46. package/dist/rules/ids.js +5 -1
  47. package/dist/rules/index.js +5 -2
  48. package/dist/rules/output.d.ts +21 -0
  49. package/dist/rules/output.js +49 -0
  50. package/dist/rules/search.js +1 -1
  51. package/dist/rules/spec.d.ts +19 -0
  52. package/dist/rules/spec.js +86 -1
  53. package/dist/sanitize.d.ts +42 -0
  54. package/dist/sanitize.js +142 -0
  55. package/dist/scan.d.ts +9 -2
  56. package/dist/scan.js +46 -24
  57. package/dist/session.d.ts +17 -0
  58. package/dist/session.js +119 -13
  59. package/dist/snapshot.js +2 -1
  60. package/package.json +5 -3
@@ -0,0 +1,142 @@
1
+ /*
2
+ * Text a server chose (tool names, descriptions, versions, error output), made safe
3
+ * for where toolmenu prints it. The menu file keeps the exact bytes; only what's
4
+ * shown changes.
5
+ *
6
+ * Shown as escapes, not dropped: a name with an escape sequence in it reads
7
+ * `a\u001b[2Jb`, so the report says what the server really sent and two different
8
+ * names never look the same. spec/tool-name says what's wrong with such a name.
9
+ */
10
+ /**
11
+ * Characters a terminal (or GitHub's log parser) acts on, or a reader can't see:
12
+ * controls (C0, C1, DEL), format characters (the bidi overrides and isolates that
13
+ * reorder what's shown, "Trojan Source"; zero-width spaces and joiners; the soft
14
+ * hyphen; tag characters), line and paragraph separators, everything Unicode says
15
+ * renders as nothing (variation selectors, Hangul fillers), and the blank braille
16
+ * cell. Newline and tab are in it: a server's newline would start a line of its own
17
+ * (a fake finding, a `::error::` workflow command). spec/tool-name uses the same class.
18
+ */
19
+ export const HIDDEN_SOURCE = '[\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\\p{Default_Ignorable_Code_Point}\\u2800]';
20
+ const UNSAFE = new RegExp(HIDDEN_SOURCE, 'gu');
21
+ // The same but newline and tab, for a report whose own lines are already made.
22
+ const UNSAFE_IN_REPORT = new RegExp(`(?![\\n\\t])${HIDDEN_SOURCE}`, 'gu');
23
+ const NAMED = { '\n': '\\n', '\r': '\\r', '\t': '\\t' };
24
+ /**
25
+ * A character as an escape: \u001b; past the first plane as its UTF-16 pair
26
+ * (\udb40\udc41), which JSON and JavaScript read back as the same character, so a
27
+ * JSON report run through terminalSafe() is still the same JSON.
28
+ */
29
+ const visible = (c) => NAMED[c] ??
30
+ [...Array(c.length).keys()].map((i) => `\\u${c.charCodeAt(i).toString(16).padStart(4, '0')}`).join('');
31
+ /** One field of server text, for the terminal: every control character shown as an escape. */
32
+ export function plain(text) {
33
+ return text.replace(UNSAFE, visible);
34
+ }
35
+ /**
36
+ * A whole report, last thing before it's printed: whatever control characters a field
37
+ * still carries are shown as escapes. Its own newlines and tabs stay; the fields that
38
+ * came from the server have already had theirs escaped by plain().
39
+ */
40
+ export function terminalSafe(report) {
41
+ return report.replace(UNSAFE_IN_REPORT, visible);
42
+ }
43
+ /** Links toolmenu's own messages make, kept clickable in markdown. */
44
+ const OWN_LINKS = ['https://github.com/niksa90/toolmenu', 'https://code.claude.com/docs/', 'https://modelcontextprotocol.io/'];
45
+ /**
46
+ * Markdown that GitHub would act on in plain text: links and images ([ ] !), HTML
47
+ * and autolinks (< >), entities (&), escapes, table cells (|) and bold (*). Not _ or
48
+ * ~: they're in most tool names, and at worst they italicize or strike text through.
49
+ */
50
+ const MD_SPECIAL = /[\\`*[\]<>!|&]/;
51
+ /** MD_SPECIAL, and what GitHub links on its own: @mentions, #refs and GH-refs, bare URLs. */
52
+ const MD_INERT = new RegExp(`${MD_SPECIAL.source}|@|#(?=\\d)|\\bGH-(?=\\d)|:\\/\\/|\\bwww\\.`, 'gi');
53
+ const MD_ESCAPE = { '&': '&amp;', '<': '&lt;', '>': '&gt;' };
54
+ /** A zero-width space, as an entity: breaks a mention, reference or autolink without showing. */
55
+ const ZWSP = '&#8203;';
56
+ /**
57
+ * A link toolmenu's own messages make: one of OWN_LINKS, then the end, "/", "#" or "?",
58
+ * with no dot segments (/../ would leave the path the prefix names) and nothing that
59
+ * markdown or a URL parser reads specially.
60
+ */
61
+ function ownLink(url) {
62
+ return OWN_LINKS.some((p) => url.startsWith(p) && (p.endsWith('/') || url.length === p.length || '/#?'.includes(url[p.length]))) && !/(^|[\\/])\.\.?([\\/?#]|$)|%2e|[\\\\*[\]!&@]/i.test(url);
63
+ }
64
+ /** Plain text (no backticks in it) as inert markdown. */
65
+ function mdPlain(text) {
66
+ let out = '';
67
+ // A link toolmenu itself makes stays a link; anything else that would autolink doesn't.
68
+ const parts = text.split(/(https?:\/\/[^\s<>()|`]+)/);
69
+ parts.forEach((part, i) => {
70
+ if (i % 2 === 1 && ownLink(part)) {
71
+ out += part;
72
+ return;
73
+ }
74
+ // One pass, so no replacement sees another's output (the # of an entity).
75
+ out += part.replace(MD_INERT, (m) => {
76
+ if (m === '@' || m === '#')
77
+ return `${m}${ZWSP}`; // no mention or mailto link; no cross-reference on an issue
78
+ if (/^GH-$/i.test(m))
79
+ return `${m.slice(0, 2)}${ZWSP}-`;
80
+ if (m === '://')
81
+ return `:${ZWSP}//`; // https://…: shown, not linked
82
+ if (/^www\./i.test(m))
83
+ return `${m.slice(0, 3)}${ZWSP}.`;
84
+ return MD_ESCAPE[m] ?? `\\${m}`;
85
+ });
86
+ });
87
+ return out;
88
+ }
89
+ /**
90
+ * Text as a markdown code span that it can't close: a fence of backticks longer than
91
+ * any run inside it, padded when it starts or ends with one. Control characters are
92
+ * shown as escapes, so it's always one line.
93
+ */
94
+ export function mdCode(text, options = {}) {
95
+ const shown = plain(text);
96
+ if (!shown)
97
+ return '';
98
+ // A loop, not Math.max(...runs): a name with 100k backtick runs would overflow the stack.
99
+ let longest = 0;
100
+ for (const run of shown.matchAll(/`+/g))
101
+ longest = Math.max(longest, run[0].length);
102
+ const fence = '`'.repeat(longest + 1);
103
+ const pad = /^[`\s]|[`\s]$/.test(shown) ? ' ' : '';
104
+ const body = options.cell ? shown.replace(/\|/g, '\\|') : shown;
105
+ return `${fence}${pad}${body}${pad}${fence}`;
106
+ }
107
+ /**
108
+ * A finding's text, or any line that carries server text, as markdown: toolmenu's own
109
+ * `code` spans stay code, everything else is escaped. A backtick in a server's text
110
+ * can only move where code starts and ends: every piece is rendered inert either way.
111
+ */
112
+ export function mdText(text, options = {}) {
113
+ const pieces = plain(text).split('`');
114
+ // An unpaired backtick: the last piece is text, the backtick shown.
115
+ const unpaired = pieces.length % 2 === 0;
116
+ return pieces
117
+ .map((piece, i) => {
118
+ const last = i === pieces.length - 1;
119
+ if (i % 2 === 1 && !(unpaired && last))
120
+ return mdCode(piece, options);
121
+ const escaped = mdPlain(piece);
122
+ return unpaired && last ? `\\\`${escaped}` : escaped;
123
+ })
124
+ .join('');
125
+ }
126
+ const unguarded = new WeakMap();
127
+ /**
128
+ * The net under every field: from here on, text written to `stream` goes through
129
+ * terminalSafe(). For the CLI's stdout and stderr, whatever the command; JSON stays
130
+ * the same JSON (its escapes read back as the same characters). Twice is once.
131
+ */
132
+ export function guardStream(stream) {
133
+ if (unguarded.has(stream))
134
+ return;
135
+ const write = stream.write.bind(stream);
136
+ unguarded.set(stream, write);
137
+ stream.write = (chunk, ...rest) => write(typeof chunk === 'string' ? terminalSafe(chunk) : chunk, ...rest);
138
+ }
139
+ /** Write past the net: only text already made safe, with toolmenu's own colour codes added. */
140
+ export function writeUnguarded(stream, text) {
141
+ (unguarded.get(stream) ?? stream.write.bind(stream))(text);
142
+ }
package/dist/scan.d.ts CHANGED
@@ -15,11 +15,18 @@ export interface ScanEntry {
15
15
  }
16
16
  export declare class ListError extends Error {
17
17
  }
18
+ /**
19
+ * Which environment variables a config's ${VAR} may be filled from: the names given
20
+ * with --allow-env, or all of them ('*'). None by default: the config file picks
21
+ * where each value goes (a URL, a header, a server's arguments), so a config from
22
+ * someone else could send any of your environment to a server of theirs.
23
+ */
24
+ export type AllowEnv = ReadonlySet<string> | '*';
18
25
  /** Words of a command line: whitespace-separated, with '…' and "…" kept together. */
19
26
  export declare function splitCommand(line: string): string[];
20
27
  /** The servers in a list file, unique names (a repeat gets #2, #3). */
21
- export declare function parseList(text: string, file: string, env?: NodeJS.ProcessEnv): ScanEntry[];
22
- export declare function loadList(file: string): Promise<ScanEntry[]>;
28
+ export declare function parseList(text: string, file: string, env?: NodeJS.ProcessEnv, allow?: AllowEnv): ScanEntry[];
29
+ export declare function loadList(file: string, allow?: AllowEnv): Promise<ScanEntry[]>;
23
30
  export interface ScanRow {
24
31
  name: string;
25
32
  /** Where the server is, printable: a URL without its query, or the command line. Never headers or env values. */
package/dist/scan.js CHANGED
@@ -8,16 +8,27 @@
8
8
  import { readFile } from 'node:fs/promises';
9
9
  import { ConnectError, clean, commandLine } from './explain.js';
10
10
  import { counts, findingLines, githubLines, mdFindings } from './report.js';
11
+ import { mdCode, mdText, plain } from './sanitize.js';
11
12
  import { snapshot } from './snapshot.js';
13
+ import { jsonErrorAt } from './menu.js';
12
14
  import { SEVERITY_RANK } from './types.js';
13
15
  export class ListError extends Error {
14
16
  }
15
- /** `${VAR}`, `${VAR:-default}` (Claude Code) and `${env:VAR}` (VS Code), from the environment. */
16
- function expand(value, env, missing) {
17
+ /** `${VAR}`, `${VAR:-default}` (Claude Code) and `${env:VAR}` (VS Code), from the environment, where allowed. */
18
+ function expand(value, env, missing, allow, blocked) {
17
19
  return value.replace(/\$\{(?:env:)?([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g, (_, name, fallback) => {
20
+ const allowed = allow === '*' || allow.has(name);
21
+ // Not allowed: the environment isn't read at all. With a default, the default, set
22
+ // or not, so a server can't learn which variables exist from which URL it gets.
23
+ if (!allowed && fallback !== undefined)
24
+ return fallback;
18
25
  const v = env[name];
19
- if (v !== undefined && v !== '')
20
- return v;
26
+ if (v !== undefined && v !== '') {
27
+ if (allowed)
28
+ return v;
29
+ blocked.add(name);
30
+ return '';
31
+ }
21
32
  if (fallback !== undefined)
22
33
  return fallback;
23
34
  missing.add(name);
@@ -33,21 +44,23 @@ function record(v) {
33
44
  return Object.fromEntries(Object.entries(v).filter(([, x]) => typeof x === 'string'));
34
45
  }
35
46
  /** One server from a client config: `{ command, args, env, cwd }` or `{ url, headers }`. */
36
- function fromConfig(name, raw, env) {
47
+ function fromConfig(name, raw, env, allow) {
37
48
  if (!raw || typeof raw !== 'object')
38
49
  return { name, problem: 'not an object' };
39
50
  const s = raw;
40
51
  if (s.disabled === true || s.enabled === false)
41
52
  return { name, disabled: true };
42
53
  const missing = new Set();
43
- const x = (v) => expand(v, env, missing);
54
+ const blocked = new Set();
55
+ const x = (v) => expand(v, env, missing, allow, blocked);
44
56
  const map = (r) => (r ? Object.fromEntries(Object.entries(r).map(([k, v]) => [k, x(v)])) : undefined);
45
57
  const type = typeof s.type === 'string' ? s.type : typeof s.transport === 'string' ? s.transport : undefined;
46
58
  let target;
47
59
  if (typeof s.url === 'string' || typeof s.serverUrl === 'string' || typeof s.httpUrl === 'string') {
48
60
  if (type === 'sse')
49
61
  return { name, problem: 'HTTP+SSE transport, which toolmenu doesn\'t speak (Streamable HTTP only)' };
50
- const url = x((s.url ?? s.serverUrl ?? s.httpUrl));
62
+ const raw = [s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string');
63
+ const url = x(raw);
51
64
  const headers = map(record(s.headers));
52
65
  target = { kind: 'http', url, ...(headers && Object.keys(headers).length ? { headers } : {}) };
53
66
  }
@@ -65,13 +78,19 @@ function fromConfig(name, raw, env) {
65
78
  }
66
79
  else
67
80
  return { name, problem: 'neither a command nor a url' };
81
+ // Set but not allowed: not checked, rather than checked with the value left out.
82
+ if (blocked.size) {
83
+ const list = [...blocked].sort();
84
+ return { name, problem: `${list.join(', ')} ${list.length === 1 ? 'is' : 'are'} set, but not passed in: scan fills \${VAR} only from the variables named with --allow-env (--allow-env ${list.join(',')}), so a config can't send the rest of your environment anywhere` };
85
+ }
68
86
  // A missing variable would send an empty credential: say so instead of checking a server set up wrong.
69
87
  if (missing.size)
70
88
  return { name, problem: `${[...missing].join(', ')} ${missing.size === 1 ? 'is' : 'are'} not set (the config refers to ${missing.size === 1 ? 'it' : 'them'})` };
89
+ // The URL as the config writes it: a filled-in ${VAR} may be a secret.
71
90
  if (target.kind === 'http' && !/^https?:\/\//.test(target.url))
72
- return { name, problem: `"${clean(target.url, 60)}" isn't an http(s) URL` };
91
+ return { name, problem: `"${clean([s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string'), 60)}" isn't an http(s) URL` };
73
92
  const shown = target.kind === 'http'
74
- ? { kind: 'http', url: (s.url ?? s.serverUrl ?? s.httpUrl) }
93
+ ? { kind: 'http', url: [s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string') }
75
94
  : { kind: 'stdio', command: s.command, args: s.args ?? [] };
76
95
  return { name, target, shown };
77
96
  }
@@ -161,7 +180,7 @@ function fromLines(text) {
161
180
  return entries;
162
181
  }
163
182
  /** The servers in a list file, unique names (a repeat gets #2, #3). */
164
- export function parseList(text, file, env = process.env) {
183
+ export function parseList(text, file, env = process.env, allow = new Set()) {
165
184
  let entries;
166
185
  if (/\.jsonc?$/i.test(file) || /^\s*\{/.test(text)) {
167
186
  let json;
@@ -170,13 +189,14 @@ export function parseList(text, file, env = process.env) {
170
189
  json = JSON.parse(text.replace(/^\s*\/\/.*$/gm, ''));
171
190
  }
172
191
  catch (error) {
173
- throw new ListError(`${file} isn't valid JSON: ${error.message}`);
192
+ const at = jsonErrorAt(error);
193
+ throw new ListError(`${file} isn't valid JSON${at ? ` (${at})` : ''}`);
174
194
  }
175
195
  const j = json;
176
196
  const servers = j.mcpServers ?? j.servers ?? j.mcp?.servers ?? j['mcp.servers'];
177
197
  if (!servers || typeof servers !== 'object' || Array.isArray(servers))
178
198
  throw new ListError(`${file} has no mcpServers or servers object, the key MCP clients keep their servers under`);
179
- entries = Object.entries(servers).map(([name, raw]) => fromConfig(name, raw, env));
199
+ entries = Object.entries(servers).map(([name, raw]) => fromConfig(name, raw, env, allow));
180
200
  }
181
201
  else
182
202
  entries = fromLines(text);
@@ -189,7 +209,7 @@ export function parseList(text, file, env = process.env) {
189
209
  return n === 1 ? e : { ...e, name: `${e.name}#${n}` };
190
210
  });
191
211
  }
192
- export async function loadList(file) {
212
+ export async function loadList(file, allow = new Set()) {
193
213
  let text;
194
214
  try {
195
215
  text = await readFile(file, 'utf8');
@@ -197,7 +217,7 @@ export async function loadList(file) {
197
217
  catch (error) {
198
218
  throw new ListError(`couldn't read ${file}: ${error.code ?? error.message}`);
199
219
  }
200
- return parseList(text, file);
220
+ return parseList(text, file, process.env, allow);
201
221
  }
202
222
  /** Printable location. A URL's query and credentials can hold keys, so they're left out. */
203
223
  export function where(target) {
@@ -297,33 +317,35 @@ export function formatScan(rows, format, file) {
297
317
  if (format === 'github') {
298
318
  const lines = [];
299
319
  for (const r of rows) {
320
+ // The title ends at the first comma or "::": the name goes in the message.
300
321
  if (r.failed)
301
- lines.push(`::error title=toolmenu scan ${r.name}::${status(r)}`);
322
+ lines.push(`::error title=toolmenu scan::${plain(`${r.name}: ${status(r)}`).replace(/%/g, '%25')}`);
302
323
  lines.push(...githubLines(r.findings.map((f) => ({ ...f, message: `${r.name}: ${f.message}` }))));
303
324
  }
304
325
  return lines.join('\n');
305
326
  }
306
327
  if (format === 'markdown') {
307
- const cell = (t) => t.replace(/\|/g, '\\|').replace(/\n/g, ' ');
308
- const lines = [`### toolmenu scan: ${cell(file)}`, '', '| Server | Tools | Tokens | Findings |', '|---|--:|--:|---|'];
328
+ const cell = (t) => mdText(t, { cell: true });
329
+ const lines = [`### toolmenu scan: ${mdText(file)}`, '', '| Server | Tools | Tokens | Findings |', '|---|--:|--:|---|'];
309
330
  for (const r of rows)
310
- lines.push(`| \`${cell(r.name)}\` | ${r.menu ? r.menu.tools.length : '–'} | ${r.menu ? `~${tokens(r.menu.totalTokens)}` : '–'} | ${cell(status(r))} |`);
331
+ lines.push(`| ${mdCode(r.name, { cell: true })} | ${r.menu ? r.menu.tools.length : '–'} | ${r.menu ? `~${tokens(r.menu.totalTokens)}` : '–'} | ${cell(status(r))} |`);
311
332
  lines.push('', totals(rows));
312
333
  for (const r of rows.filter((x) => x.findings.length))
313
- lines.push('', `#### \`${cell(r.name)}\``, '', ...mdFindings(r.findings));
334
+ lines.push('', `#### ${mdCode(r.name)}`, '', ...mdFindings(r.findings));
314
335
  return lines.join('\n');
315
336
  }
316
- const nameWidth = Math.min(32, Math.max(6, ...rows.map((r) => r.name.length)));
317
- const lines = [`toolmenu scan ${file}`, ''];
337
+ const nameWidth = Math.min(32, Math.max(6, ...rows.map((r) => plain(r.name).length)));
338
+ const lines = [`toolmenu scan ${plain(file)}`, ''];
318
339
  lines.push(` ${'server'.padEnd(nameWidth)} ${'tools'.padStart(5)} ${'tokens'.padStart(6)} findings`);
319
340
  for (const r of rows) {
320
- const name = r.name.length > nameWidth ? `${r.name.slice(0, nameWidth - 1)}…` : r.name;
321
- lines.push(` ${name.padEnd(nameWidth)} ${(r.menu ? String(r.menu.tools.length) : '–').padStart(5)} ${(r.menu ? `~${tokens(r.menu.totalTokens)}` : '–').padStart(6)} ${status(r)}`);
341
+ const shown = plain(r.name);
342
+ const name = shown.length > nameWidth ? `${shown.slice(0, nameWidth - 1)}…` : shown;
343
+ lines.push(` ${name.padEnd(nameWidth)} ${(r.menu ? String(r.menu.tools.length) : '–').padStart(5)} ${(r.menu ? `~${tokens(r.menu.totalTokens)}` : '–').padStart(6)} ${plain(status(r))}`);
322
344
  }
323
345
  lines.push('', totals(rows));
324
346
  for (const r of rows.filter((x) => x.findings.length)) {
325
347
  const s = r.menu.server;
326
- lines.push('', `── ${r.name} ${s.name ?? ''} ${s.version ?? ''} · protocol ${s.protocolVersion ?? 'unknown'}`.trimEnd(), '');
348
+ lines.push('', `── ${plain(r.name)} ${plain(s.name ?? '')} ${plain(s.version ?? '')} · protocol ${plain(s.protocolVersion ?? 'unknown')}`.trimEnd(), '');
327
349
  lines.push(...findingLines(r.findings).filter((l, i, all) => !(i === all.length - 1 && l === '')));
328
350
  }
329
351
  if (rows.some((r) => r.wantsCredentials))
package/dist/session.d.ts CHANGED
@@ -65,6 +65,21 @@ export interface StepRecord {
65
65
  tools: number;
66
66
  tokens: number;
67
67
  }
68
+ /**
69
+ * A tool in the menu after a step: its tokens, and whether this step inserted, appended,
70
+ * moved or edited it. Tools the step removed come last, marked removed.
71
+ */
72
+ export interface StepTool {
73
+ name: string;
74
+ tokens: number;
75
+ change?: 'inserted' | 'appended' | 'moved' | 'edited' | 'removed';
76
+ }
77
+ /**
78
+ * What a step's changes did to each tool of the menu after it, by position (two tools
79
+ * can share a name). An added tool is appended when only added tools follow it. The
80
+ * tools it removed (from `before`, the menu before the step) follow, marked removed.
81
+ */
82
+ export declare function stepMenu(tools: MenuTool[], changes?: ToolChange[], before?: MenuTool[]): StepTool[];
68
83
  export interface SessionResult {
69
84
  scenario: string;
70
85
  server: Menu['server'];
@@ -108,6 +123,8 @@ export interface SessionOptions {
108
123
  unionOut?: boolean;
109
124
  /** session --auto: what the plan called and skipped, so a run that called nothing isn't a clean one. */
110
125
  auto?: AutoSummary;
126
+ /** Called with the menu after each step that listed it (stepMenu), for the HTML report. Not part of the result, so no report carries it. */
127
+ onStepMenu?: (step: number, menu: StepTool[]) => void;
111
128
  /** Waits before retrying a request refused for being too many (default 2, 4, 8, 16, 32 s: a per-minute limit clears within them). */
112
129
  rateLimitWaitsMs?: number[];
113
130
  }
package/dist/session.js CHANGED
@@ -5,7 +5,7 @@ import { listTools } from './connect.js';
5
5
  import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
6
6
  import { buildMenu, toolDefinition } from './menu.js';
7
7
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeSameEra, probeVariance, seeded } from './probe.js';
8
- import { varianceFinding } from './rules/determinism.js';
8
+ import { outsideModel, varianceFinding } from './rules/determinism.js';
9
9
  import { classifyFailure, errorWords, structuredError, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
10
10
  import { SEVERITY_RANK } from './types.js';
11
11
  import { leadingJson } from './catalog.js';
@@ -13,6 +13,7 @@ import { allDifferences, describeToolDifference } from './difference.js';
13
13
  import { autoNextStep, neededValues, writeSign } from './auto.js';
14
14
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
15
15
  import { handleFromResult } from './rules/handles.js';
16
+ import { OUTPUT_SCHEMA_SINCE, outputCheck } from './rules/output.js';
16
17
  const LIST_CHANGED = 'notifications/tools/list_changed';
17
18
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
18
19
  const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
@@ -72,6 +73,36 @@ export function stepLabel(step) {
72
73
  return `call ${step.tool}${Object.keys(step.args).length ? ' ' + JSON.stringify(step.args) : ''}`;
73
74
  return `wait_for ${step.event}`;
74
75
  }
76
+ /**
77
+ * What a step's changes did to each tool of the menu after it, by position (two tools
78
+ * can share a name). An added tool is appended when only added tools follow it. The
79
+ * tools it removed (from `before`, the menu before the step) follow, marked removed.
80
+ */
81
+ export function stepMenu(tools, changes = [], before = []) {
82
+ const added = new Set();
83
+ const moved = new Set();
84
+ const edited = new Set();
85
+ const removed = [];
86
+ for (const c of changes) {
87
+ if (c.kind === 'added')
88
+ added.add(c.position);
89
+ else if (c.kind === 'moved')
90
+ moved.add(c.position);
91
+ else if (c.kind === 'removed')
92
+ removed.push({ name: c.tool, tokens: before[c.position]?.tokens ?? 0, change: 'removed' });
93
+ else if (c.kind !== 'serialization')
94
+ edited.add(c.position);
95
+ }
96
+ // Where the trailing run of added tools starts: those are appended, earlier ones inserted.
97
+ let tail = tools.length;
98
+ while (tail > 0 && added.has(tail - 1))
99
+ tail--;
100
+ const menu = tools.map((t, i) => {
101
+ const change = added.has(i) ? (i >= tail ? 'appended' : 'inserted') : moved.has(i) ? 'moved' : edited.has(i) ? 'edited' : undefined;
102
+ return { name: t.name, tokens: t.tokens, ...(change ? { change } : {}) };
103
+ });
104
+ return [...menu, ...removed];
105
+ }
75
106
  /** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
76
107
  export async function session(target, scenario, options = {}) {
77
108
  const timeoutMs = options.timeoutMs ?? 30_000;
@@ -84,6 +115,8 @@ export async function session(target, scenario, options = {}) {
84
115
  const roundtrips = [];
85
116
  // HTTP errors passed on as plain answers, not isError (session/error-not-flagged).
86
117
  const unflagged = [];
118
+ // Successful results that break the tool's own outputSchema (session/output-schema).
119
+ const outputs = [];
87
120
  // --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
88
121
  const handleErrors = [];
89
122
  // The main process runs with a pinned hash seed, and so does every scope probe:
@@ -112,7 +145,10 @@ export async function session(target, scenario, options = {}) {
112
145
  listening = false;
113
146
  }
114
147
  }
115
- const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
148
+ const menuOf = async (c) => {
149
+ const listed = await listTools(c, { timeoutMs });
150
+ return buildMenu(listed.tools, serverOf(c), listed.listMeta);
151
+ };
116
152
  const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
117
153
  // A refusal that outlasts the waits, or a write's that isn't sent again, ends
118
154
  // the run with one finding, not one per step. `tool`: the write not sent again.
@@ -222,6 +258,10 @@ export async function session(target, scenario, options = {}) {
222
258
  const tried = new Set();
223
259
  const quiet = new Set();
224
260
  const stopped = new Map();
261
+ // Where a wait_for starts counting list_changed: from the start of the last call,
262
+ // or from the end of the last wait_for. The notification a call causes usually
263
+ // arrives while the call step runs, before the wait_for after it begins.
264
+ let waitFrom = conn.wire.notifications.length;
225
265
  for (const [i, step] of scenario.steps.entries()) {
226
266
  const index = i + 1;
227
267
  if (step.kind === 'call' && step.tentative) {
@@ -235,6 +275,8 @@ export async function session(target, scenario, options = {}) {
235
275
  tried.add(step.tool);
236
276
  const label = stepLabel(step);
237
277
  const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
278
+ let stepChanges = [];
279
+ let stepBefore = [];
238
280
  const mark = conn.wire.notifications.length;
239
281
  let word = false;
240
282
  const done = () => {
@@ -243,6 +285,7 @@ export async function session(target, scenario, options = {}) {
243
285
  steps.push(record);
244
286
  };
245
287
  if (step.kind === 'call') {
288
+ waitFrom = mark;
246
289
  const tool = current.tools.find((t) => t.name === step.tool);
247
290
  const write = tool ? writeSign(tool) : undefined;
248
291
  word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
@@ -319,6 +362,12 @@ export async function session(target, scenario, options = {}) {
319
362
  else {
320
363
  const result = call.result;
321
364
  record.outcome = result.isError ? 'tool-error' : 'answered';
365
+ // A tool that declares an outputSchema promises its successful results (2025-06-18 on).
366
+ if (!result.isError && tool?.outputSchema && (conn.protocolVersion ?? '') >= OUTPUT_SCHEMA_SINCE && !outputs.some((o) => o.tool === step.tool)) {
367
+ const checked = outputCheck(tool.outputSchema, result);
368
+ if (checked)
369
+ outputs.push({ step: index, tool: step.tool, ...checked });
370
+ }
322
371
  if (call.input.rounds && !result.isError)
323
372
  record.note = inputNote(call.input, answer);
324
373
  if (step.creates) {
@@ -431,7 +480,11 @@ export async function session(target, scenario, options = {}) {
431
480
  record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
432
481
  }
433
482
  else {
434
- record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
483
+ // A 401 or 403 on the call itself is the credentials, whatever the words: Google's
484
+ // hosted servers list tools to anyone and answer each call with 401 and a
485
+ // WWW-Authenticate for that tool.
486
+ const status = httpStatus(error);
487
+ record.failure = status === 401 || status === 403 ? 'auth' : classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
435
488
  // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
436
489
  const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
437
490
  if (!SETUP_FAILURES.has(record.failure))
@@ -440,8 +493,10 @@ export async function session(target, scenario, options = {}) {
440
493
  }
441
494
  }
442
495
  else if (step.kind === 'wait_for') {
443
- const arrived = await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, step.timeoutMs);
444
- if (!arrived) {
496
+ const arrived = await waitFor(() => conn.wire.notificationsSince(waitFrom, LIST_CHANGED) > 0, step.timeoutMs);
497
+ if (arrived)
498
+ waitFrom = conn.wire.notifications.length;
499
+ else {
445
500
  record.status = 'timed-out';
446
501
  record.note = `no list_changed within ${step.timeoutMs} ms`;
447
502
  }
@@ -503,6 +558,8 @@ export async function session(target, scenario, options = {}) {
503
558
  continue;
504
559
  }
505
560
  const changes = compareMenus(current.tools, next.tools);
561
+ stepChanges = changes;
562
+ stepBefore = current.tools;
506
563
  if (changes.length) {
507
564
  record.changed = true;
508
565
  const paths = variedPaths(current.tools, next.tools, changes);
@@ -567,6 +624,7 @@ export async function session(target, scenario, options = {}) {
567
624
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
568
625
  record.tools = current.tools.length;
569
626
  record.tokens = current.totalTokens;
627
+ options.onStepMenu?.(index, stepMenu(current.tools, stepChanges, stepBefore));
570
628
  if (firstTry && !record.changed && record.status === 'ok' && !record.failure && record.outcome !== 'needs-input')
571
629
  quiet.add(step.tool);
572
630
  done();
@@ -575,6 +633,7 @@ export async function session(target, scenario, options = {}) {
575
633
  raw.push(...inputFindings(inputs, answer));
576
634
  raw.push(...handleErrorFindings(handleErrors));
577
635
  raw.push(...roundtripFindings(roundtrips));
636
+ raw.push(...outputFindings(outputs));
578
637
  // `ignore` is applied by tool name; this finding sums several tools up, so it's applied here.
579
638
  const ignored = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
580
639
  raw.push(...unflaggedFindings(unflagged.filter((u) => !ignored.some((re) => re.test(u.tool)))));
@@ -608,10 +667,12 @@ export async function session(target, scenario, options = {}) {
608
667
  final: { tools: current.tools.length, tokens: current.totalTokens },
609
668
  connectionCheck,
610
669
  steps,
611
- findings: settle(mergeRepeats(raw), options),
670
+ findings: settle(mergeRepeats(options.auto?.everything ? raw.map(madeUpInput) : raw), options),
612
671
  ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
613
672
  // Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
614
- union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
673
+ // With the starting list's cache hints (ttlMs, cacheScope), as snapshot's menu file has
674
+ // them: without, review saw spec/cache-hints as gone from a baseline made this way.
675
+ union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server, baseline.listMeta), from: 'session' },
615
676
  };
616
677
  }
617
678
  finally {
@@ -838,9 +899,12 @@ function failureFindings(failures, auto) {
838
899
  const call = `${c.tool} ${JSON.stringify(c.args)}`;
839
900
  return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
840
901
  });
902
+ // --call-everything: made-up values go in, so a readable tool error is the right
903
+ // answer to most of them (the spec asks for exactly that). Said as info.
904
+ const madeUp = auto?.everything === true && f.rule === 'session/tool-error';
841
905
  return {
842
906
  rule: f.rule,
843
- severity: f.severity,
907
+ severity: madeUp ? 'info' : f.severity,
844
908
  step: f.steps[0],
845
909
  ...(many ? { steps: f.steps } : {}),
846
910
  tool: f.tool,
@@ -848,9 +912,11 @@ function failureFindings(failures, auto) {
848
912
  ...(f.text && shown !== f.text ? { serverText: f.text } : {}),
849
913
  ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
850
914
  message: f.rule === 'session/tool-error'
851
- ? tools.length > 1
852
- ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
853
- : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
915
+ ? madeUp
916
+ ? `${lead}${tools.length > 1 ? `${who} answered` : `${f.tool} answered`} the made-up values with a tool error the model can read: ${said} With --call-everything that's the right answer; it says nothing about a call with real values.`
917
+ : tools.length > 1
918
+ ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
919
+ : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
854
920
  : tools.length > 1
855
921
  ? `${lead}Calling ${who} failed the same way: ${said}`
856
922
  : `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
@@ -1049,14 +1115,19 @@ export function changeFindings(before, after, changes, step, origin) {
1049
1115
  // timestamp in a description, rewritten at every call). A change that touches
1050
1116
  // anything more is a change of its own.
1051
1117
  const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
1118
+ // Only in fields the model doesn't read (a per-tool version hash, the @pipeworx
1119
+ // servers): the cached prompt is untouched, so not an error about it.
1120
+ const outside = a && b ? outsideModel([a], [b], changes.filter((c) => c.tool === tool)) : undefined;
1052
1121
  out.push({
1053
1122
  rule: 'session/edit',
1054
- severity: 'error',
1123
+ severity: outside ? 'warn' : 'error',
1055
1124
  step,
1056
1125
  tool,
1057
1126
  cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
1058
1127
  each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
1059
- message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
1128
+ message: outside
1129
+ ? `${tool}: ${outside.join(', ')} changed mid-session, ${outside.length === 1 ? 'a field' : 'fields'} clients keep to themselves: the model doesn't read ${outside.length === 1 ? 'it' : 'them'}, so the cached prompt isn't affected, but a client that compares tool lists sees the tool change.`
1130
+ : `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
1060
1131
  detail: [...seen, ...cost, ...why],
1061
1132
  fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
1062
1133
  });
@@ -1698,3 +1769,38 @@ function inputFindings(inputs, answer) {
1698
1769
  }
1699
1770
  return out;
1700
1771
  }
1772
+ /** session/output-schema: one finding for the run, every tool whose result broke its own outputSchema. */
1773
+ function outputFindings(list) {
1774
+ if (!list.length)
1775
+ return [];
1776
+ const one = list.length === 1;
1777
+ // Only format checks failed (a URI, a date): many validators skip format, so a warning.
1778
+ const formatOnly = list.every((o) => o.formatOnly);
1779
+ return [
1780
+ {
1781
+ rule: 'session/output-schema',
1782
+ severity: formatOnly ? 'warn' : 'error',
1783
+ step: list[0].step,
1784
+ ...(one ? { tool: list[0].tool } : {}),
1785
+ message: `${one ? `${list[0].tool}'s result` : `${list.length} tools' results`} broke ${one ? 'its' : 'their'} own outputSchema. A tool that declares one MUST return structuredContent that conforms (2025-06-18 and later), and the official SDK client checks: it raises an error where the tool answered.${formatOnly ? ' Only format checks failed (a URI, a date), which many validators skip, so a client may let it through.' : ''}`,
1786
+ detail: list.map((o) => `${o.tool} (step ${o.step}): ${clip(o.problem, 200)}`),
1787
+ fix: 'Return structuredContent that matches the outputSchema on every successful call, or drop the outputSchema from tools whose results vary.',
1788
+ },
1789
+ ];
1790
+ }
1791
+ /**
1792
+ * --call-everything sends made-up values into a sandbox with no network, so a call
1793
+ * that fails, or reaches no network, or none that got through, says how the tool met
1794
+ * that input, not that the server is broken: info. What a result breaks whatever the
1795
+ * input (its own outputSchema, resultType, an error not flagged as one) keeps its level.
1796
+ */
1797
+ function madeUpInput(f) {
1798
+ const note = ' (--call-everything: made-up values, no network)';
1799
+ // Offline, a "rate limit" is most often a network call that couldn't be made.
1800
+ if (f.rule === 'session/step-failed' || f.rule === 'session/untested' || f.rule === 'session/rate-limited')
1801
+ return { ...f, severity: 'info', message: f.message + note };
1802
+ // A server that stops answering after a call is worth knowing, but offline it may be waiting on the network.
1803
+ if (f.rule === 'session/server-unresponsive' && f.severity === 'error')
1804
+ return { ...f, severity: 'warn', message: f.message + note };
1805
+ return f;
1806
+ }
package/dist/snapshot.js CHANGED
@@ -65,8 +65,9 @@ export async function snapshot(given, options = {}) {
65
65
  catch (error) {
66
66
  const stderr = connection.stderr().trim();
67
67
  // A ConnectError already shows the stderr lines that matter.
68
+ // Each line marked, as ConnectError does: a server's line can't pass for toolmenu's.
68
69
  if (stderr && error instanceof Error && !(error instanceof ConnectError))
69
- error.message += `\nserver stderr:\n${stderr}`;
70
+ error.message += `\nserver stderr:\n${stderr.split('\n').map((l) => ` │ ${l}`).join('\n')}`;
70
71
  throw error;
71
72
  }
72
73
  finally {