toolmenu 0.25.2 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +170 -683
- package/dist/args.d.ts +14 -2
- package/dist/args.js +44 -1
- package/dist/auth-check.d.ts +27 -0
- package/dist/auth-check.js +177 -0
- package/dist/auth.d.ts +7 -0
- package/dist/auth.js +29 -1
- package/dist/auto.d.ts +8 -0
- package/dist/auto.js +9 -2
- package/dist/breakdown.js +14 -10
- package/dist/cli.js +151 -49
- package/dist/clients.d.ts +13 -2
- package/dist/clients.js +65 -11
- package/dist/config.js +6 -0
- package/dist/diff.d.ts +35 -1
- package/dist/diff.js +57 -12
- package/dist/failures.js +4 -3
- package/dist/help.d.ts +1 -1
- package/dist/help.js +45 -18
- package/dist/history.js +5 -0
- package/dist/html.d.ts +29 -0
- package/dist/html.js +360 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -2
- package/dist/init.d.ts +14 -0
- package/dist/init.js +20 -6
- package/dist/junit.d.ts +6 -1
- package/dist/junit.js +12 -3
- package/dist/mcp-env.d.ts +6 -0
- package/dist/mcp-env.js +6 -0
- package/dist/mcp.d.ts +11 -1
- package/dist/mcp.js +123 -22
- package/dist/menu.d.ts +6 -0
- package/dist/menu.js +12 -1
- package/dist/options.d.ts +10 -0
- package/dist/options.js +3 -0
- package/dist/report.d.ts +14 -0
- package/dist/report.js +57 -51
- package/dist/review.d.ts +48 -0
- package/dist/review.js +212 -0
- package/dist/rule-docs.json +49 -4
- package/dist/rules/clients.d.ts +11 -0
- package/dist/rules/clients.js +65 -2
- package/dist/rules/determinism.d.ts +3 -0
- package/dist/rules/determinism.js +44 -7
- package/dist/rules/ids.js +5 -1
- package/dist/rules/index.js +5 -2
- package/dist/rules/output.d.ts +21 -0
- package/dist/rules/output.js +49 -0
- package/dist/rules/search.js +1 -1
- package/dist/rules/spec.d.ts +19 -0
- package/dist/rules/spec.js +86 -1
- package/dist/sanitize.d.ts +42 -0
- package/dist/sanitize.js +142 -0
- package/dist/scan.d.ts +9 -2
- package/dist/scan.js +46 -24
- package/dist/session.d.ts +17 -0
- package/dist/session.js +119 -13
- package/dist/snapshot.js +2 -1
- package/package.json +5 -3
package/dist/sanitize.js
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Text a server chose (tool names, descriptions, versions, error output), made safe
|
|
3
|
+
* for where toolmenu prints it. The menu file keeps the exact bytes; only what's
|
|
4
|
+
* shown changes.
|
|
5
|
+
*
|
|
6
|
+
* Shown as escapes, not dropped: a name with an escape sequence in it reads
|
|
7
|
+
* `a\u001b[2Jb`, so the report says what the server really sent and two different
|
|
8
|
+
* names never look the same. spec/tool-name says what's wrong with such a name.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Characters a terminal (or GitHub's log parser) acts on, or a reader can't see:
|
|
12
|
+
* controls (C0, C1, DEL), format characters (the bidi overrides and isolates that
|
|
13
|
+
* reorder what's shown, "Trojan Source"; zero-width spaces and joiners; the soft
|
|
14
|
+
* hyphen; tag characters), line and paragraph separators, everything Unicode says
|
|
15
|
+
* renders as nothing (variation selectors, Hangul fillers), and the blank braille
|
|
16
|
+
* cell. Newline and tab are in it: a server's newline would start a line of its own
|
|
17
|
+
* (a fake finding, a `::error::` workflow command). spec/tool-name uses the same class.
|
|
18
|
+
*/
|
|
19
|
+
export const HIDDEN_SOURCE = '[\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\\p{Default_Ignorable_Code_Point}\\u2800]';
|
|
20
|
+
const UNSAFE = new RegExp(HIDDEN_SOURCE, 'gu');
|
|
21
|
+
// The same but newline and tab, for a report whose own lines are already made.
|
|
22
|
+
const UNSAFE_IN_REPORT = new RegExp(`(?![\\n\\t])${HIDDEN_SOURCE}`, 'gu');
|
|
23
|
+
const NAMED = { '\n': '\\n', '\r': '\\r', '\t': '\\t' };
|
|
24
|
+
/**
|
|
25
|
+
* A character as an escape: \u001b; past the first plane as its UTF-16 pair
|
|
26
|
+
* (\udb40\udc41), which JSON and JavaScript read back as the same character, so a
|
|
27
|
+
* JSON report run through terminalSafe() is still the same JSON.
|
|
28
|
+
*/
|
|
29
|
+
const visible = (c) => NAMED[c] ??
|
|
30
|
+
[...Array(c.length).keys()].map((i) => `\\u${c.charCodeAt(i).toString(16).padStart(4, '0')}`).join('');
|
|
31
|
+
/** One field of server text, for the terminal: every control character shown as an escape. */
|
|
32
|
+
export function plain(text) {
|
|
33
|
+
return text.replace(UNSAFE, visible);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A whole report, last thing before it's printed: whatever control characters a field
|
|
37
|
+
* still carries are shown as escapes. Its own newlines and tabs stay; the fields that
|
|
38
|
+
* came from the server have already had theirs escaped by plain().
|
|
39
|
+
*/
|
|
40
|
+
export function terminalSafe(report) {
|
|
41
|
+
return report.replace(UNSAFE_IN_REPORT, visible);
|
|
42
|
+
}
|
|
43
|
+
/** Links toolmenu's own messages make, kept clickable in markdown. */
|
|
44
|
+
const OWN_LINKS = ['https://github.com/niksa90/toolmenu', 'https://code.claude.com/docs/', 'https://modelcontextprotocol.io/'];
|
|
45
|
+
/**
|
|
46
|
+
* Markdown that GitHub would act on in plain text: links and images ([ ] !), HTML
|
|
47
|
+
* and autolinks (< >), entities (&), escapes, table cells (|) and bold (*). Not _ or
|
|
48
|
+
* ~: they're in most tool names, and at worst they italicize or strike text through.
|
|
49
|
+
*/
|
|
50
|
+
const MD_SPECIAL = /[\\`*[\]<>!|&]/;
|
|
51
|
+
/** MD_SPECIAL, and what GitHub links on its own: @mentions, #refs and GH-refs, bare URLs. */
|
|
52
|
+
const MD_INERT = new RegExp(`${MD_SPECIAL.source}|@|#(?=\\d)|\\bGH-(?=\\d)|:\\/\\/|\\bwww\\.`, 'gi');
|
|
53
|
+
const MD_ESCAPE = { '&': '&', '<': '<', '>': '>' };
|
|
54
|
+
/** A zero-width space, as an entity: breaks a mention, reference or autolink without showing. */
|
|
55
|
+
const ZWSP = '​';
|
|
56
|
+
/**
|
|
57
|
+
* A link toolmenu's own messages make: one of OWN_LINKS, then the end, "/", "#" or "?",
|
|
58
|
+
* with no dot segments (/../ would leave the path the prefix names) and nothing that
|
|
59
|
+
* markdown or a URL parser reads specially.
|
|
60
|
+
*/
|
|
61
|
+
function ownLink(url) {
|
|
62
|
+
return OWN_LINKS.some((p) => url.startsWith(p) && (p.endsWith('/') || url.length === p.length || '/#?'.includes(url[p.length]))) && !/(^|[\\/])\.\.?([\\/?#]|$)|%2e|[\\\\*[\]!&@]/i.test(url);
|
|
63
|
+
}
|
|
64
|
+
/** Plain text (no backticks in it) as inert markdown. */
|
|
65
|
+
function mdPlain(text) {
|
|
66
|
+
let out = '';
|
|
67
|
+
// A link toolmenu itself makes stays a link; anything else that would autolink doesn't.
|
|
68
|
+
const parts = text.split(/(https?:\/\/[^\s<>()|`]+)/);
|
|
69
|
+
parts.forEach((part, i) => {
|
|
70
|
+
if (i % 2 === 1 && ownLink(part)) {
|
|
71
|
+
out += part;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
// One pass, so no replacement sees another's output (the # of an entity).
|
|
75
|
+
out += part.replace(MD_INERT, (m) => {
|
|
76
|
+
if (m === '@' || m === '#')
|
|
77
|
+
return `${m}${ZWSP}`; // no mention or mailto link; no cross-reference on an issue
|
|
78
|
+
if (/^GH-$/i.test(m))
|
|
79
|
+
return `${m.slice(0, 2)}${ZWSP}-`;
|
|
80
|
+
if (m === '://')
|
|
81
|
+
return `:${ZWSP}//`; // https://…: shown, not linked
|
|
82
|
+
if (/^www\./i.test(m))
|
|
83
|
+
return `${m.slice(0, 3)}${ZWSP}.`;
|
|
84
|
+
return MD_ESCAPE[m] ?? `\\${m}`;
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Text as a markdown code span that it can't close: a fence of backticks longer than
|
|
91
|
+
* any run inside it, padded when it starts or ends with one. Control characters are
|
|
92
|
+
* shown as escapes, so it's always one line.
|
|
93
|
+
*/
|
|
94
|
+
export function mdCode(text, options = {}) {
|
|
95
|
+
const shown = plain(text);
|
|
96
|
+
if (!shown)
|
|
97
|
+
return '';
|
|
98
|
+
// A loop, not Math.max(...runs): a name with 100k backtick runs would overflow the stack.
|
|
99
|
+
let longest = 0;
|
|
100
|
+
for (const run of shown.matchAll(/`+/g))
|
|
101
|
+
longest = Math.max(longest, run[0].length);
|
|
102
|
+
const fence = '`'.repeat(longest + 1);
|
|
103
|
+
const pad = /^[`\s]|[`\s]$/.test(shown) ? ' ' : '';
|
|
104
|
+
const body = options.cell ? shown.replace(/\|/g, '\\|') : shown;
|
|
105
|
+
return `${fence}${pad}${body}${pad}${fence}`;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* A finding's text, or any line that carries server text, as markdown: toolmenu's own
|
|
109
|
+
* `code` spans stay code, everything else is escaped. A backtick in a server's text
|
|
110
|
+
* can only move where code starts and ends: every piece is rendered inert either way.
|
|
111
|
+
*/
|
|
112
|
+
export function mdText(text, options = {}) {
|
|
113
|
+
const pieces = plain(text).split('`');
|
|
114
|
+
// An unpaired backtick: the last piece is text, the backtick shown.
|
|
115
|
+
const unpaired = pieces.length % 2 === 0;
|
|
116
|
+
return pieces
|
|
117
|
+
.map((piece, i) => {
|
|
118
|
+
const last = i === pieces.length - 1;
|
|
119
|
+
if (i % 2 === 1 && !(unpaired && last))
|
|
120
|
+
return mdCode(piece, options);
|
|
121
|
+
const escaped = mdPlain(piece);
|
|
122
|
+
return unpaired && last ? `\\\`${escaped}` : escaped;
|
|
123
|
+
})
|
|
124
|
+
.join('');
|
|
125
|
+
}
|
|
126
|
+
const unguarded = new WeakMap();
|
|
127
|
+
/**
|
|
128
|
+
* The net under every field: from here on, text written to `stream` goes through
|
|
129
|
+
* terminalSafe(). For the CLI's stdout and stderr, whatever the command; JSON stays
|
|
130
|
+
* the same JSON (its escapes read back as the same characters). Twice is once.
|
|
131
|
+
*/
|
|
132
|
+
export function guardStream(stream) {
|
|
133
|
+
if (unguarded.has(stream))
|
|
134
|
+
return;
|
|
135
|
+
const write = stream.write.bind(stream);
|
|
136
|
+
unguarded.set(stream, write);
|
|
137
|
+
stream.write = (chunk, ...rest) => write(typeof chunk === 'string' ? terminalSafe(chunk) : chunk, ...rest);
|
|
138
|
+
}
|
|
139
|
+
/** Write past the net: only text already made safe, with toolmenu's own colour codes added. */
|
|
140
|
+
export function writeUnguarded(stream, text) {
|
|
141
|
+
(unguarded.get(stream) ?? stream.write.bind(stream))(text);
|
|
142
|
+
}
|
package/dist/scan.d.ts
CHANGED
|
@@ -15,11 +15,18 @@ export interface ScanEntry {
|
|
|
15
15
|
}
|
|
16
16
|
export declare class ListError extends Error {
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Which environment variables a config's ${VAR} may be filled from: the names given
|
|
20
|
+
* with --allow-env, or all of them ('*'). None by default: the config file picks
|
|
21
|
+
* where each value goes (a URL, a header, a server's arguments), so a config from
|
|
22
|
+
* someone else could send any of your environment to a server of theirs.
|
|
23
|
+
*/
|
|
24
|
+
export type AllowEnv = ReadonlySet<string> | '*';
|
|
18
25
|
/** Words of a command line: whitespace-separated, with '…' and "…" kept together. */
|
|
19
26
|
export declare function splitCommand(line: string): string[];
|
|
20
27
|
/** The servers in a list file, unique names (a repeat gets #2, #3). */
|
|
21
|
-
export declare function parseList(text: string, file: string, env?: NodeJS.ProcessEnv): ScanEntry[];
|
|
22
|
-
export declare function loadList(file: string): Promise<ScanEntry[]>;
|
|
28
|
+
export declare function parseList(text: string, file: string, env?: NodeJS.ProcessEnv, allow?: AllowEnv): ScanEntry[];
|
|
29
|
+
export declare function loadList(file: string, allow?: AllowEnv): Promise<ScanEntry[]>;
|
|
23
30
|
export interface ScanRow {
|
|
24
31
|
name: string;
|
|
25
32
|
/** Where the server is, printable: a URL without its query, or the command line. Never headers or env values. */
|
package/dist/scan.js
CHANGED
|
@@ -8,16 +8,27 @@
|
|
|
8
8
|
import { readFile } from 'node:fs/promises';
|
|
9
9
|
import { ConnectError, clean, commandLine } from './explain.js';
|
|
10
10
|
import { counts, findingLines, githubLines, mdFindings } from './report.js';
|
|
11
|
+
import { mdCode, mdText, plain } from './sanitize.js';
|
|
11
12
|
import { snapshot } from './snapshot.js';
|
|
13
|
+
import { jsonErrorAt } from './menu.js';
|
|
12
14
|
import { SEVERITY_RANK } from './types.js';
|
|
13
15
|
export class ListError extends Error {
|
|
14
16
|
}
|
|
15
|
-
/** `${VAR}`, `${VAR:-default}` (Claude Code) and `${env:VAR}` (VS Code), from the environment. */
|
|
16
|
-
function expand(value, env, missing) {
|
|
17
|
+
/** `${VAR}`, `${VAR:-default}` (Claude Code) and `${env:VAR}` (VS Code), from the environment, where allowed. */
|
|
18
|
+
function expand(value, env, missing, allow, blocked) {
|
|
17
19
|
return value.replace(/\$\{(?:env:)?([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g, (_, name, fallback) => {
|
|
20
|
+
const allowed = allow === '*' || allow.has(name);
|
|
21
|
+
// Not allowed: the environment isn't read at all. With a default, the default, set
|
|
22
|
+
// or not, so a server can't learn which variables exist from which URL it gets.
|
|
23
|
+
if (!allowed && fallback !== undefined)
|
|
24
|
+
return fallback;
|
|
18
25
|
const v = env[name];
|
|
19
|
-
if (v !== undefined && v !== '')
|
|
20
|
-
|
|
26
|
+
if (v !== undefined && v !== '') {
|
|
27
|
+
if (allowed)
|
|
28
|
+
return v;
|
|
29
|
+
blocked.add(name);
|
|
30
|
+
return '';
|
|
31
|
+
}
|
|
21
32
|
if (fallback !== undefined)
|
|
22
33
|
return fallback;
|
|
23
34
|
missing.add(name);
|
|
@@ -33,21 +44,23 @@ function record(v) {
|
|
|
33
44
|
return Object.fromEntries(Object.entries(v).filter(([, x]) => typeof x === 'string'));
|
|
34
45
|
}
|
|
35
46
|
/** One server from a client config: `{ command, args, env, cwd }` or `{ url, headers }`. */
|
|
36
|
-
function fromConfig(name, raw, env) {
|
|
47
|
+
function fromConfig(name, raw, env, allow) {
|
|
37
48
|
if (!raw || typeof raw !== 'object')
|
|
38
49
|
return { name, problem: 'not an object' };
|
|
39
50
|
const s = raw;
|
|
40
51
|
if (s.disabled === true || s.enabled === false)
|
|
41
52
|
return { name, disabled: true };
|
|
42
53
|
const missing = new Set();
|
|
43
|
-
const
|
|
54
|
+
const blocked = new Set();
|
|
55
|
+
const x = (v) => expand(v, env, missing, allow, blocked);
|
|
44
56
|
const map = (r) => (r ? Object.fromEntries(Object.entries(r).map(([k, v]) => [k, x(v)])) : undefined);
|
|
45
57
|
const type = typeof s.type === 'string' ? s.type : typeof s.transport === 'string' ? s.transport : undefined;
|
|
46
58
|
let target;
|
|
47
59
|
if (typeof s.url === 'string' || typeof s.serverUrl === 'string' || typeof s.httpUrl === 'string') {
|
|
48
60
|
if (type === 'sse')
|
|
49
61
|
return { name, problem: 'HTTP+SSE transport, which toolmenu doesn\'t speak (Streamable HTTP only)' };
|
|
50
|
-
const
|
|
62
|
+
const raw = [s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string');
|
|
63
|
+
const url = x(raw);
|
|
51
64
|
const headers = map(record(s.headers));
|
|
52
65
|
target = { kind: 'http', url, ...(headers && Object.keys(headers).length ? { headers } : {}) };
|
|
53
66
|
}
|
|
@@ -65,13 +78,19 @@ function fromConfig(name, raw, env) {
|
|
|
65
78
|
}
|
|
66
79
|
else
|
|
67
80
|
return { name, problem: 'neither a command nor a url' };
|
|
81
|
+
// Set but not allowed: not checked, rather than checked with the value left out.
|
|
82
|
+
if (blocked.size) {
|
|
83
|
+
const list = [...blocked].sort();
|
|
84
|
+
return { name, problem: `${list.join(', ')} ${list.length === 1 ? 'is' : 'are'} set, but not passed in: scan fills \${VAR} only from the variables named with --allow-env (--allow-env ${list.join(',')}), so a config can't send the rest of your environment anywhere` };
|
|
85
|
+
}
|
|
68
86
|
// A missing variable would send an empty credential: say so instead of checking a server set up wrong.
|
|
69
87
|
if (missing.size)
|
|
70
88
|
return { name, problem: `${[...missing].join(', ')} ${missing.size === 1 ? 'is' : 'are'} not set (the config refers to ${missing.size === 1 ? 'it' : 'them'})` };
|
|
89
|
+
// The URL as the config writes it: a filled-in ${VAR} may be a secret.
|
|
71
90
|
if (target.kind === 'http' && !/^https?:\/\//.test(target.url))
|
|
72
|
-
return { name, problem: `"${clean(
|
|
91
|
+
return { name, problem: `"${clean([s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string'), 60)}" isn't an http(s) URL` };
|
|
73
92
|
const shown = target.kind === 'http'
|
|
74
|
-
? { kind: 'http', url:
|
|
93
|
+
? { kind: 'http', url: [s.url, s.serverUrl, s.httpUrl].find((u) => typeof u === 'string') }
|
|
75
94
|
: { kind: 'stdio', command: s.command, args: s.args ?? [] };
|
|
76
95
|
return { name, target, shown };
|
|
77
96
|
}
|
|
@@ -161,7 +180,7 @@ function fromLines(text) {
|
|
|
161
180
|
return entries;
|
|
162
181
|
}
|
|
163
182
|
/** The servers in a list file, unique names (a repeat gets #2, #3). */
|
|
164
|
-
export function parseList(text, file, env = process.env) {
|
|
183
|
+
export function parseList(text, file, env = process.env, allow = new Set()) {
|
|
165
184
|
let entries;
|
|
166
185
|
if (/\.jsonc?$/i.test(file) || /^\s*\{/.test(text)) {
|
|
167
186
|
let json;
|
|
@@ -170,13 +189,14 @@ export function parseList(text, file, env = process.env) {
|
|
|
170
189
|
json = JSON.parse(text.replace(/^\s*\/\/.*$/gm, ''));
|
|
171
190
|
}
|
|
172
191
|
catch (error) {
|
|
173
|
-
|
|
192
|
+
const at = jsonErrorAt(error);
|
|
193
|
+
throw new ListError(`${file} isn't valid JSON${at ? ` (${at})` : ''}`);
|
|
174
194
|
}
|
|
175
195
|
const j = json;
|
|
176
196
|
const servers = j.mcpServers ?? j.servers ?? j.mcp?.servers ?? j['mcp.servers'];
|
|
177
197
|
if (!servers || typeof servers !== 'object' || Array.isArray(servers))
|
|
178
198
|
throw new ListError(`${file} has no mcpServers or servers object, the key MCP clients keep their servers under`);
|
|
179
|
-
entries = Object.entries(servers).map(([name, raw]) => fromConfig(name, raw, env));
|
|
199
|
+
entries = Object.entries(servers).map(([name, raw]) => fromConfig(name, raw, env, allow));
|
|
180
200
|
}
|
|
181
201
|
else
|
|
182
202
|
entries = fromLines(text);
|
|
@@ -189,7 +209,7 @@ export function parseList(text, file, env = process.env) {
|
|
|
189
209
|
return n === 1 ? e : { ...e, name: `${e.name}#${n}` };
|
|
190
210
|
});
|
|
191
211
|
}
|
|
192
|
-
export async function loadList(file) {
|
|
212
|
+
export async function loadList(file, allow = new Set()) {
|
|
193
213
|
let text;
|
|
194
214
|
try {
|
|
195
215
|
text = await readFile(file, 'utf8');
|
|
@@ -197,7 +217,7 @@ export async function loadList(file) {
|
|
|
197
217
|
catch (error) {
|
|
198
218
|
throw new ListError(`couldn't read ${file}: ${error.code ?? error.message}`);
|
|
199
219
|
}
|
|
200
|
-
return parseList(text, file);
|
|
220
|
+
return parseList(text, file, process.env, allow);
|
|
201
221
|
}
|
|
202
222
|
/** Printable location. A URL's query and credentials can hold keys, so they're left out. */
|
|
203
223
|
export function where(target) {
|
|
@@ -297,33 +317,35 @@ export function formatScan(rows, format, file) {
|
|
|
297
317
|
if (format === 'github') {
|
|
298
318
|
const lines = [];
|
|
299
319
|
for (const r of rows) {
|
|
320
|
+
// The title ends at the first comma or "::": the name goes in the message.
|
|
300
321
|
if (r.failed)
|
|
301
|
-
lines.push(`::error title=toolmenu scan
|
|
322
|
+
lines.push(`::error title=toolmenu scan::${plain(`${r.name}: ${status(r)}`).replace(/%/g, '%25')}`);
|
|
302
323
|
lines.push(...githubLines(r.findings.map((f) => ({ ...f, message: `${r.name}: ${f.message}` }))));
|
|
303
324
|
}
|
|
304
325
|
return lines.join('\n');
|
|
305
326
|
}
|
|
306
327
|
if (format === 'markdown') {
|
|
307
|
-
const cell = (t) => t
|
|
308
|
-
const lines = [`### toolmenu scan: ${
|
|
328
|
+
const cell = (t) => mdText(t, { cell: true });
|
|
329
|
+
const lines = [`### toolmenu scan: ${mdText(file)}`, '', '| Server | Tools | Tokens | Findings |', '|---|--:|--:|---|'];
|
|
309
330
|
for (const r of rows)
|
|
310
|
-
lines.push(`|
|
|
331
|
+
lines.push(`| ${mdCode(r.name, { cell: true })} | ${r.menu ? r.menu.tools.length : '–'} | ${r.menu ? `~${tokens(r.menu.totalTokens)}` : '–'} | ${cell(status(r))} |`);
|
|
311
332
|
lines.push('', totals(rows));
|
|
312
333
|
for (const r of rows.filter((x) => x.findings.length))
|
|
313
|
-
lines.push('', `####
|
|
334
|
+
lines.push('', `#### ${mdCode(r.name)}`, '', ...mdFindings(r.findings));
|
|
314
335
|
return lines.join('\n');
|
|
315
336
|
}
|
|
316
|
-
const nameWidth = Math.min(32, Math.max(6, ...rows.map((r) => r.name.length)));
|
|
317
|
-
const lines = [`toolmenu scan ${file}`, ''];
|
|
337
|
+
const nameWidth = Math.min(32, Math.max(6, ...rows.map((r) => plain(r.name).length)));
|
|
338
|
+
const lines = [`toolmenu scan ${plain(file)}`, ''];
|
|
318
339
|
lines.push(` ${'server'.padEnd(nameWidth)} ${'tools'.padStart(5)} ${'tokens'.padStart(6)} findings`);
|
|
319
340
|
for (const r of rows) {
|
|
320
|
-
const
|
|
321
|
-
|
|
341
|
+
const shown = plain(r.name);
|
|
342
|
+
const name = shown.length > nameWidth ? `${shown.slice(0, nameWidth - 1)}…` : shown;
|
|
343
|
+
lines.push(` ${name.padEnd(nameWidth)} ${(r.menu ? String(r.menu.tools.length) : '–').padStart(5)} ${(r.menu ? `~${tokens(r.menu.totalTokens)}` : '–').padStart(6)} ${plain(status(r))}`);
|
|
322
344
|
}
|
|
323
345
|
lines.push('', totals(rows));
|
|
324
346
|
for (const r of rows.filter((x) => x.findings.length)) {
|
|
325
347
|
const s = r.menu.server;
|
|
326
|
-
lines.push('', `── ${r.name} ${s.name ?? ''} ${s.version ?? ''} · protocol ${s.protocolVersion ?? 'unknown'}`.trimEnd(), '');
|
|
348
|
+
lines.push('', `── ${plain(r.name)} ${plain(s.name ?? '')} ${plain(s.version ?? '')} · protocol ${plain(s.protocolVersion ?? 'unknown')}`.trimEnd(), '');
|
|
327
349
|
lines.push(...findingLines(r.findings).filter((l, i, all) => !(i === all.length - 1 && l === '')));
|
|
328
350
|
}
|
|
329
351
|
if (rows.some((r) => r.wantsCredentials))
|
package/dist/session.d.ts
CHANGED
|
@@ -65,6 +65,21 @@ export interface StepRecord {
|
|
|
65
65
|
tools: number;
|
|
66
66
|
tokens: number;
|
|
67
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* A tool in the menu after a step: its tokens, and whether this step inserted, appended,
|
|
70
|
+
* moved or edited it. Tools the step removed come last, marked removed.
|
|
71
|
+
*/
|
|
72
|
+
export interface StepTool {
|
|
73
|
+
name: string;
|
|
74
|
+
tokens: number;
|
|
75
|
+
change?: 'inserted' | 'appended' | 'moved' | 'edited' | 'removed';
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* What a step's changes did to each tool of the menu after it, by position (two tools
|
|
79
|
+
* can share a name). An added tool is appended when only added tools follow it. The
|
|
80
|
+
* tools it removed (from `before`, the menu before the step) follow, marked removed.
|
|
81
|
+
*/
|
|
82
|
+
export declare function stepMenu(tools: MenuTool[], changes?: ToolChange[], before?: MenuTool[]): StepTool[];
|
|
68
83
|
export interface SessionResult {
|
|
69
84
|
scenario: string;
|
|
70
85
|
server: Menu['server'];
|
|
@@ -108,6 +123,8 @@ export interface SessionOptions {
|
|
|
108
123
|
unionOut?: boolean;
|
|
109
124
|
/** session --auto: what the plan called and skipped, so a run that called nothing isn't a clean one. */
|
|
110
125
|
auto?: AutoSummary;
|
|
126
|
+
/** Called with the menu after each step that listed it (stepMenu), for the HTML report. Not part of the result, so no report carries it. */
|
|
127
|
+
onStepMenu?: (step: number, menu: StepTool[]) => void;
|
|
111
128
|
/** Waits before retrying a request refused for being too many (default 2, 4, 8, 16, 32 s: a per-minute limit clears within them). */
|
|
112
129
|
rateLimitWaitsMs?: number[];
|
|
113
130
|
}
|
package/dist/session.js
CHANGED
|
@@ -5,7 +5,7 @@ import { listTools } from './connect.js';
|
|
|
5
5
|
import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
|
|
6
6
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
7
7
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeSameEra, probeVariance, seeded } from './probe.js';
|
|
8
|
-
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
+
import { outsideModel, varianceFinding } from './rules/determinism.js';
|
|
9
9
|
import { classifyFailure, errorWords, structuredError, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
10
10
|
import { SEVERITY_RANK } from './types.js';
|
|
11
11
|
import { leadingJson } from './catalog.js';
|
|
@@ -13,6 +13,7 @@ import { allDifferences, describeToolDifference } from './difference.js';
|
|
|
13
13
|
import { autoNextStep, neededValues, writeSign } from './auto.js';
|
|
14
14
|
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
15
15
|
import { handleFromResult } from './rules/handles.js';
|
|
16
|
+
import { OUTPUT_SCHEMA_SINCE, outputCheck } from './rules/output.js';
|
|
16
17
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
17
18
|
/** A server saying the session is gone: "Session not found or expired" (toolception). */
|
|
18
19
|
const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
|
|
@@ -72,6 +73,36 @@ export function stepLabel(step) {
|
|
|
72
73
|
return `call ${step.tool}${Object.keys(step.args).length ? ' ' + JSON.stringify(step.args) : ''}`;
|
|
73
74
|
return `wait_for ${step.event}`;
|
|
74
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* What a step's changes did to each tool of the menu after it, by position (two tools
|
|
78
|
+
* can share a name). An added tool is appended when only added tools follow it. The
|
|
79
|
+
* tools it removed (from `before`, the menu before the step) follow, marked removed.
|
|
80
|
+
*/
|
|
81
|
+
export function stepMenu(tools, changes = [], before = []) {
|
|
82
|
+
const added = new Set();
|
|
83
|
+
const moved = new Set();
|
|
84
|
+
const edited = new Set();
|
|
85
|
+
const removed = [];
|
|
86
|
+
for (const c of changes) {
|
|
87
|
+
if (c.kind === 'added')
|
|
88
|
+
added.add(c.position);
|
|
89
|
+
else if (c.kind === 'moved')
|
|
90
|
+
moved.add(c.position);
|
|
91
|
+
else if (c.kind === 'removed')
|
|
92
|
+
removed.push({ name: c.tool, tokens: before[c.position]?.tokens ?? 0, change: 'removed' });
|
|
93
|
+
else if (c.kind !== 'serialization')
|
|
94
|
+
edited.add(c.position);
|
|
95
|
+
}
|
|
96
|
+
// Where the trailing run of added tools starts: those are appended, earlier ones inserted.
|
|
97
|
+
let tail = tools.length;
|
|
98
|
+
while (tail > 0 && added.has(tail - 1))
|
|
99
|
+
tail--;
|
|
100
|
+
const menu = tools.map((t, i) => {
|
|
101
|
+
const change = added.has(i) ? (i >= tail ? 'appended' : 'inserted') : moved.has(i) ? 'moved' : edited.has(i) ? 'edited' : undefined;
|
|
102
|
+
return { name: t.name, tokens: t.tokens, ...(change ? { change } : {}) };
|
|
103
|
+
});
|
|
104
|
+
return [...menu, ...removed];
|
|
105
|
+
}
|
|
75
106
|
/** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
|
|
76
107
|
export async function session(target, scenario, options = {}) {
|
|
77
108
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
@@ -84,6 +115,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
84
115
|
const roundtrips = [];
|
|
85
116
|
// HTTP errors passed on as plain answers, not isError (session/error-not-flagged).
|
|
86
117
|
const unflagged = [];
|
|
118
|
+
// Successful results that break the tool's own outputSchema (session/output-schema).
|
|
119
|
+
const outputs = [];
|
|
87
120
|
// --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
|
|
88
121
|
const handleErrors = [];
|
|
89
122
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
@@ -112,7 +145,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
112
145
|
listening = false;
|
|
113
146
|
}
|
|
114
147
|
}
|
|
115
|
-
const menuOf = async (c) =>
|
|
148
|
+
const menuOf = async (c) => {
|
|
149
|
+
const listed = await listTools(c, { timeoutMs });
|
|
150
|
+
return buildMenu(listed.tools, serverOf(c), listed.listMeta);
|
|
151
|
+
};
|
|
116
152
|
const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
|
|
117
153
|
// A refusal that outlasts the waits, or a write's that isn't sent again, ends
|
|
118
154
|
// the run with one finding, not one per step. `tool`: the write not sent again.
|
|
@@ -222,6 +258,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
222
258
|
const tried = new Set();
|
|
223
259
|
const quiet = new Set();
|
|
224
260
|
const stopped = new Map();
|
|
261
|
+
// Where a wait_for starts counting list_changed: from the start of the last call,
|
|
262
|
+
// or from the end of the last wait_for. The notification a call causes usually
|
|
263
|
+
// arrives while the call step runs, before the wait_for after it begins.
|
|
264
|
+
let waitFrom = conn.wire.notifications.length;
|
|
225
265
|
for (const [i, step] of scenario.steps.entries()) {
|
|
226
266
|
const index = i + 1;
|
|
227
267
|
if (step.kind === 'call' && step.tentative) {
|
|
@@ -235,6 +275,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
235
275
|
tried.add(step.tool);
|
|
236
276
|
const label = stepLabel(step);
|
|
237
277
|
const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
|
|
278
|
+
let stepChanges = [];
|
|
279
|
+
let stepBefore = [];
|
|
238
280
|
const mark = conn.wire.notifications.length;
|
|
239
281
|
let word = false;
|
|
240
282
|
const done = () => {
|
|
@@ -243,6 +285,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
243
285
|
steps.push(record);
|
|
244
286
|
};
|
|
245
287
|
if (step.kind === 'call') {
|
|
288
|
+
waitFrom = mark;
|
|
246
289
|
const tool = current.tools.find((t) => t.name === step.tool);
|
|
247
290
|
const write = tool ? writeSign(tool) : undefined;
|
|
248
291
|
word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
|
|
@@ -319,6 +362,12 @@ export async function session(target, scenario, options = {}) {
|
|
|
319
362
|
else {
|
|
320
363
|
const result = call.result;
|
|
321
364
|
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
365
|
+
// A tool that declares an outputSchema promises its successful results (2025-06-18 on).
|
|
366
|
+
if (!result.isError && tool?.outputSchema && (conn.protocolVersion ?? '') >= OUTPUT_SCHEMA_SINCE && !outputs.some((o) => o.tool === step.tool)) {
|
|
367
|
+
const checked = outputCheck(tool.outputSchema, result);
|
|
368
|
+
if (checked)
|
|
369
|
+
outputs.push({ step: index, tool: step.tool, ...checked });
|
|
370
|
+
}
|
|
322
371
|
if (call.input.rounds && !result.isError)
|
|
323
372
|
record.note = inputNote(call.input, answer);
|
|
324
373
|
if (step.creates) {
|
|
@@ -431,7 +480,11 @@ export async function session(target, scenario, options = {}) {
|
|
|
431
480
|
record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
|
|
432
481
|
}
|
|
433
482
|
else {
|
|
434
|
-
|
|
483
|
+
// A 401 or 403 on the call itself is the credentials, whatever the words: Google's
|
|
484
|
+
// hosted servers list tools to anyone and answer each call with 401 and a
|
|
485
|
+
// WWW-Authenticate for that tool.
|
|
486
|
+
const status = httpStatus(error);
|
|
487
|
+
record.failure = status === 401 || status === 403 ? 'auth' : classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
435
488
|
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
436
489
|
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
437
490
|
if (!SETUP_FAILURES.has(record.failure))
|
|
@@ -440,8 +493,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
440
493
|
}
|
|
441
494
|
}
|
|
442
495
|
else if (step.kind === 'wait_for') {
|
|
443
|
-
const arrived = await waitFor(() => conn.wire.notificationsSince(
|
|
444
|
-
if (
|
|
496
|
+
const arrived = await waitFor(() => conn.wire.notificationsSince(waitFrom, LIST_CHANGED) > 0, step.timeoutMs);
|
|
497
|
+
if (arrived)
|
|
498
|
+
waitFrom = conn.wire.notifications.length;
|
|
499
|
+
else {
|
|
445
500
|
record.status = 'timed-out';
|
|
446
501
|
record.note = `no list_changed within ${step.timeoutMs} ms`;
|
|
447
502
|
}
|
|
@@ -503,6 +558,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
503
558
|
continue;
|
|
504
559
|
}
|
|
505
560
|
const changes = compareMenus(current.tools, next.tools);
|
|
561
|
+
stepChanges = changes;
|
|
562
|
+
stepBefore = current.tools;
|
|
506
563
|
if (changes.length) {
|
|
507
564
|
record.changed = true;
|
|
508
565
|
const paths = variedPaths(current.tools, next.tools, changes);
|
|
@@ -567,6 +624,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
567
624
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
568
625
|
record.tools = current.tools.length;
|
|
569
626
|
record.tokens = current.totalTokens;
|
|
627
|
+
options.onStepMenu?.(index, stepMenu(current.tools, stepChanges, stepBefore));
|
|
570
628
|
if (firstTry && !record.changed && record.status === 'ok' && !record.failure && record.outcome !== 'needs-input')
|
|
571
629
|
quiet.add(step.tool);
|
|
572
630
|
done();
|
|
@@ -575,6 +633,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
575
633
|
raw.push(...inputFindings(inputs, answer));
|
|
576
634
|
raw.push(...handleErrorFindings(handleErrors));
|
|
577
635
|
raw.push(...roundtripFindings(roundtrips));
|
|
636
|
+
raw.push(...outputFindings(outputs));
|
|
578
637
|
// `ignore` is applied by tool name; this finding sums several tools up, so it's applied here.
|
|
579
638
|
const ignored = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
580
639
|
raw.push(...unflaggedFindings(unflagged.filter((u) => !ignored.some((re) => re.test(u.tool)))));
|
|
@@ -608,10 +667,12 @@ export async function session(target, scenario, options = {}) {
|
|
|
608
667
|
final: { tools: current.tools.length, tokens: current.totalTokens },
|
|
609
668
|
connectionCheck,
|
|
610
669
|
steps,
|
|
611
|
-
findings: settle(mergeRepeats(raw), options),
|
|
670
|
+
findings: settle(mergeRepeats(options.auto?.everything ? raw.map(madeUpInput) : raw), options),
|
|
612
671
|
...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
|
|
613
672
|
// Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
|
|
614
|
-
|
|
673
|
+
// With the starting list's cache hints (ttlMs, cacheScope), as snapshot's menu file has
|
|
674
|
+
// them: without, review saw spec/cache-hints as gone from a baseline made this way.
|
|
675
|
+
union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server, baseline.listMeta), from: 'session' },
|
|
615
676
|
};
|
|
616
677
|
}
|
|
617
678
|
finally {
|
|
@@ -838,9 +899,12 @@ function failureFindings(failures, auto) {
|
|
|
838
899
|
const call = `${c.tool} ${JSON.stringify(c.args)}`;
|
|
839
900
|
return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
|
|
840
901
|
});
|
|
902
|
+
// --call-everything: made-up values go in, so a readable tool error is the right
|
|
903
|
+
// answer to most of them (the spec asks for exactly that). Said as info.
|
|
904
|
+
const madeUp = auto?.everything === true && f.rule === 'session/tool-error';
|
|
841
905
|
return {
|
|
842
906
|
rule: f.rule,
|
|
843
|
-
severity: f.severity,
|
|
907
|
+
severity: madeUp ? 'info' : f.severity,
|
|
844
908
|
step: f.steps[0],
|
|
845
909
|
...(many ? { steps: f.steps } : {}),
|
|
846
910
|
tool: f.tool,
|
|
@@ -848,9 +912,11 @@ function failureFindings(failures, auto) {
|
|
|
848
912
|
...(f.text && shown !== f.text ? { serverText: f.text } : {}),
|
|
849
913
|
...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
|
|
850
914
|
message: f.rule === 'session/tool-error'
|
|
851
|
-
?
|
|
852
|
-
? `${lead}${who}
|
|
853
|
-
:
|
|
915
|
+
? madeUp
|
|
916
|
+
? `${lead}${tools.length > 1 ? `${who} answered` : `${f.tool} answered`} the made-up values with a tool error the model can read: ${said} With --call-everything that's the right answer; it says nothing about a call with real values.`
|
|
917
|
+
: tools.length > 1
|
|
918
|
+
? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
919
|
+
: `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
854
920
|
: tools.length > 1
|
|
855
921
|
? `${lead}Calling ${who} failed the same way: ${said}`
|
|
856
922
|
: `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
|
|
@@ -1049,14 +1115,19 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
1049
1115
|
// timestamp in a description, rewritten at every call). A change that touches
|
|
1050
1116
|
// anything more is a change of its own.
|
|
1051
1117
|
const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
|
|
1118
|
+
// Only in fields the model doesn't read (a per-tool version hash, the @pipeworx
|
|
1119
|
+
// servers): the cached prompt is untouched, so not an error about it.
|
|
1120
|
+
const outside = a && b ? outsideModel([a], [b], changes.filter((c) => c.tool === tool)) : undefined;
|
|
1052
1121
|
out.push({
|
|
1053
1122
|
rule: 'session/edit',
|
|
1054
|
-
severity: 'error',
|
|
1123
|
+
severity: outside ? 'warn' : 'error',
|
|
1055
1124
|
step,
|
|
1056
1125
|
tool,
|
|
1057
1126
|
cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
|
|
1058
1127
|
each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
|
|
1059
|
-
message:
|
|
1128
|
+
message: outside
|
|
1129
|
+
? `${tool}: ${outside.join(', ')} changed mid-session, ${outside.length === 1 ? 'a field' : 'fields'} clients keep to themselves: the model doesn't read ${outside.length === 1 ? 'it' : 'them'}, so the cached prompt isn't affected, but a client that compares tool lists sees the tool change.`
|
|
1130
|
+
: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
1060
1131
|
detail: [...seen, ...cost, ...why],
|
|
1061
1132
|
fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
|
|
1062
1133
|
});
|
|
@@ -1698,3 +1769,38 @@ function inputFindings(inputs, answer) {
|
|
|
1698
1769
|
}
|
|
1699
1770
|
return out;
|
|
1700
1771
|
}
|
|
1772
|
+
/** session/output-schema: one finding for the run, every tool whose result broke its own outputSchema. */
|
|
1773
|
+
function outputFindings(list) {
|
|
1774
|
+
if (!list.length)
|
|
1775
|
+
return [];
|
|
1776
|
+
const one = list.length === 1;
|
|
1777
|
+
// Only format checks failed (a URI, a date): many validators skip format, so a warning.
|
|
1778
|
+
const formatOnly = list.every((o) => o.formatOnly);
|
|
1779
|
+
return [
|
|
1780
|
+
{
|
|
1781
|
+
rule: 'session/output-schema',
|
|
1782
|
+
severity: formatOnly ? 'warn' : 'error',
|
|
1783
|
+
step: list[0].step,
|
|
1784
|
+
...(one ? { tool: list[0].tool } : {}),
|
|
1785
|
+
message: `${one ? `${list[0].tool}'s result` : `${list.length} tools' results`} broke ${one ? 'its' : 'their'} own outputSchema. A tool that declares one MUST return structuredContent that conforms (2025-06-18 and later), and the official SDK client checks: it raises an error where the tool answered.${formatOnly ? ' Only format checks failed (a URI, a date), which many validators skip, so a client may let it through.' : ''}`,
|
|
1786
|
+
detail: list.map((o) => `${o.tool} (step ${o.step}): ${clip(o.problem, 200)}`),
|
|
1787
|
+
fix: 'Return structuredContent that matches the outputSchema on every successful call, or drop the outputSchema from tools whose results vary.',
|
|
1788
|
+
},
|
|
1789
|
+
];
|
|
1790
|
+
}
|
|
1791
|
+
/**
|
|
1792
|
+
* --call-everything sends made-up values into a sandbox with no network, so a call
|
|
1793
|
+
* that fails, or reaches no network, or none that got through, says how the tool met
|
|
1794
|
+
* that input, not that the server is broken: info. What a result breaks whatever the
|
|
1795
|
+
* input (its own outputSchema, resultType, an error not flagged as one) keeps its level.
|
|
1796
|
+
*/
|
|
1797
|
+
function madeUpInput(f) {
|
|
1798
|
+
const note = ' (--call-everything: made-up values, no network)';
|
|
1799
|
+
// Offline, a "rate limit" is most often a network call that couldn't be made.
|
|
1800
|
+
if (f.rule === 'session/step-failed' || f.rule === 'session/untested' || f.rule === 'session/rate-limited')
|
|
1801
|
+
return { ...f, severity: 'info', message: f.message + note };
|
|
1802
|
+
// A server that stops answering after a call is worth knowing, but offline it may be waiting on the network.
|
|
1803
|
+
if (f.rule === 'session/server-unresponsive' && f.severity === 'error')
|
|
1804
|
+
return { ...f, severity: 'warn', message: f.message + note };
|
|
1805
|
+
return f;
|
|
1806
|
+
}
|
package/dist/snapshot.js
CHANGED
|
@@ -65,8 +65,9 @@ export async function snapshot(given, options = {}) {
|
|
|
65
65
|
catch (error) {
|
|
66
66
|
const stderr = connection.stderr().trim();
|
|
67
67
|
// A ConnectError already shows the stderr lines that matter.
|
|
68
|
+
// Each line marked, as ConnectError does: a server's line can't pass for toolmenu's.
|
|
68
69
|
if (stderr && error instanceof Error && !(error instanceof ConnectError))
|
|
69
|
-
error.message += `\nserver stderr:\n${stderr}`;
|
|
70
|
+
error.message += `\nserver stderr:\n${stderr.split('\n').map((l) => ` │ ${l}`).join('\n')}`;
|
|
70
71
|
throw error;
|
|
71
72
|
}
|
|
72
73
|
finally {
|