toolmenu 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -9
- package/dist/args.d.ts +22 -0
- package/dist/args.js +81 -0
- package/dist/auth.d.ts +77 -0
- package/dist/auth.js +304 -0
- package/dist/auto.d.ts +28 -0
- package/dist/auto.js +81 -0
- package/dist/cli.js +105 -9
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/connect.d.ts +1 -0
- package/dist/connect.js +11 -1
- package/dist/difference.d.ts +17 -0
- package/dist/difference.js +70 -0
- package/dist/failures.d.ts +14 -0
- package/dist/failures.js +47 -0
- package/dist/history.js +3 -1
- package/dist/probe.d.ts +27 -0
- package/dist/probe.js +58 -0
- package/dist/report.js +29 -3
- package/dist/rules/determinism.d.ts +23 -1
- package/dist/rules/determinism.js +72 -2
- package/dist/rules/ids.d.ts +2 -0
- package/dist/rules/ids.js +34 -12
- package/dist/rules/index.js +3 -1
- package/dist/rules/naming.js +2 -1
- package/dist/rules/rule.d.ts +6 -0
- package/dist/session.d.ts +16 -0
- package/dist/session.js +81 -44
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +28 -18
- package/package.json +1 -1
package/dist/auto.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { synthesizeArgs } from './args.js';
|
|
2
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf } from './words.js';
|
|
3
|
+
/**
|
|
4
|
+
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
5
|
+
* whose required arguments the schema can fill, cheapest first (no arguments,
|
|
6
|
+
* then lists and searches, then single lookups, then the rest), then the first
|
|
7
|
+
* call again, which should change nothing. Never a tool that isn't marked
|
|
8
|
+
* readOnlyHint, whatever allow_writes says.
|
|
9
|
+
*/
|
|
10
|
+
export function autoScenario(menu, options = {}) {
|
|
11
|
+
const maxCalls = options.maxCalls ?? 20;
|
|
12
|
+
const skipped = [];
|
|
13
|
+
const candidates = [];
|
|
14
|
+
for (const tool of menu.tools) {
|
|
15
|
+
const a = tool.annotations ?? {};
|
|
16
|
+
if (a.readOnlyHint !== true) {
|
|
17
|
+
skipped.push({ tool: tool.name, reason: 'not read-only' });
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
// readOnlyHint is the server's own claim, and a read-only search still spends
|
|
21
|
+
// credits on a paid API. Servers that mark a tool openWorldHint: true are the
|
|
22
|
+
// web search, fetch and scraping ones (Exa, Firecrawl, Context7, Playwright):
|
|
23
|
+
// opt-in. Unmarked tools are called: on GitHub, Atlassian or Notion that's
|
|
24
|
+
// every read, of the account the credentials belong to (FINDINGS F13).
|
|
25
|
+
if (a.openWorldHint === true && !options.openWorld) {
|
|
26
|
+
skipped.push({ tool: tool.name, reason: 'open world' });
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
const filled = synthesizeArgs(tool.inputSchema);
|
|
30
|
+
if (!filled.ok) {
|
|
31
|
+
skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args) });
|
|
35
|
+
}
|
|
36
|
+
candidates.sort((x, y) => x.rank - y.rank);
|
|
37
|
+
const chosen = candidates.slice(0, maxCalls);
|
|
38
|
+
for (const c of candidates.slice(maxCalls))
|
|
39
|
+
skipped.push({ tool: c.tool.name, reason: 'over the call budget' });
|
|
40
|
+
const steps = [{ kind: 'list' }];
|
|
41
|
+
for (const c of chosen)
|
|
42
|
+
steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
|
|
43
|
+
if (chosen.length)
|
|
44
|
+
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
45
|
+
return { scenario: { allowWrites: false, steps }, called: chosen.map((c) => c.tool.name), skipped };
|
|
46
|
+
}
|
|
47
|
+
function rank(tool, args) {
|
|
48
|
+
if (Object.keys(args).length === 0)
|
|
49
|
+
return 0;
|
|
50
|
+
const verb = verbOf(tool.name);
|
|
51
|
+
if (verb && COLLECTION_VERBS.has(verb))
|
|
52
|
+
return 1;
|
|
53
|
+
if (verb && LOOKUP_VERBS.has(verb))
|
|
54
|
+
return 2;
|
|
55
|
+
return 3;
|
|
56
|
+
}
|
|
57
|
+
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
58
|
+
export function scenarioYaml(plan, serverName) {
|
|
59
|
+
const lines = [
|
|
60
|
+
`# Written by \`toolmenu session --auto\`${serverName ? ` for ${serverName}` : ''}: every read-only tool whose`,
|
|
61
|
+
'# required arguments the schema could fill. Edit freely.',
|
|
62
|
+
'allow_writes: false',
|
|
63
|
+
'steps:',
|
|
64
|
+
];
|
|
65
|
+
for (const step of plan.scenario.steps) {
|
|
66
|
+
if (step.kind === 'list')
|
|
67
|
+
lines.push(' - list');
|
|
68
|
+
else if (step.kind === 'call') {
|
|
69
|
+
lines.push(` - call: ${step.tool}`);
|
|
70
|
+
if (Object.keys(step.args).length)
|
|
71
|
+
lines.push(` args: ${JSON.stringify(step.args)}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const needs = plan.skipped.filter((s) => s.reason === 'needs values');
|
|
75
|
+
if (needs.length) {
|
|
76
|
+
lines.push('', ' # Read-only tools that need values the schema doesn\'t give:');
|
|
77
|
+
for (const s of needs.slice(0, 15))
|
|
78
|
+
lines.push(` # - call: ${s.tool}`, ` # args: { ${s.missing.map((m) => `${m}: TODO`).join(', ')} }`);
|
|
79
|
+
}
|
|
80
|
+
return lines.join('\n') + '\n';
|
|
81
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -13,6 +13,9 @@ import { buildMenu } from './menu.js';
|
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { loadRoutes } from './routes.js';
|
|
15
15
|
import { snapshot } from './snapshot.js';
|
|
16
|
+
import { MAIN_SEED, probeMenu, seeded } from './probe.js';
|
|
17
|
+
import { autoScenario, scenarioYaml } from './auto.js';
|
|
18
|
+
import { authDir, listLogins, login, logout } from './auth.js';
|
|
16
19
|
import { SEVERITY_RANK } from './types.js';
|
|
17
20
|
import { VERSION } from './version.js';
|
|
18
21
|
const HELP = `toolmenu ${VERSION}
|
|
@@ -23,14 +26,18 @@ Usage:
|
|
|
23
26
|
toolmenu snapshot [options] <url> Streamable HTTP server
|
|
24
27
|
toolmenu diff [options] <old.json> <new.json> compare two snapshots
|
|
25
28
|
toolmenu history [options] <npm-package> snapshot and diff published versions
|
|
29
|
+
toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
|
|
26
30
|
toolmenu session --scenario <file> [options] -- <command> | <url>
|
|
27
31
|
run a scripted session, watch the menu
|
|
32
|
+
toolmenu session --auto [options] -- <command> | <url>
|
|
33
|
+
the same, with steps built from the menu
|
|
28
34
|
|
|
29
35
|
Commands:
|
|
30
36
|
snapshot establish the menu: list tools twice, write menu.json, run the menu rules
|
|
31
37
|
diff compare releases: breaking changes, token change, semver bump
|
|
32
38
|
session observe the menu changing while a scripted session runs
|
|
33
39
|
history research release history: install, snapshot and diff published npm versions
|
|
40
|
+
auth log in to an OAuth-protected server once; snapshot and session then use it
|
|
34
41
|
|
|
35
42
|
Options:
|
|
36
43
|
--out <path> where to write the menu (default: menu.json)
|
|
@@ -43,7 +50,10 @@ Options:
|
|
|
43
50
|
--header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
|
|
44
51
|
--env <K=V> environment variable for a stdio server, repeatable. The server
|
|
45
52
|
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
53
|
+
--no-auth don't use a stored OAuth login for this server
|
|
46
54
|
--timeout <ms> per-request timeout (default: 30000)
|
|
55
|
+
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
56
|
+
the main one included (default: 2; 1 turns the check off)
|
|
47
57
|
-h, --help show this help
|
|
48
58
|
-v, --version show the version
|
|
49
59
|
|
|
@@ -58,6 +68,15 @@ session options:
|
|
|
58
68
|
--plan print the steps without connecting or running anything
|
|
59
69
|
--init write a starter scenario from the server's menu (to --scenario,
|
|
60
70
|
default scenario.yml; never overwrites)
|
|
71
|
+
--auto build the steps from the menu: every read-only tool whose
|
|
72
|
+
required arguments the schema can fill (const, default,
|
|
73
|
+
examples, enum, type), then the first call again
|
|
74
|
+
--open-world with --auto, also call read-only tools marked openWorldHint:
|
|
75
|
+
true (web search, fetch, scraping): they may cost API credits
|
|
76
|
+
--max-calls <n> with --auto, at most n calls (default: 20)
|
|
77
|
+
--save-scenario <p> with --auto, write the steps it ran as a scenario file
|
|
78
|
+
--union-out <path> write every tool the session saw as a menu file, to commit as
|
|
79
|
+
the baseline for diff (tools behind unlocks included)
|
|
61
80
|
|
|
62
81
|
history options (best effort; installs and runs third-party code, so use a container):
|
|
63
82
|
--versions <n|all> number of published versions to inspect (default: 10)
|
|
@@ -71,6 +90,12 @@ history options (best effort; installs and runs third-party code, so use a conta
|
|
|
71
90
|
--keep-installs keep each version's install directory
|
|
72
91
|
--out <dir> where menus and history.json go (default: toolmenu-history/<package>)
|
|
73
92
|
|
|
93
|
+
auth options:
|
|
94
|
+
--port <n> loopback port for the login redirect (default: 33418)
|
|
95
|
+
--scope <scopes> scopes to ask for (default: the server's)
|
|
96
|
+
--client-id <id> a pre-registered client, for servers without dynamic registration
|
|
97
|
+
--client-secret <s> its secret, if it has one
|
|
98
|
+
|
|
74
99
|
Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
|
|
75
100
|
`;
|
|
76
101
|
class UsageError extends Error {
|
|
@@ -93,6 +118,12 @@ export async function main(argv) {
|
|
|
93
118
|
header: { type: 'string', multiple: true },
|
|
94
119
|
env: { type: 'string', multiple: true },
|
|
95
120
|
timeout: { type: 'string' },
|
|
121
|
+
'no-auth': { type: 'boolean' },
|
|
122
|
+
port: { type: 'string' },
|
|
123
|
+
scope: { type: 'string' },
|
|
124
|
+
'client-id': { type: 'string' },
|
|
125
|
+
'client-secret': { type: 'string' },
|
|
126
|
+
processes: { type: 'string' },
|
|
96
127
|
versions: { type: 'string' },
|
|
97
128
|
release: { type: 'string' },
|
|
98
129
|
'server-version-is-release': { type: 'boolean' },
|
|
@@ -107,6 +138,11 @@ export async function main(argv) {
|
|
|
107
138
|
scenario: { type: 'string' },
|
|
108
139
|
plan: { type: 'boolean' },
|
|
109
140
|
init: { type: 'boolean' },
|
|
141
|
+
'union-out': { type: 'string' },
|
|
142
|
+
auto: { type: 'boolean' },
|
|
143
|
+
'open-world': { type: 'boolean' },
|
|
144
|
+
'max-calls': { type: 'string' },
|
|
145
|
+
'save-scenario': { type: 'string' },
|
|
110
146
|
help: { type: 'boolean', short: 'h' },
|
|
111
147
|
version: { type: 'boolean', short: 'v' },
|
|
112
148
|
},
|
|
@@ -120,7 +156,7 @@ export async function main(argv) {
|
|
|
120
156
|
process.stdout.write(HELP);
|
|
121
157
|
return sub || values.help ? 0 : 2;
|
|
122
158
|
}
|
|
123
|
-
if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
|
|
159
|
+
if (!['snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
124
160
|
throw new UsageError(`Unknown command "${sub}". Try --help.`);
|
|
125
161
|
const format = (values.json ? 'json' : values.format ?? 'text');
|
|
126
162
|
if (!['text', 'json', 'github', 'markdown'].includes(format))
|
|
@@ -131,7 +167,12 @@ export async function main(argv) {
|
|
|
131
167
|
const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
|
|
132
168
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
133
169
|
throw new UsageError(`--timeout must be a positive number of milliseconds`);
|
|
170
|
+
if (sub === 'auth')
|
|
171
|
+
return authCommand(rest, values);
|
|
134
172
|
const config = await loadConfig(values.config);
|
|
173
|
+
const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
|
|
174
|
+
if (!Number.isInteger(processes) || processes < 1)
|
|
175
|
+
throw new UsageError('--processes must be a whole number, 1 or more');
|
|
135
176
|
if (sub === 'diff') {
|
|
136
177
|
if (command)
|
|
137
178
|
throw new UsageError('diff compares two menu files; it doesn\'t start a server. Run snapshot first.');
|
|
@@ -186,8 +227,8 @@ export async function main(argv) {
|
|
|
186
227
|
const path = values.scenario ?? 'scenario.yml';
|
|
187
228
|
if (existsSync(path))
|
|
188
229
|
throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
|
|
189
|
-
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
190
|
-
const conn = await connect(initTarget, { timeoutMs });
|
|
230
|
+
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
231
|
+
const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
|
|
191
232
|
try {
|
|
192
233
|
const list = await listTools(conn, { timeoutMs });
|
|
193
234
|
const menu = buildMenu(list.tools, { name: conn.server.name, version: conn.server.version, protocolVersion: conn.protocolVersion });
|
|
@@ -199,6 +240,33 @@ export async function main(argv) {
|
|
|
199
240
|
}
|
|
200
241
|
return 0;
|
|
201
242
|
}
|
|
243
|
+
if (sub === 'session' && values.auto) {
|
|
244
|
+
if (values.scenario)
|
|
245
|
+
throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
|
|
246
|
+
const maxCalls = values['max-calls'] ? Number(values['max-calls']) : 20;
|
|
247
|
+
if (!Number.isInteger(maxCalls) || maxCalls < 1)
|
|
248
|
+
throw new UsageError('--max-calls must be a whole number, 1 or more');
|
|
249
|
+
const savePath = values['save-scenario'];
|
|
250
|
+
if (savePath && existsSync(savePath))
|
|
251
|
+
throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
|
|
252
|
+
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
253
|
+
const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
|
|
254
|
+
const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
|
|
255
|
+
if (savePath)
|
|
256
|
+
await writeFile(savePath, scenarioYaml(plan, menu.server.name));
|
|
257
|
+
if (values.plan) {
|
|
258
|
+
process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
|
|
259
|
+
return 0;
|
|
260
|
+
}
|
|
261
|
+
const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
|
|
262
|
+
result.auto = { called: plan.called, skipped: plan.skipped };
|
|
263
|
+
if (values['union-out'])
|
|
264
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
265
|
+
const output = formatSession(result, format);
|
|
266
|
+
if (output)
|
|
267
|
+
process.stdout.write(output + '\n');
|
|
268
|
+
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
269
|
+
}
|
|
202
270
|
if (sub === 'session') {
|
|
203
271
|
if (!values.scenario)
|
|
204
272
|
throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
|
|
@@ -213,17 +281,19 @@ export async function main(argv) {
|
|
|
213
281
|
process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
|
|
214
282
|
return 0;
|
|
215
283
|
}
|
|
216
|
-
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
217
|
-
const result = await session(sessionTarget, scenario, { timeoutMs, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
284
|
+
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
285
|
+
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
286
|
+
if (values['union-out'])
|
|
287
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
218
288
|
const output = formatSession(result, format);
|
|
219
289
|
if (output)
|
|
220
290
|
process.stdout.write(output + '\n');
|
|
221
291
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
222
292
|
}
|
|
223
|
-
const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
293
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
224
294
|
const routesPath = values.routes ?? config.routes;
|
|
225
295
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
226
|
-
const { menu, findings } = await snapshot(target, { routes, timeoutMs, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
296
|
+
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
227
297
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
228
298
|
if (outPath)
|
|
229
299
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
|
@@ -232,7 +302,7 @@ export async function main(argv) {
|
|
|
232
302
|
process.stdout.write(output + '\n');
|
|
233
303
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
234
304
|
}
|
|
235
|
-
function parseTarget(positionals, command, headers, env) {
|
|
305
|
+
function parseTarget(positionals, command, headers, env, noAuth = false) {
|
|
236
306
|
if (command) {
|
|
237
307
|
if (command.length === 0)
|
|
238
308
|
throw new UsageError('Nothing after "--": give the command that starts the server.');
|
|
@@ -247,7 +317,33 @@ function parseTarget(positionals, command, headers, env) {
|
|
|
247
317
|
throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
|
|
248
318
|
if (!/^https?:\/\//.test(url))
|
|
249
319
|
throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
|
|
250
|
-
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
|
|
320
|
+
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
|
|
321
|
+
}
|
|
322
|
+
async function authCommand(args, values) {
|
|
323
|
+
const [action, url, ...extra] = args;
|
|
324
|
+
if (action === 'list') {
|
|
325
|
+
const logins = await listLogins();
|
|
326
|
+
process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
|
|
327
|
+
return 0;
|
|
328
|
+
}
|
|
329
|
+
if (action !== 'login' && action !== 'logout')
|
|
330
|
+
throw new UsageError('auth takes login <url>, logout <url> or list.');
|
|
331
|
+
if (!url || extra.length)
|
|
332
|
+
throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
|
|
333
|
+
if (!/^https?:\/\//.test(url))
|
|
334
|
+
throw new UsageError(`"${url}" isn't an http(s) URL.`);
|
|
335
|
+
if (action === 'logout') {
|
|
336
|
+
process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
|
|
337
|
+
return 0;
|
|
338
|
+
}
|
|
339
|
+
const port = values.port ? Number(values.port) : undefined;
|
|
340
|
+
if (port !== undefined && !(Number.isInteger(port) && port > 0 && port < 65536))
|
|
341
|
+
throw new UsageError('--port must be a port number');
|
|
342
|
+
if (values['client-secret'] && !values['client-id'])
|
|
343
|
+
throw new UsageError('--client-secret goes with --client-id');
|
|
344
|
+
const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
|
|
345
|
+
process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
|
|
346
|
+
return 0;
|
|
251
347
|
}
|
|
252
348
|
function parsePairs(items, separator, flag) {
|
|
253
349
|
const pairs = {};
|
package/dist/config.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ export interface Config {
|
|
|
14
14
|
descriptionLimit?: number | string;
|
|
15
15
|
/** Tools (names or globs) your client sends with the full description. */
|
|
16
16
|
fullDescriptions?: string[];
|
|
17
|
+
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
18
|
+
processes?: number;
|
|
17
19
|
}
|
|
18
20
|
/** Optional: no config file is fine. */
|
|
19
21
|
export declare function loadConfig(path?: string): Promise<Config>;
|
package/dist/config.js
CHANGED
|
@@ -23,5 +23,8 @@ export async function loadConfig(path) {
|
|
|
23
23
|
if (limit !== undefined && !(typeof limit === 'number' && limit > 0) && !(typeof limit === 'string' && limit in CLIENT_LIMITS)) {
|
|
24
24
|
throw new Error(`${file}: descriptionLimit must be a positive number or one of ${Object.keys(CLIENT_LIMITS).join(', ')}`);
|
|
25
25
|
}
|
|
26
|
+
if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
|
|
27
|
+
throw new Error(`${file}: processes must be a whole number, 1 or more`);
|
|
28
|
+
}
|
|
26
29
|
return config;
|
|
27
30
|
}
|
package/dist/connect.d.ts
CHANGED
package/dist/connect.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
2
|
import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
|
|
3
|
+
import { hasLogin, StoredOAuthProvider } from './auth.js';
|
|
3
4
|
import { eraOf } from './menu.js';
|
|
4
5
|
import { VERSION } from './version.js';
|
|
5
6
|
/** Every response the server sent, matched to the request method, as it came off the wire. */
|
|
@@ -48,6 +49,7 @@ export async function connect(target, options = {}) {
|
|
|
48
49
|
const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
|
|
49
50
|
let transport;
|
|
50
51
|
let stderrText = '';
|
|
52
|
+
let oauth = false;
|
|
51
53
|
if (target.kind === 'stdio') {
|
|
52
54
|
const stdio = new StdioClientTransport({
|
|
53
55
|
command: target.command,
|
|
@@ -62,8 +64,13 @@ export async function connect(target, options = {}) {
|
|
|
62
64
|
transport = stdio;
|
|
63
65
|
}
|
|
64
66
|
else {
|
|
67
|
+
// A stored OAuth login (toolmenu auth login) is used unless the request brings
|
|
68
|
+
// its own Authorization header, or --no-auth.
|
|
69
|
+
const ownAuth = Object.keys(target.headers ?? {}).some((h) => h.toLowerCase() === 'authorization');
|
|
70
|
+
oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
|
|
65
71
|
transport = new StreamableHTTPClientTransport(new URL(target.url), {
|
|
66
72
|
requestInit: { headers: target.headers ?? {} },
|
|
73
|
+
...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
|
|
67
74
|
});
|
|
68
75
|
}
|
|
69
76
|
try {
|
|
@@ -75,6 +82,9 @@ export async function connect(target, options = {}) {
|
|
|
75
82
|
const stderr = stderrText.trim();
|
|
76
83
|
if (stderr && error instanceof Error)
|
|
77
84
|
error.message += `\nserver stderr:\n${stderr}`;
|
|
85
|
+
if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
|
|
86
|
+
error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
|
|
87
|
+
}
|
|
78
88
|
await client.close().catch(() => { });
|
|
79
89
|
throw error;
|
|
80
90
|
}
|
|
@@ -90,7 +100,7 @@ export async function connect(target, options = {}) {
|
|
|
90
100
|
server: { name: server.name, version: server.version },
|
|
91
101
|
capabilities: (client.getServerCapabilities() ?? {}),
|
|
92
102
|
usedAuth: target.kind === 'http' &&
|
|
93
|
-
Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h)),
|
|
103
|
+
(oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
|
|
94
104
|
stderr: () => stderrText,
|
|
95
105
|
close: () => client.close(),
|
|
96
106
|
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { MenuTool } from './types.js';
|
|
2
|
+
export interface Difference {
|
|
3
|
+
/** JSON path of the first value that differs, from the tool (`inputSchema.properties.fields.default`). */
|
|
4
|
+
path: string;
|
|
5
|
+
before: unknown;
|
|
6
|
+
after: unknown;
|
|
7
|
+
/** Same items in a different order: a comma-separated string or an array. */
|
|
8
|
+
reordered: boolean;
|
|
9
|
+
}
|
|
10
|
+
/** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
|
|
11
|
+
export declare function firstDifference(a: unknown, b: unknown, path?: string): Difference | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* One line saying what differs in a tool between two menus, for a finding's
|
|
14
|
+
* detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
|
|
15
|
+
* different order ("assignee,…" vs "issuetype,…")`.
|
|
16
|
+
*/
|
|
17
|
+
export declare function describeToolDifference(before: MenuTool, after: MenuTool): string;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { canonical } from './compare.js';
|
|
2
|
+
import { toolDefinition } from './menu.js';
|
|
3
|
+
/** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
|
|
4
|
+
export function firstDifference(a, b, path = '') {
|
|
5
|
+
if (canonical(a) === canonical(b))
|
|
6
|
+
return undefined;
|
|
7
|
+
if (isObject(a) && isObject(b)) {
|
|
8
|
+
for (const key of [...new Set([...Object.keys(a), ...Object.keys(b)])].sort()) {
|
|
9
|
+
const found = firstDifference(a[key], b[key], join(path, key));
|
|
10
|
+
if (found)
|
|
11
|
+
return found;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
if (Array.isArray(a) && Array.isArray(b) && a.length === b.length && !samePermutation(a, b)) {
|
|
15
|
+
for (let i = 0; i < a.length; i++) {
|
|
16
|
+
const found = firstDifference(a[i], b[i], `${path}[${i}]`);
|
|
17
|
+
if (found)
|
|
18
|
+
return found;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return { path, before: a, after: b, reordered: samePermutation(a, b) };
|
|
22
|
+
}
|
|
23
|
+
/** Different order, same items: arrays, or strings that are comma-separated lists. */
|
|
24
|
+
function samePermutation(a, b) {
|
|
25
|
+
const items = (v) => {
|
|
26
|
+
if (Array.isArray(v))
|
|
27
|
+
return v.map((x) => canonical(x));
|
|
28
|
+
if (typeof v === 'string' && v.includes(','))
|
|
29
|
+
return v.split(',').map((s) => s.trim());
|
|
30
|
+
return undefined;
|
|
31
|
+
};
|
|
32
|
+
const x = items(a);
|
|
33
|
+
const y = items(b);
|
|
34
|
+
if (!x || !y || x.length !== y.length || x.length < 2)
|
|
35
|
+
return false;
|
|
36
|
+
return canonical([...x].sort()) === canonical([...y].sort()) && canonical(x) !== canonical(y);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* One line saying what differs in a tool between two menus, for a finding's
|
|
40
|
+
* detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
|
|
41
|
+
* different order ("assignee,…" vs "issuetype,…")`.
|
|
42
|
+
*/
|
|
43
|
+
export function describeToolDifference(before, after) {
|
|
44
|
+
const a = toolDefinition(before);
|
|
45
|
+
const b = toolDefinition(after);
|
|
46
|
+
const d = firstDifference(a, b);
|
|
47
|
+
if (!d) {
|
|
48
|
+
return JSON.stringify(a) === JSON.stringify(b)
|
|
49
|
+
? `${after.name}: identical`
|
|
50
|
+
: `${after.name}: same content, keys in a different order (the bytes differ)`;
|
|
51
|
+
}
|
|
52
|
+
const where = d.path || '(the whole tool)';
|
|
53
|
+
if (d.reordered) {
|
|
54
|
+
const n = Array.isArray(d.before) ? d.before.length : String(d.before).split(',').length;
|
|
55
|
+
return `${after.name}: ${where}: same ${n} items, different order (${show(d.before)} vs ${show(d.after)}). Sort them.`;
|
|
56
|
+
}
|
|
57
|
+
return `${after.name}: ${where}: ${show(d.before)} vs ${show(d.after)}`;
|
|
58
|
+
}
|
|
59
|
+
function show(value) {
|
|
60
|
+
if (value === undefined)
|
|
61
|
+
return '(absent)';
|
|
62
|
+
const text = JSON.stringify(value);
|
|
63
|
+
return text.length > 60 ? text.slice(0, 57) + '…' : text;
|
|
64
|
+
}
|
|
65
|
+
function isObject(v) {
|
|
66
|
+
return !!v && typeof v === 'object' && !Array.isArray(v);
|
|
67
|
+
}
|
|
68
|
+
function join(path, key) {
|
|
69
|
+
return /^[A-Za-z_$][\w$]*$/.test(key) ? (path ? `${path}.${key}` : key) : `${path}[${JSON.stringify(key)}]`;
|
|
70
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'invalid-arguments' | 'other';
|
|
2
|
+
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
3
|
+
export declare const SETUP_FAILURES: Set<FailureClass>;
|
|
4
|
+
/**
|
|
5
|
+
* Classify a failed call. `code` is the JSON-RPC error code when the call threw.
|
|
6
|
+
* `hadArguments`: a 404 on a call that named nothing (no arguments) means the
|
|
7
|
+
* account or site behind the credentials isn't there (HubSpot answers a bad token
|
|
8
|
+
* with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
|
|
9
|
+
*/
|
|
10
|
+
export declare function classifyFailure(text: string, options?: {
|
|
11
|
+
code?: number;
|
|
12
|
+
hadArguments?: boolean;
|
|
13
|
+
}): FailureClass;
|
|
14
|
+
export declare const FAILURE_LABELS: Record<FailureClass, string>;
|
package/dist/failures.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Why a tool call failed, from its error text. The patterns come from real
|
|
3
|
+
* servers run with dummy credentials (FINDINGS F12): a failure that stops a call
|
|
4
|
+
* before the tool's own logic runs says nothing about the menu, and shouldn't read
|
|
5
|
+
* as a finding.
|
|
6
|
+
*/
|
|
7
|
+
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
8
|
+
export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found']);
|
|
9
|
+
// Something this machine lacks: "Could not find Google Chrome executable"
|
|
10
|
+
// (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright).
|
|
11
|
+
const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b/i;
|
|
12
|
+
// "401 Bad credentials" (GitHub), "Authorization Expired" (Sentry), "Invalid
|
|
13
|
+
// token" (Firecrawl), "authentication token is not valid" (Apify), "Please provide
|
|
14
|
+
// a valid access token" (Supabase). Not a bare "token": plenty of errors mention one.
|
|
15
|
+
const AUTH = /\b(401|403)\b|unauthori[sz]ed|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|valid (access )?token/i;
|
|
16
|
+
const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
|
|
17
|
+
// "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
|
|
18
|
+
// "**Input Error**" (Sentry), JSON-RPC's invalid params.
|
|
19
|
+
const INVALID_ARGUMENTS = /invalid (params|parameters|arguments|input)|input error|validation (error|failed)|must be provided|is required\b|\beither\b[^.]{0,80}\bor\b[^.]{0,80}\b(must|required)/i;
|
|
20
|
+
const NOT_FOUND = /\b404\b|not found/i;
|
|
21
|
+
/**
|
|
22
|
+
* Classify a failed call. `code` is the JSON-RPC error code when the call threw.
|
|
23
|
+
* `hadArguments`: a 404 on a call that named nothing (no arguments) means the
|
|
24
|
+
* account or site behind the credentials isn't there (HubSpot answers a bad token
|
|
25
|
+
* with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
|
|
26
|
+
*/
|
|
27
|
+
export function classifyFailure(text, options = {}) {
|
|
28
|
+
if (ENVIRONMENT.test(text))
|
|
29
|
+
return 'environment';
|
|
30
|
+
if (AUTH.test(text))
|
|
31
|
+
return 'auth';
|
|
32
|
+
if (NETWORK.test(text))
|
|
33
|
+
return 'network';
|
|
34
|
+
if (options.code === -32602 || INVALID_ARGUMENTS.test(text))
|
|
35
|
+
return 'invalid-arguments';
|
|
36
|
+
if (NOT_FOUND.test(text) && !options.hadArguments)
|
|
37
|
+
return 'not-found';
|
|
38
|
+
return 'other';
|
|
39
|
+
}
|
|
40
|
+
export const FAILURE_LABELS = {
|
|
41
|
+
auth: 'authentication',
|
|
42
|
+
environment: 'something missing on this machine',
|
|
43
|
+
network: 'the network',
|
|
44
|
+
'not-found': 'a 404 on a call that named nothing (usually the account or site behind the credentials)',
|
|
45
|
+
'invalid-arguments': 'the arguments',
|
|
46
|
+
other: 'other errors',
|
|
47
|
+
};
|
package/dist/history.js
CHANGED
|
@@ -80,7 +80,9 @@ async function inspectVersion(pkg, v, options, source) {
|
|
|
80
80
|
return { row: fail(row, 'no-bin', new Error('The package has no bin to start, and no --cmd was given.')) };
|
|
81
81
|
let result;
|
|
82
82
|
try {
|
|
83
|
-
result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir },
|
|
83
|
+
result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir },
|
|
84
|
+
// One process per version: history already starts one for each, and runs long.
|
|
85
|
+
{ timeoutMs: options.timeoutMs ?? 30_000, processes: 1 });
|
|
84
86
|
}
|
|
85
87
|
catch (error) {
|
|
86
88
|
return { row: fail(row, classifyFailure(error), error) };
|
package/dist/probe.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type Target } from './connect.js';
|
|
2
|
+
import type { Probe } from './rules/determinism.js';
|
|
3
|
+
import type { Menu, MenuTool } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* The hash seed of the process whose menu toolmenu saves and compares. Python
|
|
6
|
+
* orders sets of strings by a per-process random seed; pinning it makes a Python
|
|
7
|
+
* server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAIN_SEED = "0";
|
|
10
|
+
/** The probe's seed: different from the main one, so a set-order bug shows every time. */
|
|
11
|
+
export declare const PROBE_SEED = "1";
|
|
12
|
+
/**
|
|
13
|
+
* The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
|
|
14
|
+
* a stdio server. Other runtimes ignore it.
|
|
15
|
+
*/
|
|
16
|
+
export declare function seeded(target: Target, seed: string): Target;
|
|
17
|
+
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
18
|
+
export declare function isContainerWrapper(target: Target): boolean;
|
|
19
|
+
/** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
|
|
20
|
+
export declare function probeMenu(target: Target, timeoutMs: number): Promise<Menu>;
|
|
21
|
+
/**
|
|
22
|
+
* Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
|
|
23
|
+
* compare with the main one. stdio probes run with a different hash seed each. An
|
|
24
|
+
* HTTP probe that differs is retried once: a rolling deploy can serve two
|
|
25
|
+
* versions for a moment.
|
|
26
|
+
*/
|
|
27
|
+
export declare function probeVariance(target: Target, main: MenuTool[], processes: number, timeoutMs: number): Promise<Probe[]>;
|
package/dist/probe.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { basename } from 'node:path';
|
|
2
|
+
import { connect, listTools } from './connect.js';
|
|
3
|
+
import { compareMenus } from './compare.js';
|
|
4
|
+
import { buildMenu } from './menu.js';
|
|
5
|
+
/**
|
|
6
|
+
* The hash seed of the process whose menu toolmenu saves and compares. Python
|
|
7
|
+
* orders sets of strings by a per-process random seed; pinning it makes a Python
|
|
8
|
+
* server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
|
|
9
|
+
*/
|
|
10
|
+
export const MAIN_SEED = '0';
|
|
11
|
+
/** The probe's seed: different from the main one, so a set-order bug shows every time. */
|
|
12
|
+
export const PROBE_SEED = '1';
|
|
13
|
+
/**
|
|
14
|
+
* The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
|
|
15
|
+
* a stdio server. Other runtimes ignore it.
|
|
16
|
+
*/
|
|
17
|
+
export function seeded(target, seed) {
|
|
18
|
+
if (target.kind !== 'stdio' || target.env?.PYTHONHASHSEED !== undefined)
|
|
19
|
+
return target;
|
|
20
|
+
return { ...target, env: { ...target.env, PYTHONHASHSEED: seed } };
|
|
21
|
+
}
|
|
22
|
+
/** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
|
|
23
|
+
export function isContainerWrapper(target) {
|
|
24
|
+
return target.kind === 'stdio' && /^(docker|podman|nerdctl)(\.exe)?$/.test(basename(target.command));
|
|
25
|
+
}
|
|
26
|
+
/** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
|
|
27
|
+
export async function probeMenu(target, timeoutMs) {
|
|
28
|
+
const c = await connect(target, { timeoutMs });
|
|
29
|
+
try {
|
|
30
|
+
const list = await listTools(c, { timeoutMs });
|
|
31
|
+
return buildMenu(list.tools, { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era });
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
await c.close().catch(() => { });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
|
|
39
|
+
* compare with the main one. stdio probes run with a different hash seed each. An
|
|
40
|
+
* HTTP probe that differs is retried once: a rolling deploy can serve two
|
|
41
|
+
* versions for a moment.
|
|
42
|
+
*/
|
|
43
|
+
export async function probeVariance(target, main, processes, timeoutMs) {
|
|
44
|
+
const probes = [];
|
|
45
|
+
for (let i = 0; i < processes - 1; i++) {
|
|
46
|
+
const seed = String(Number(PROBE_SEED) + i);
|
|
47
|
+
try {
|
|
48
|
+
let tools = (await probeMenu(seeded(target, seed), timeoutMs)).tools;
|
|
49
|
+
if (target.kind === 'http' && compareMenus(main, tools).length)
|
|
50
|
+
tools = (await probeMenu(target, timeoutMs)).tools;
|
|
51
|
+
probes.push({ tools });
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
probes.push({ error: error instanceof Error ? error.message : String(error) });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return probes;
|
|
58
|
+
}
|