toolmenu 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -8
- package/dist/args.d.ts +22 -0
- package/dist/args.js +81 -0
- package/dist/auth.d.ts +77 -0
- package/dist/auth.js +304 -0
- package/dist/auto.d.ts +28 -0
- package/dist/auto.js +81 -0
- package/dist/breakdown.d.ts +30 -0
- package/dist/breakdown.js +63 -0
- package/dist/catalog.d.ts +50 -0
- package/dist/catalog.js +168 -0
- package/dist/cli.js +139 -11
- package/dist/config.d.ts +6 -0
- package/dist/connect.d.ts +1 -0
- package/dist/connect.js +11 -1
- package/dist/diff.js +43 -0
- package/dist/init.d.ts +22 -0
- package/dist/init.js +105 -0
- package/dist/menu.js +2 -0
- package/dist/report.js +34 -2
- package/dist/session.d.ts +10 -0
- package/dist/session.js +7 -1
- package/dist/snapshot.d.ts +3 -0
- package/dist/snapshot.js +4 -0
- package/dist/types.d.ts +2 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,37 +1,46 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { writeFile } from 'node:fs/promises';
|
|
3
|
-
import { join } from 'node:path';
|
|
2
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
4
|
import { parseArgs } from 'node:util';
|
|
5
5
|
import { loadConfig } from './config.js';
|
|
6
6
|
import { diffMenus } from './diff.js';
|
|
7
7
|
import { loadMenu } from './menu.js';
|
|
8
8
|
import { history, historyCsv } from './history.js';
|
|
9
|
-
import { formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
|
|
9
|
+
import { counts, formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
|
|
10
10
|
import { loadScenario, session, starterScenario } from './session.js';
|
|
11
11
|
import { connect, listTools } from './connect.js';
|
|
12
12
|
import { buildMenu } from './menu.js';
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { loadRoutes } from './routes.js';
|
|
15
15
|
import { snapshot } from './snapshot.js';
|
|
16
|
-
import { MAIN_SEED, seeded } from './probe.js';
|
|
16
|
+
import { MAIN_SEED, probeMenu, seeded } from './probe.js';
|
|
17
|
+
import { autoScenario, scenarioYaml } from './auto.js';
|
|
18
|
+
import { authDir, listLogins, login, logout } from './auth.js';
|
|
19
|
+
import { detectProject, secretsNeeded, workflowYaml } from './init.js';
|
|
17
20
|
import { SEVERITY_RANK } from './types.js';
|
|
18
21
|
import { VERSION } from './version.js';
|
|
19
22
|
const HELP = `toolmenu ${VERSION}
|
|
20
23
|
Lint your MCP server's tool menu for changes that confuse agents or break caches.
|
|
21
24
|
|
|
22
25
|
Usage:
|
|
26
|
+
toolmenu init [options] -- <command> | <url> set up CI: a baseline and a workflow
|
|
23
27
|
toolmenu snapshot [options] -- <command> [args...] stdio server
|
|
24
28
|
toolmenu snapshot [options] <url> Streamable HTTP server
|
|
25
29
|
toolmenu diff [options] <old.json> <new.json> compare two snapshots
|
|
26
30
|
toolmenu history [options] <npm-package> snapshot and diff published versions
|
|
31
|
+
toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
|
|
27
32
|
toolmenu session --scenario <file> [options] -- <command> | <url>
|
|
28
33
|
run a scripted session, watch the menu
|
|
34
|
+
toolmenu session --auto [options] -- <command> | <url>
|
|
35
|
+
the same, with steps built from the menu
|
|
29
36
|
|
|
30
37
|
Commands:
|
|
38
|
+
init snapshot the server into menu.json and write .github/workflows/toolmenu.yml
|
|
31
39
|
snapshot establish the menu: list tools twice, write menu.json, run the menu rules
|
|
32
40
|
diff compare releases: breaking changes, token change, semver bump
|
|
33
41
|
session observe the menu changing while a scripted session runs
|
|
34
42
|
history research release history: install, snapshot and diff published npm versions
|
|
43
|
+
auth log in to an OAuth-protected server once; snapshot and session then use it
|
|
35
44
|
|
|
36
45
|
Options:
|
|
37
46
|
--out <path> where to write the menu (default: menu.json)
|
|
@@ -44,7 +53,10 @@ Options:
|
|
|
44
53
|
--header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
|
|
45
54
|
--env <K=V> environment variable for a stdio server, repeatable. The server
|
|
46
55
|
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
56
|
+
--no-auth don't use a stored OAuth login for this server
|
|
47
57
|
--timeout <ms> per-request timeout (default: 30000)
|
|
58
|
+
--catalog also read the operations behind a search tool (search and execute:
|
|
59
|
+
discover, search_*_tools) and keep them in the menu file for diff
|
|
48
60
|
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
49
61
|
the main one included (default: 2; 1 turns the check off)
|
|
50
62
|
-h, --help show this help
|
|
@@ -61,6 +73,15 @@ session options:
|
|
|
61
73
|
--plan print the steps without connecting or running anything
|
|
62
74
|
--init write a starter scenario from the server's menu (to --scenario,
|
|
63
75
|
default scenario.yml; never overwrites)
|
|
76
|
+
--auto build the steps from the menu: every read-only tool whose
|
|
77
|
+
required arguments the schema can fill (const, default,
|
|
78
|
+
examples, enum, type), then the first call again
|
|
79
|
+
--open-world with --auto, also call read-only tools marked openWorldHint:
|
|
80
|
+
true (web search, fetch, scraping): they may cost API credits
|
|
81
|
+
--max-calls <n> with --auto, at most n calls (default: 20)
|
|
82
|
+
--save-scenario <p> with --auto, write the steps it ran as a scenario file
|
|
83
|
+
--union-out <path> write every tool the session saw as a menu file, to commit as
|
|
84
|
+
the baseline for diff (tools behind unlocks included)
|
|
64
85
|
|
|
65
86
|
history options (best effort; installs and runs third-party code, so use a container):
|
|
66
87
|
--versions <n|all> number of published versions to inspect (default: 10)
|
|
@@ -74,6 +95,15 @@ history options (best effort; installs and runs third-party code, so use a conta
|
|
|
74
95
|
--keep-installs keep each version's install directory
|
|
75
96
|
--out <dir> where menus and history.json go (default: toolmenu-history/<package>)
|
|
76
97
|
|
|
98
|
+
init options:
|
|
99
|
+
--with-session the workflow also runs session --auto (calls read-only tools)
|
|
100
|
+
|
|
101
|
+
auth options:
|
|
102
|
+
--port <n> loopback port for the login redirect (default: 33418)
|
|
103
|
+
--scope <scopes> scopes to ask for (default: the server's)
|
|
104
|
+
--client-id <id> a pre-registered client, for servers without dynamic registration
|
|
105
|
+
--client-secret <s> its secret, if it has one
|
|
106
|
+
|
|
77
107
|
Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
|
|
78
108
|
`;
|
|
79
109
|
class UsageError extends Error {
|
|
@@ -96,6 +126,11 @@ export async function main(argv) {
|
|
|
96
126
|
header: { type: 'string', multiple: true },
|
|
97
127
|
env: { type: 'string', multiple: true },
|
|
98
128
|
timeout: { type: 'string' },
|
|
129
|
+
'no-auth': { type: 'boolean' },
|
|
130
|
+
port: { type: 'string' },
|
|
131
|
+
scope: { type: 'string' },
|
|
132
|
+
'client-id': { type: 'string' },
|
|
133
|
+
'client-secret': { type: 'string' },
|
|
99
134
|
processes: { type: 'string' },
|
|
100
135
|
versions: { type: 'string' },
|
|
101
136
|
release: { type: 'string' },
|
|
@@ -111,6 +146,13 @@ export async function main(argv) {
|
|
|
111
146
|
scenario: { type: 'string' },
|
|
112
147
|
plan: { type: 'boolean' },
|
|
113
148
|
init: { type: 'boolean' },
|
|
149
|
+
'union-out': { type: 'string' },
|
|
150
|
+
auto: { type: 'boolean' },
|
|
151
|
+
'with-session': { type: 'boolean' },
|
|
152
|
+
catalog: { type: 'boolean' },
|
|
153
|
+
'open-world': { type: 'boolean' },
|
|
154
|
+
'max-calls': { type: 'string' },
|
|
155
|
+
'save-scenario': { type: 'string' },
|
|
114
156
|
help: { type: 'boolean', short: 'h' },
|
|
115
157
|
version: { type: 'boolean', short: 'v' },
|
|
116
158
|
},
|
|
@@ -124,7 +166,7 @@ export async function main(argv) {
|
|
|
124
166
|
process.stdout.write(HELP);
|
|
125
167
|
return sub || values.help ? 0 : 2;
|
|
126
168
|
}
|
|
127
|
-
if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
|
|
169
|
+
if (!['init', 'snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
128
170
|
throw new UsageError(`Unknown command "${sub}". Try --help.`);
|
|
129
171
|
const format = (values.json ? 'json' : values.format ?? 'text');
|
|
130
172
|
if (!['text', 'json', 'github', 'markdown'].includes(format))
|
|
@@ -135,6 +177,8 @@ export async function main(argv) {
|
|
|
135
177
|
const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
|
|
136
178
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
137
179
|
throw new UsageError(`--timeout must be a positive number of milliseconds`);
|
|
180
|
+
if (sub === 'auth')
|
|
181
|
+
return authCommand(rest, values);
|
|
138
182
|
const config = await loadConfig(values.config);
|
|
139
183
|
const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
|
|
140
184
|
if (!Number.isInteger(processes) || processes < 1)
|
|
@@ -193,7 +237,7 @@ export async function main(argv) {
|
|
|
193
237
|
const path = values.scenario ?? 'scenario.yml';
|
|
194
238
|
if (existsSync(path))
|
|
195
239
|
throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
|
|
196
|
-
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
240
|
+
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
197
241
|
const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
|
|
198
242
|
try {
|
|
199
243
|
const list = await listTools(conn, { timeoutMs });
|
|
@@ -206,6 +250,62 @@ export async function main(argv) {
|
|
|
206
250
|
}
|
|
207
251
|
return 0;
|
|
208
252
|
}
|
|
253
|
+
if (sub === 'init') {
|
|
254
|
+
const workflow = join('.github', 'workflows', 'toolmenu.yml');
|
|
255
|
+
const baseline = values.out ?? 'menu.json';
|
|
256
|
+
for (const f of [workflow, baseline])
|
|
257
|
+
if (existsSync(f))
|
|
258
|
+
throw new UsageError(`${f} already exists; init never overwrites. Remove it, or set things up by hand (README: "In CI").`);
|
|
259
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
260
|
+
const { menu, findings } = await snapshot(target, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
261
|
+
await writeFile(baseline, JSON.stringify(menu, null, 2) + '\n');
|
|
262
|
+
await mkdir(dirname(workflow), { recursive: true });
|
|
263
|
+
const project = detectProject('.');
|
|
264
|
+
await writeFile(workflow, workflowYaml({ target, project, session: values['with-session'], baseline }));
|
|
265
|
+
const c = counts(findings);
|
|
266
|
+
const secrets = secretsNeeded(target);
|
|
267
|
+
const lines = [
|
|
268
|
+
`toolmenu init ${menu.server.name ?? 'server'} · ${menu.tools.length} tools · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
269
|
+
` wrote ${baseline}: the baseline every pull request is compared with`,
|
|
270
|
+
` wrote ${workflow}${project.kind === 'unknown' ? ' (fill in the TODO: how CI builds your server)' : ` (${project.kind} project)`}`,
|
|
271
|
+
` today's menu: ${c.error} errors, ${c.warn} warnings, ${c.info} info (run toolmenu snapshot to see them)`,
|
|
272
|
+
...(secrets.length ? ['', `Add these repository secrets (Settings → Secrets and variables → Actions): ${secrets.join(', ')}`] : []),
|
|
273
|
+
'',
|
|
274
|
+
'Next:',
|
|
275
|
+
` git add ${baseline} ${workflow} && git commit -m "Check the MCP tool menu on every PR"`,
|
|
276
|
+
' Then open a pull request: toolmenu comments on it. Refresh the baseline when a change is intended:',
|
|
277
|
+
` toolmenu snapshot --out ${baseline} ${target.kind === 'stdio' ? '-- ' + [target.command, ...target.args].join(' ') : target.url}`,
|
|
278
|
+
];
|
|
279
|
+
process.stdout.write(lines.join('\n') + '\n');
|
|
280
|
+
return 0;
|
|
281
|
+
}
|
|
282
|
+
if (sub === 'session' && values.auto) {
|
|
283
|
+
if (values.scenario)
|
|
284
|
+
throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
|
|
285
|
+
const maxCalls = values['max-calls'] ? Number(values['max-calls']) : 20;
|
|
286
|
+
if (!Number.isInteger(maxCalls) || maxCalls < 1)
|
|
287
|
+
throw new UsageError('--max-calls must be a whole number, 1 or more');
|
|
288
|
+
const savePath = values['save-scenario'];
|
|
289
|
+
if (savePath && existsSync(savePath))
|
|
290
|
+
throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
|
|
291
|
+
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
292
|
+
const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
|
|
293
|
+
const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
|
|
294
|
+
if (savePath)
|
|
295
|
+
await writeFile(savePath, scenarioYaml(plan, menu.server.name));
|
|
296
|
+
if (values.plan) {
|
|
297
|
+
process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
|
|
298
|
+
return 0;
|
|
299
|
+
}
|
|
300
|
+
const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
|
|
301
|
+
result.auto = { called: plan.called, skipped: plan.skipped };
|
|
302
|
+
if (values['union-out'])
|
|
303
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
304
|
+
const output = formatSession(result, format);
|
|
305
|
+
if (output)
|
|
306
|
+
process.stdout.write(output + '\n');
|
|
307
|
+
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
308
|
+
}
|
|
209
309
|
if (sub === 'session') {
|
|
210
310
|
if (!values.scenario)
|
|
211
311
|
throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
|
|
@@ -220,17 +320,19 @@ export async function main(argv) {
|
|
|
220
320
|
process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
|
|
221
321
|
return 0;
|
|
222
322
|
}
|
|
223
|
-
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
323
|
+
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
224
324
|
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
|
|
325
|
+
if (values['union-out'])
|
|
326
|
+
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
225
327
|
const output = formatSession(result, format);
|
|
226
328
|
if (output)
|
|
227
329
|
process.stdout.write(output + '\n');
|
|
228
330
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
229
331
|
}
|
|
230
|
-
const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
|
|
332
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
231
333
|
const routesPath = values.routes ?? config.routes;
|
|
232
334
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
233
|
-
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
335
|
+
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
234
336
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
235
337
|
if (outPath)
|
|
236
338
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
|
@@ -239,7 +341,7 @@ export async function main(argv) {
|
|
|
239
341
|
process.stdout.write(output + '\n');
|
|
240
342
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
241
343
|
}
|
|
242
|
-
function parseTarget(positionals, command, headers, env) {
|
|
344
|
+
function parseTarget(positionals, command, headers, env, noAuth = false) {
|
|
243
345
|
if (command) {
|
|
244
346
|
if (command.length === 0)
|
|
245
347
|
throw new UsageError('Nothing after "--": give the command that starts the server.');
|
|
@@ -254,7 +356,33 @@ function parseTarget(positionals, command, headers, env) {
|
|
|
254
356
|
throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
|
|
255
357
|
if (!/^https?:\/\//.test(url))
|
|
256
358
|
throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
|
|
257
|
-
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
|
|
359
|
+
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
|
|
360
|
+
}
|
|
361
|
+
async function authCommand(args, values) {
|
|
362
|
+
const [action, url, ...extra] = args;
|
|
363
|
+
if (action === 'list') {
|
|
364
|
+
const logins = await listLogins();
|
|
365
|
+
process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
|
|
366
|
+
return 0;
|
|
367
|
+
}
|
|
368
|
+
if (action !== 'login' && action !== 'logout')
|
|
369
|
+
throw new UsageError('auth takes login <url>, logout <url> or list.');
|
|
370
|
+
if (!url || extra.length)
|
|
371
|
+
throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
|
|
372
|
+
if (!/^https?:\/\//.test(url))
|
|
373
|
+
throw new UsageError(`"${url}" isn't an http(s) URL.`);
|
|
374
|
+
if (action === 'logout') {
|
|
375
|
+
process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
|
|
376
|
+
return 0;
|
|
377
|
+
}
|
|
378
|
+
const port = values.port ? Number(values.port) : undefined;
|
|
379
|
+
if (port !== undefined && !(Number.isInteger(port) && port > 0 && port < 65536))
|
|
380
|
+
throw new UsageError('--port must be a port number');
|
|
381
|
+
if (values['client-secret'] && !values['client-id'])
|
|
382
|
+
throw new UsageError('--client-secret goes with --client-id');
|
|
383
|
+
const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
|
|
384
|
+
process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
|
|
385
|
+
return 0;
|
|
258
386
|
}
|
|
259
387
|
function parsePairs(items, separator, flag) {
|
|
260
388
|
const pairs = {};
|
package/dist/config.d.ts
CHANGED
|
@@ -16,6 +16,12 @@ export interface Config {
|
|
|
16
16
|
fullDescriptions?: string[];
|
|
17
17
|
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
18
18
|
processes?: number;
|
|
19
|
+
/** snapshot --catalog: which search tool and which queries (default: detected, derived from the menu). */
|
|
20
|
+
catalog?: {
|
|
21
|
+
tool?: string;
|
|
22
|
+
queries?: string[];
|
|
23
|
+
pauseMs?: number;
|
|
24
|
+
};
|
|
19
25
|
}
|
|
20
26
|
/** Optional: no config file is fine. */
|
|
21
27
|
export declare function loadConfig(path?: string): Promise<Config>;
|
package/dist/connect.d.ts
CHANGED
package/dist/connect.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
2
|
import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
|
|
3
|
+
import { hasLogin, StoredOAuthProvider } from './auth.js';
|
|
3
4
|
import { eraOf } from './menu.js';
|
|
4
5
|
import { VERSION } from './version.js';
|
|
5
6
|
/** Every response the server sent, matched to the request method, as it came off the wire. */
|
|
@@ -48,6 +49,7 @@ export async function connect(target, options = {}) {
|
|
|
48
49
|
const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
|
|
49
50
|
let transport;
|
|
50
51
|
let stderrText = '';
|
|
52
|
+
let oauth = false;
|
|
51
53
|
if (target.kind === 'stdio') {
|
|
52
54
|
const stdio = new StdioClientTransport({
|
|
53
55
|
command: target.command,
|
|
@@ -62,8 +64,13 @@ export async function connect(target, options = {}) {
|
|
|
62
64
|
transport = stdio;
|
|
63
65
|
}
|
|
64
66
|
else {
|
|
67
|
+
// A stored OAuth login (toolmenu auth login) is used unless the request brings
|
|
68
|
+
// its own Authorization header, or --no-auth.
|
|
69
|
+
const ownAuth = Object.keys(target.headers ?? {}).some((h) => h.toLowerCase() === 'authorization');
|
|
70
|
+
oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
|
|
65
71
|
transport = new StreamableHTTPClientTransport(new URL(target.url), {
|
|
66
72
|
requestInit: { headers: target.headers ?? {} },
|
|
73
|
+
...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
|
|
67
74
|
});
|
|
68
75
|
}
|
|
69
76
|
try {
|
|
@@ -75,6 +82,9 @@ export async function connect(target, options = {}) {
|
|
|
75
82
|
const stderr = stderrText.trim();
|
|
76
83
|
if (stderr && error instanceof Error)
|
|
77
84
|
error.message += `\nserver stderr:\n${stderr}`;
|
|
85
|
+
if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
|
|
86
|
+
error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
|
|
87
|
+
}
|
|
78
88
|
await client.close().catch(() => { });
|
|
79
89
|
throw error;
|
|
80
90
|
}
|
|
@@ -90,7 +100,7 @@ export async function connect(target, options = {}) {
|
|
|
90
100
|
server: { name: server.name, version: server.version },
|
|
91
101
|
capabilities: (client.getServerCapabilities() ?? {}),
|
|
92
102
|
usedAuth: target.kind === 'http' &&
|
|
93
|
-
Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h)),
|
|
103
|
+
(oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
|
|
94
104
|
stderr: () => stderrText,
|
|
95
105
|
close: () => client.close(),
|
|
96
106
|
};
|
package/dist/diff.js
CHANGED
|
@@ -24,6 +24,12 @@ export const DIFF_RULES = {
|
|
|
24
24
|
'diff/version-bump': { severity: 'warn' },
|
|
25
25
|
'diff/version-backwards': { severity: 'warn' },
|
|
26
26
|
'diff/token-budget': { severity: 'error' },
|
|
27
|
+
// Operations behind a search tool (snapshot --catalog). A search returns its top
|
|
28
|
+
// matches, so one not found this time may be ranked out, not removed: a notice,
|
|
29
|
+
// no bump. Changes to operations found in both runs use the rules above.
|
|
30
|
+
'diff/catalog-missing': { severity: 'info' },
|
|
31
|
+
'diff/catalog-added': { severity: 'info' },
|
|
32
|
+
'diff/catalog-queries': { severity: 'info' },
|
|
27
33
|
};
|
|
28
34
|
export function diffMenus(before, after, options = {}) {
|
|
29
35
|
const raw = [];
|
|
@@ -62,6 +68,7 @@ export function diffMenus(before, after, options = {}) {
|
|
|
62
68
|
message: `Tool order changed (${moved.map((m) => m.tool).join(', ')}). Only a notice between releases: prompt caches are short-lived, so a deploy costs roughly one rewrite. It matters within a session.`,
|
|
63
69
|
});
|
|
64
70
|
}
|
|
71
|
+
raw.push(...compareCatalogs(before.catalog, after.catalog, ignored));
|
|
65
72
|
// Ignored tools leave the token counts and the budget too.
|
|
66
73
|
const tokens = tokenChange(oldTools, newTools);
|
|
67
74
|
if (options.tokenBudget !== undefined && tokens.after > options.tokenBudget) {
|
|
@@ -207,6 +214,42 @@ function compareTool(old, t) {
|
|
|
207
214
|
}
|
|
208
215
|
return out.concat(compareRest(old, t));
|
|
209
216
|
}
|
|
217
|
+
/** Operations behind a search tool, compared like tools where both runs found them. */
|
|
218
|
+
function compareCatalogs(before, after, ignored) {
|
|
219
|
+
if (!before && !after)
|
|
220
|
+
return [];
|
|
221
|
+
if (!before || !after) {
|
|
222
|
+
return [{ rule: 'diff/catalog-queries', message: `Only the ${before ? 'older' : 'newer'} snapshot has a catalog (snapshot --catalog), so the operations behind ${(before ?? after).tool} weren't compared.` }];
|
|
223
|
+
}
|
|
224
|
+
const out = [];
|
|
225
|
+
const where = `behind ${after.tool}`;
|
|
226
|
+
if (JSON.stringify(before.queries) !== JSON.stringify(after.queries)) {
|
|
227
|
+
out.push({ rule: 'diff/catalog-queries', message: `The two catalogs were read with different queries, so operations found on one side only say little. Pin them with catalog.queries in the config.` });
|
|
228
|
+
}
|
|
229
|
+
const partial = [before, after].filter((c) => c.failed?.length);
|
|
230
|
+
if (partial.length) {
|
|
231
|
+
out.push({ rule: 'diff/catalog-queries', message: `${partial.length === 2 ? 'Both catalogs are' : `The ${partial[0] === before ? 'older' : 'newer'} catalog is`} partial: some queries failed (rate limits or errors), so operations found on one side only say even less.` });
|
|
232
|
+
}
|
|
233
|
+
const oldOps = new Map(before.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
|
|
234
|
+
const newOps = new Map(after.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
|
|
235
|
+
for (const [name, op] of newOps) {
|
|
236
|
+
const old = oldOps.get(name);
|
|
237
|
+
if (!old) {
|
|
238
|
+
out.push({ rule: 'diff/catalog-added', tool: name, message: `${name} (${where}) is new, or newly found by the same queries.` });
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
for (const r of compareTool(old, op))
|
|
242
|
+
out.push({ ...r, message: `${r.message.replace(/\.$/, '')} (an operation ${where}).` });
|
|
243
|
+
}
|
|
244
|
+
const missing = [...oldOps.keys()].filter((n) => !newOps.has(n));
|
|
245
|
+
if (missing.length) {
|
|
246
|
+
out.push({
|
|
247
|
+
rule: 'diff/catalog-missing',
|
|
248
|
+
message: `${missing.length} operation${missing.length === 1 ? '' : 's'} ${where} weren't returned by the same queries this time: removed, renamed, or ranked lower. Check before relying on them: ${missing.slice(0, 8).join(', ')}${missing.length > 8 ? ', …' : ''}.`,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return out;
|
|
252
|
+
}
|
|
210
253
|
const TOOL_FIELDS = new Set(['name', 'description', 'inputSchema', 'outputSchema', 'annotations', 'tokens']);
|
|
211
254
|
/** Output schema, annotations and any other fields (title, icons, _meta…). */
|
|
212
255
|
function compareRest(old, t) {
|
package/dist/init.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Target } from './connect.js';
|
|
2
|
+
export type ProjectKind = 'node' | 'python' | 'go' | 'rust' | 'unknown';
|
|
3
|
+
export interface Project {
|
|
4
|
+
kind: ProjectKind;
|
|
5
|
+
/** The steps that install and build the server in CI, as YAML list items. */
|
|
6
|
+
setup: string[];
|
|
7
|
+
}
|
|
8
|
+
/** What the repository in `dir` looks like, and how CI gets it ready to run. */
|
|
9
|
+
export declare function detectProject(dir: string): Project;
|
|
10
|
+
/** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
|
|
11
|
+
export declare function looksSecret(key: string): boolean;
|
|
12
|
+
export interface WorkflowOptions {
|
|
13
|
+
target: Target;
|
|
14
|
+
project: Project;
|
|
15
|
+
/** Also run `session --auto` (calls read-only tools with the job's credentials). */
|
|
16
|
+
session?: boolean;
|
|
17
|
+
baseline?: string;
|
|
18
|
+
}
|
|
19
|
+
/** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
|
|
20
|
+
export declare function workflowYaml(opts: WorkflowOptions): string;
|
|
21
|
+
/** The secret names the workflow refers to, to tell the user what to add. */
|
|
22
|
+
export declare function secretsNeeded(target: Target): string[];
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { VERSION } from './version.js';
|
|
4
|
+
/** What the repository in `dir` looks like, and how CI gets it ready to run. */
|
|
5
|
+
export function detectProject(dir) {
|
|
6
|
+
const has = (f) => existsSync(join(dir, f));
|
|
7
|
+
if (has('package.json')) {
|
|
8
|
+
let scripts = {};
|
|
9
|
+
try {
|
|
10
|
+
scripts = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')).scripts ?? {};
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
// an unreadable package.json still means node
|
|
14
|
+
}
|
|
15
|
+
const pm = has('pnpm-lock.yaml') ? 'pnpm' : has('yarn.lock') ? 'yarn' : 'npm';
|
|
16
|
+
const install = pm === 'pnpm' ? 'pnpm install --frozen-lockfile' : pm === 'yarn' ? 'yarn install --frozen-lockfile' : has('package-lock.json') ? 'npm ci' : 'npm install';
|
|
17
|
+
const run = pm === 'npm' ? 'npm run' : pm;
|
|
18
|
+
return {
|
|
19
|
+
kind: 'node',
|
|
20
|
+
setup: [
|
|
21
|
+
...(pm === 'pnpm' ? ['- uses: pnpm/action-setup@v6'] : []),
|
|
22
|
+
'- uses: actions/setup-node@v7',
|
|
23
|
+
` with: { node-version: 22${pm === 'npm' ? '' : `, cache: ${pm}`} }`,
|
|
24
|
+
`- run: ${install}${scripts.build ? ` && ${run} build` : ''}`,
|
|
25
|
+
],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (has('pyproject.toml') || has('requirements.txt')) {
|
|
29
|
+
const uv = has('uv.lock');
|
|
30
|
+
return {
|
|
31
|
+
kind: 'python',
|
|
32
|
+
setup: uv
|
|
33
|
+
? ['- uses: astral-sh/setup-uv@v10', '- run: uv sync']
|
|
34
|
+
: ['- uses: actions/setup-python@v7', " with: { python-version: '3.12' }", `- run: pip install ${has('pyproject.toml') ? '.' : '-r requirements.txt'}`],
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
if (has('go.mod'))
|
|
38
|
+
return { kind: 'go', setup: ['- uses: actions/setup-go@v7', " with: { go-version-file: go.mod }", '- run: go build ./...'] };
|
|
39
|
+
if (has('Cargo.toml'))
|
|
40
|
+
return { kind: 'rust', setup: ['- run: cargo build --release'] };
|
|
41
|
+
return { kind: 'unknown', setup: ['# TODO: install and build your server here'] };
|
|
42
|
+
}
|
|
43
|
+
/** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
|
|
44
|
+
export function looksSecret(key) {
|
|
45
|
+
return /token|secret|password|passwd|api[_-]?key|(^|_)key$|credential|auth/i.test(key);
|
|
46
|
+
}
|
|
47
|
+
/** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
|
|
48
|
+
export function workflowYaml(opts) {
|
|
49
|
+
const t = opts.target;
|
|
50
|
+
const lines = [
|
|
51
|
+
'# Written by `toolmenu init`. On every pull request: snapshot the tool menu this',
|
|
52
|
+
'# change produces, diff it against the committed baseline, and post one comment.',
|
|
53
|
+
'# https://github.com/niksa90/toolmenu',
|
|
54
|
+
'name: toolmenu',
|
|
55
|
+
'on: pull_request',
|
|
56
|
+
'permissions:',
|
|
57
|
+
' contents: read',
|
|
58
|
+
' pull-requests: write',
|
|
59
|
+
'jobs:',
|
|
60
|
+
' toolmenu:',
|
|
61
|
+
' runs-on: ubuntu-latest',
|
|
62
|
+
' steps:',
|
|
63
|
+
' - uses: actions/checkout@v7',
|
|
64
|
+
...(t.kind === 'stdio' ? opts.project.setup.map((l) => ` ${l}`) : []),
|
|
65
|
+
` - uses: niksa90/toolmenu@v${VERSION}`,
|
|
66
|
+
' with:',
|
|
67
|
+
];
|
|
68
|
+
if (t.kind === 'stdio') {
|
|
69
|
+
lines.push(` command: ${yamlString([t.command, ...t.args].map(shellWord).join(' '))}`);
|
|
70
|
+
const env = Object.keys(t.env ?? {});
|
|
71
|
+
if (env.length) {
|
|
72
|
+
lines.push(' env: |');
|
|
73
|
+
for (const k of env)
|
|
74
|
+
lines.push(` ${k}=${looksSecret(k) ? `\${{ secrets.${k} }}` : t.env[k]}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
lines.push(` url: ${yamlString(t.url)} # checks the server at this URL, not this PR's code: see docs/github-action.md`);
|
|
79
|
+
const headers = Object.keys(t.headers ?? {});
|
|
80
|
+
if (headers.length) {
|
|
81
|
+
lines.push(' headers: |');
|
|
82
|
+
for (const h of headers)
|
|
83
|
+
lines.push(` ${h}: \${{ secrets.${secretName(h)} }}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
lines.push(` baseline: ${opts.baseline ?? 'menu.json'}`);
|
|
87
|
+
if (opts.session)
|
|
88
|
+
lines.push(' scenario: auto # also call read-only tools and watch the menu');
|
|
89
|
+
return lines.join('\n') + '\n';
|
|
90
|
+
}
|
|
91
|
+
/** The secret names the workflow refers to, to tell the user what to add. */
|
|
92
|
+
export function secretsNeeded(target) {
|
|
93
|
+
if (target.kind === 'stdio')
|
|
94
|
+
return Object.keys(target.env ?? {}).filter(looksSecret);
|
|
95
|
+
return Object.keys(target.headers ?? {}).map(secretName);
|
|
96
|
+
}
|
|
97
|
+
function secretName(header) {
|
|
98
|
+
return 'MCP_' + header.toUpperCase().replace(/[^A-Z0-9]+/g, '_');
|
|
99
|
+
}
|
|
100
|
+
function shellWord(w) {
|
|
101
|
+
return /^[\w./:=@%+-]+$/.test(w) ? w : `'${w.replace(/'/g, `'\\''`)}'`;
|
|
102
|
+
}
|
|
103
|
+
function yamlString(s) {
|
|
104
|
+
return /^[\w./:@%+ =-]+$/.test(s) && !/^[-?:,[\]{}#&*!|>'"%@`]/.test(s) && !/: |\s#/.test(s) ? s : JSON.stringify(s);
|
|
105
|
+
}
|
package/dist/menu.js
CHANGED
|
@@ -61,6 +61,8 @@ export async function loadMenu(path) {
|
|
|
61
61
|
t.tokens = toolTokens(t);
|
|
62
62
|
}
|
|
63
63
|
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
64
|
+
for (const op of menu.catalog?.operations ?? [])
|
|
65
|
+
op.tokens = toolTokens(op);
|
|
64
66
|
menu.server ??= {};
|
|
65
67
|
return menu;
|
|
66
68
|
}
|
package/dist/report.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { stepLabel } from './session.js';
|
|
2
|
+
import { breakdown, breakdownLines } from './breakdown.js';
|
|
2
3
|
const LABEL = { error: 'ERROR', warn: 'WARN ', info: 'INFO ' };
|
|
3
4
|
export function counts(findings) {
|
|
4
5
|
const c = { error: 0, warn: 0, info: 0 };
|
|
@@ -17,6 +18,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
17
18
|
tools: menu.tools.length,
|
|
18
19
|
totalTokens: menu.totalTokens,
|
|
19
20
|
listMeta: menu.listMeta,
|
|
21
|
+
breakdown: breakdown(menu),
|
|
20
22
|
written: outPath ?? null,
|
|
21
23
|
counts: counts(findings),
|
|
22
24
|
findings,
|
|
@@ -32,6 +34,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
32
34
|
`Protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
33
35
|
'',
|
|
34
36
|
...mdFindings(findings),
|
|
37
|
+
...mdBreakdown(menu),
|
|
35
38
|
].join('\n');
|
|
36
39
|
}
|
|
37
40
|
const s = menu.server;
|
|
@@ -42,10 +45,19 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
42
45
|
if (outPath)
|
|
43
46
|
lines.push(` wrote ${outPath}`);
|
|
44
47
|
lines.push('');
|
|
48
|
+
const where = breakdownLines(breakdown(menu), menu.tools.length);
|
|
49
|
+
if (where.length)
|
|
50
|
+
lines.push(...where, '');
|
|
45
51
|
lines.push(...findingLines(findings));
|
|
46
52
|
lines.push(summaryLine(findings));
|
|
47
53
|
return lines.join('\n');
|
|
48
54
|
}
|
|
55
|
+
function mdBreakdown(menu) {
|
|
56
|
+
const where = breakdownLines(breakdown(menu), menu.tools.length);
|
|
57
|
+
if (!where.length)
|
|
58
|
+
return [];
|
|
59
|
+
return ['', `<details><summary>${where[0]}</summary>`, '', '```', ...where.slice(1).map((l) => l.trimStart()), '```', '', '</details>'];
|
|
60
|
+
}
|
|
49
61
|
function findingLines(findings) {
|
|
50
62
|
const lines = [];
|
|
51
63
|
for (const f of findings) {
|
|
@@ -197,8 +209,11 @@ export function formatHistory(h, format, outDir) {
|
|
|
197
209
|
return lines.join('\n');
|
|
198
210
|
}
|
|
199
211
|
export function formatSession(s, format) {
|
|
200
|
-
if (format === 'json')
|
|
201
|
-
|
|
212
|
+
if (format === 'json') {
|
|
213
|
+
// The union menu itself goes to --union-out; the report says how big it is.
|
|
214
|
+
const { union, ...rest } = s;
|
|
215
|
+
return JSON.stringify({ command: 'session', ...rest, union: { tools: union.tools.length, totalTokens: union.totalTokens }, counts: counts(s.findings) }, null, 2);
|
|
216
|
+
}
|
|
202
217
|
if (format === 'github') {
|
|
203
218
|
return githubLines(s.findings.map((f) => ({ ...f, message: f.step ? `step ${f.step} (${s.steps[f.step - 1]?.label}): ${f.message}` : f.message }))).join('\n');
|
|
204
219
|
}
|
|
@@ -208,6 +223,7 @@ export function formatSession(s, format) {
|
|
|
208
223
|
'',
|
|
209
224
|
`\`${s.server.name ?? 'server'}\` ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport} · ${s.baseline.tools} → ${s.final.tools} tools · ~${s.baseline.tokens.toLocaleString('en-US')} → ~${s.final.tokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
210
225
|
'',
|
|
226
|
+
...(autoLine(s) ? [autoLine(s), ''] : []),
|
|
211
227
|
'| Step | Menu | list_changed | Scope |',
|
|
212
228
|
'|---|---|---|---|',
|
|
213
229
|
...s.steps.map((st) => `| ${st.index}. ${mdCell(st.label)} | ${st.status !== 'ok' ? st.status : st.changed ? `changed (${st.tools} tools)` : 'no change'} | ${st.changed ? (st.listChanged ? 'received' : '**missing**') : ''} | ${st.scope ?? ''} |`),
|
|
@@ -221,6 +237,7 @@ export function formatSession(s, format) {
|
|
|
221
237
|
`toolmenu session ${s.scenario}`,
|
|
222
238
|
` ${s.server.name ?? 'server'} ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport}`,
|
|
223
239
|
` baseline: ${plural(s.baseline.tools, 'tool')} · ${tok(s.baseline.tokens)} tokens (estimate) · listening for list_changed: ${s.listening ? 'yes' : 'no'}`,
|
|
240
|
+
...(autoLine(s) ? [` ${autoLine(s)}`] : []),
|
|
224
241
|
` fresh-${s.transport === 'stdio' ? 'process' : 'connection'} check: ${s.connectionCheck === undefined ? 'not checked' : s.connectionCheck === 'same' ? 'the same menu' : 'a DIFFERENT menu'}`,
|
|
225
242
|
'',
|
|
226
243
|
];
|
|
@@ -264,6 +281,21 @@ export function formatPlan(scenario, name) {
|
|
|
264
281
|
lines.push('', 'After every step the menu is listed and compared with the one before. Nothing was run.');
|
|
265
282
|
return lines.join('\n');
|
|
266
283
|
}
|
|
284
|
+
/** One line on what --auto called and what it left out, and how to reach more. */
|
|
285
|
+
function autoLine(s) {
|
|
286
|
+
if (!s.auto)
|
|
287
|
+
return undefined;
|
|
288
|
+
const by = (r) => s.auto.skipped.filter((x) => x.reason === r);
|
|
289
|
+
const needs = by('needs values');
|
|
290
|
+
const params = [...new Set(needs.flatMap((x) => x.missing ?? []))];
|
|
291
|
+
const parts = [
|
|
292
|
+
by('not read-only').length && `${by('not read-only').length} not read-only`,
|
|
293
|
+
by('open world').length && `${by('open world').length} marked openWorldHint: true (--open-world to call them)`,
|
|
294
|
+
needs.length && `${needs.length} need values the schema doesn't give (${params.slice(0, 6).join(', ')}${params.length > 6 ? ', …' : ''}: write a scenario, or --save-scenario and fill them in)`,
|
|
295
|
+
by('over the call budget').length && `${by('over the call budget').length} over --max-calls`,
|
|
296
|
+
].filter(Boolean);
|
|
297
|
+
return `auto: called ${s.auto.called.length} tool${s.auto.called.length === 1 ? '' : 's'}${parts.length ? ` · skipped ${parts.join(', ')}` : ''}`;
|
|
298
|
+
}
|
|
267
299
|
function mdCell(text) {
|
|
268
300
|
return text.replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
269
301
|
}
|