toolmenu 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -20
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +53 -17
- package/dist/auto.d.ts +70 -6
- package/dist/auto.js +248 -17
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +62 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +80 -170
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +526 -114
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +9 -0
- package/dist/failures.js +17 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +92 -2
- package/dist/init.js +507 -20
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +85 -42
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +145 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +12 -4
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.js +33 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/session.d.ts +18 -8
- package/dist/session.js +484 -92
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +17 -2
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
4
|
import { parseArgs } from 'node:util';
|
|
5
5
|
import { loadConfig } from './config.js';
|
|
6
6
|
import { diffMenus } from './diff.js';
|
|
@@ -15,99 +15,13 @@ import { existsSync } from 'node:fs';
|
|
|
15
15
|
import { loadRoutes } from './routes.js';
|
|
16
16
|
import { snapshot } from './snapshot.js';
|
|
17
17
|
import { connectPatiently, MAIN_SEED, probeMenu, refusedForTooMany, seeded } from './probe.js';
|
|
18
|
-
import { autoScenario, scenarioYaml } from './auto.js';
|
|
18
|
+
import { autoScenario, loadValuesFile, parseAssumeReadOnly, parseValueFlag, scenarioYaml } from './auto.js';
|
|
19
19
|
import { authDir, listLogins, login, logout } from './auth.js';
|
|
20
|
-
import {
|
|
20
|
+
import { existingMessage, initReport, planInit, writeInit } from './init.js';
|
|
21
21
|
import { SEVERITY_RANK } from './types.js';
|
|
22
22
|
import { VERSION } from './version.js';
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Usage:
|
|
27
|
-
toolmenu init [options] -- <command> | <url> set up CI: a baseline and a workflow
|
|
28
|
-
toolmenu snapshot [options] -- <command> [args...] stdio server
|
|
29
|
-
toolmenu snapshot [options] <url> Streamable HTTP server
|
|
30
|
-
toolmenu diff [options] <old.json> <new.json> compare two snapshots
|
|
31
|
-
toolmenu history [options] <npm-package> snapshot and diff published versions
|
|
32
|
-
toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
|
|
33
|
-
toolmenu session --scenario <file> [options] -- <command> | <url>
|
|
34
|
-
run a scripted session, watch the menu
|
|
35
|
-
toolmenu session --auto [options] -- <command> | <url>
|
|
36
|
-
the same, with steps built from the menu
|
|
37
|
-
|
|
38
|
-
Commands:
|
|
39
|
-
init snapshot the server into menu.json and write .github/workflows/toolmenu.yml
|
|
40
|
-
snapshot establish the menu: list tools twice, write menu.json, run the menu rules
|
|
41
|
-
diff compare releases: breaking changes, token change, semver bump
|
|
42
|
-
session observe the menu changing while a scripted session runs
|
|
43
|
-
history research release history: install, snapshot and diff published npm versions
|
|
44
|
-
auth log in to an OAuth-protected server once; snapshot and session then use it
|
|
45
|
-
|
|
46
|
-
Options:
|
|
47
|
-
--out <path> where to write the menu (default: menu.json)
|
|
48
|
-
--no-write don't write the menu file
|
|
49
|
-
--routes <path> routes.yml: pin which words route to which tools
|
|
50
|
-
--config <path> config file (default: toolmenu.config.json, if present)
|
|
51
|
-
--format <fmt> text (default), json, github (annotations) or markdown (PR comments)
|
|
52
|
-
--json same as --format json
|
|
53
|
-
--fail-on <level> error (default), warn or info
|
|
54
|
-
--header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
|
|
55
|
-
--env <K=V> environment variable for a stdio server, repeatable. The server
|
|
56
|
-
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
57
|
-
--no-auth don't use a stored OAuth login for this server
|
|
58
|
-
--timeout <ms> per-request timeout (default: 30000)
|
|
59
|
-
--catalog also read the operations behind a search tool (search and execute:
|
|
60
|
-
discover, search_*_tools) and keep them in the menu file for diff
|
|
61
|
-
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
62
|
-
the main one included (default: 2; 1 opens no second one, for
|
|
63
|
-
session's scope check either: for servers with one session per client)
|
|
64
|
-
-h, --help show this help
|
|
65
|
-
-v, --version show the version
|
|
66
|
-
|
|
67
|
-
diff options:
|
|
68
|
-
--release <old>..<new> the release versions (npm, git tag) to check the bump
|
|
69
|
-
against, e.g. 1.4.0..1.5.0. Without it, the bump isn't
|
|
70
|
-
checked: serverInfo.version is often not the release
|
|
71
|
-
--server-version-is-release check the bump against serverInfo.version instead
|
|
72
|
-
|
|
73
|
-
session options:
|
|
74
|
-
--scenario <path> scenario.yml: the steps to run (list, call, wait_for)
|
|
75
|
-
--plan print the steps without connecting or running anything
|
|
76
|
-
--init write a starter scenario from the server's menu (to --scenario,
|
|
77
|
-
default scenario.yml; never overwrites)
|
|
78
|
-
--auto build the steps from the menu: every read-only tool whose
|
|
79
|
-
required arguments the schema can fill (const, default,
|
|
80
|
-
examples, enum, type), then the first call again
|
|
81
|
-
--open-world with --auto, also call read-only tools marked openWorldHint:
|
|
82
|
-
true (web search, fetch, scraping): they may cost API credits
|
|
83
|
-
--max-calls <n> with --auto, at most n calls (default: 20)
|
|
84
|
-
--save-scenario <p> with --auto, write the steps it ran as a scenario file
|
|
85
|
-
--union-out <path> write every tool the session saw as a menu file, to commit as
|
|
86
|
-
the baseline for diff (tools behind unlocks included)
|
|
87
|
-
|
|
88
|
-
history options (best effort; installs and runs third-party code, so use a container):
|
|
89
|
-
--versions <n|all> number of published versions to inspect (default: 10)
|
|
90
|
-
--include-prereleases include versions like 1.2.0-beta.1
|
|
91
|
-
--arg <value> argument for the server's bin, repeatable (use --arg=--flag for flags)
|
|
92
|
-
--bin <name> which bin to run when the package has several
|
|
93
|
-
--cmd <template> command instead of the bin; {bin} and {dir} are filled in
|
|
94
|
-
--allow-scripts run install scripts (off by default)
|
|
95
|
-
--install-timeout <ms> per-version install timeout (default: 180000)
|
|
96
|
-
--csv <path> also write the dataset as CSV
|
|
97
|
-
--keep-installs keep each version's install directory
|
|
98
|
-
--out <dir> where menus and history.json go (default: toolmenu-history/<package>)
|
|
99
|
-
|
|
100
|
-
init options:
|
|
101
|
-
--with-session the workflow also runs session --auto (calls read-only tools)
|
|
102
|
-
|
|
103
|
-
auth options:
|
|
104
|
-
--port <n> loopback port for the login redirect (default: 33418)
|
|
105
|
-
--scope <scopes> scopes to ask for (default: the server's)
|
|
106
|
-
--client-id <id> a pre-registered client, for servers without dynamic registration
|
|
107
|
-
--client-secret <s> its secret, if it has one
|
|
108
|
-
|
|
109
|
-
Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
|
|
110
|
-
`;
|
|
23
|
+
import { CLI_OPTIONS } from './options.js';
|
|
24
|
+
import { commandHelp, isCommand, overview, unknownCommand } from './help.js';
|
|
111
25
|
class UsageError extends Error {
|
|
112
26
|
}
|
|
113
27
|
export async function main(argv) {
|
|
@@ -117,59 +31,26 @@ export async function main(argv) {
|
|
|
117
31
|
const { values, positionals } = parseArgs({
|
|
118
32
|
args: before,
|
|
119
33
|
allowPositionals: true,
|
|
120
|
-
options:
|
|
121
|
-
out: { type: 'string' },
|
|
122
|
-
'no-write': { type: 'boolean' },
|
|
123
|
-
routes: { type: 'string' },
|
|
124
|
-
config: { type: 'string' },
|
|
125
|
-
format: { type: 'string' },
|
|
126
|
-
json: { type: 'boolean' },
|
|
127
|
-
'fail-on': { type: 'string' },
|
|
128
|
-
header: { type: 'string', multiple: true },
|
|
129
|
-
env: { type: 'string', multiple: true },
|
|
130
|
-
timeout: { type: 'string' },
|
|
131
|
-
'no-auth': { type: 'boolean' },
|
|
132
|
-
port: { type: 'string' },
|
|
133
|
-
scope: { type: 'string' },
|
|
134
|
-
'client-id': { type: 'string' },
|
|
135
|
-
'client-secret': { type: 'string' },
|
|
136
|
-
processes: { type: 'string' },
|
|
137
|
-
versions: { type: 'string' },
|
|
138
|
-
release: { type: 'string' },
|
|
139
|
-
'server-version-is-release': { type: 'boolean' },
|
|
140
|
-
'include-prereleases': { type: 'boolean' },
|
|
141
|
-
arg: { type: 'string', multiple: true },
|
|
142
|
-
bin: { type: 'string' },
|
|
143
|
-
cmd: { type: 'string' },
|
|
144
|
-
'allow-scripts': { type: 'boolean' },
|
|
145
|
-
'install-timeout': { type: 'string' },
|
|
146
|
-
csv: { type: 'string' },
|
|
147
|
-
'keep-installs': { type: 'boolean' },
|
|
148
|
-
scenario: { type: 'string' },
|
|
149
|
-
plan: { type: 'boolean' },
|
|
150
|
-
init: { type: 'boolean' },
|
|
151
|
-
'union-out': { type: 'string' },
|
|
152
|
-
auto: { type: 'boolean' },
|
|
153
|
-
'with-session': { type: 'boolean' },
|
|
154
|
-
catalog: { type: 'boolean' },
|
|
155
|
-
'open-world': { type: 'boolean' },
|
|
156
|
-
'max-calls': { type: 'string' },
|
|
157
|
-
'save-scenario': { type: 'string' },
|
|
158
|
-
help: { type: 'boolean', short: 'h' },
|
|
159
|
-
version: { type: 'boolean', short: 'v' },
|
|
160
|
-
},
|
|
34
|
+
options: CLI_OPTIONS,
|
|
161
35
|
});
|
|
162
36
|
if (values.version) {
|
|
163
37
|
process.stdout.write(`${VERSION}\n`);
|
|
164
38
|
return 0;
|
|
165
39
|
}
|
|
166
40
|
const [sub, ...rest] = positionals;
|
|
41
|
+
// toolmenu help [command] reads like toolmenu [command] --help.
|
|
42
|
+
if (sub === 'help' && rest.length <= 1) {
|
|
43
|
+
if (rest[0] && !isCommand(rest[0]))
|
|
44
|
+
throw new UsageError(unknownCommand(rest[0]));
|
|
45
|
+
process.stdout.write(rest[0] && isCommand(rest[0]) ? commandHelp(rest[0]) : overview());
|
|
46
|
+
return 0;
|
|
47
|
+
}
|
|
48
|
+
if (sub && !isCommand(sub))
|
|
49
|
+
throw new UsageError(unknownCommand(sub));
|
|
167
50
|
if (values.help || !sub) {
|
|
168
|
-
process.stdout.write(
|
|
51
|
+
process.stdout.write(sub && isCommand(sub) ? commandHelp(sub) : overview());
|
|
169
52
|
return sub || values.help ? 0 : 2;
|
|
170
53
|
}
|
|
171
|
-
if (!['init', 'snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
172
|
-
throw new UsageError(`Unknown command "${sub}". Try --help.`);
|
|
173
54
|
const format = (values.json ? 'json' : values.format ?? 'text');
|
|
174
55
|
if (!['text', 'json', 'github', 'markdown'].includes(format))
|
|
175
56
|
throw new UsageError(`--format must be text, json, github or markdown`);
|
|
@@ -232,7 +113,7 @@ export async function main(argv) {
|
|
|
232
113
|
});
|
|
233
114
|
if (values.csv)
|
|
234
115
|
await writeFile(values.csv, historyCsv(result));
|
|
235
|
-
process.stdout.write(formatHistory(result, format, outDir) + '\n');
|
|
116
|
+
process.stdout.write(formatHistory(result, format, outDir, values.csv) + '\n');
|
|
236
117
|
return result.rows.length > 0 && result.rows.every((r) => r.status === 'failed') ? 1 : 0;
|
|
237
118
|
}
|
|
238
119
|
if (sub === 'session' && values.init) {
|
|
@@ -270,32 +151,13 @@ export async function main(argv) {
|
|
|
270
151
|
return 0;
|
|
271
152
|
}
|
|
272
153
|
if (sub === 'init') {
|
|
273
|
-
const workflow = join('.github', 'workflows', 'toolmenu.yml');
|
|
274
|
-
const baseline = values.out ?? 'menu.json';
|
|
275
|
-
for (const f of [workflow, baseline])
|
|
276
|
-
if (existsSync(f))
|
|
277
|
-
throw new UsageError(`${f} already exists; init never overwrites. Remove it, or set things up by hand (README: "In CI").`);
|
|
278
154
|
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
155
|
+
const plan = planInit({ cwd: process.cwd(), target, out: values.out, session: values['with-session'] });
|
|
156
|
+
if (plan.existing.length)
|
|
157
|
+
throw new UsageError(existingMessage(plan));
|
|
279
158
|
const { menu, findings } = await snapshot(target, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
280
|
-
await
|
|
281
|
-
|
|
282
|
-
const project = detectProject('.');
|
|
283
|
-
await writeFile(workflow, workflowYaml({ target, project, session: values['with-session'], baseline }));
|
|
284
|
-
const c = counts(findings);
|
|
285
|
-
const secrets = secretsNeeded(target);
|
|
286
|
-
const lines = [
|
|
287
|
-
`toolmenu init ${menu.server.name ?? 'server'} · ${menu.tools.length} tools · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
288
|
-
` wrote ${baseline}: the baseline every pull request is compared with`,
|
|
289
|
-
` wrote ${workflow}${project.kind === 'unknown' ? ' (fill in the TODO: how CI builds your server)' : ` (${project.kind} project)`}`,
|
|
290
|
-
` today's menu: ${c.error} errors, ${c.warn} warnings, ${c.info} info (run toolmenu snapshot to see them)`,
|
|
291
|
-
...(secrets.length ? ['', `Add these repository secrets (Settings → Secrets and variables → Actions): ${secrets.join(', ')}`] : []),
|
|
292
|
-
'',
|
|
293
|
-
'Next:',
|
|
294
|
-
` git add ${baseline} ${workflow} && git commit -m "Check the MCP tool menu on every PR"`,
|
|
295
|
-
' Then open a pull request: toolmenu comments on it. Refresh the baseline when a change is intended:',
|
|
296
|
-
` toolmenu snapshot --out ${baseline} ${target.kind === 'stdio' ? '-- ' + [target.command, ...target.args].join(' ') : target.url}`,
|
|
297
|
-
];
|
|
298
|
-
process.stdout.write(lines.join('\n') + '\n');
|
|
159
|
+
await writeInit(plan, JSON.stringify(menu, null, 2) + '\n');
|
|
160
|
+
process.stdout.write(initReport(plan, { server: menu.server.name, version: menu.server.version, tools: menu.tools.length, tokens: menu.totalTokens, counts: counts(findings) }, target));
|
|
299
161
|
return 0;
|
|
300
162
|
}
|
|
301
163
|
if (sub === 'session' && values.auto) {
|
|
@@ -307,20 +169,36 @@ export async function main(argv) {
|
|
|
307
169
|
const savePath = values['save-scenario'];
|
|
308
170
|
if (savePath && existsSync(savePath))
|
|
309
171
|
throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
|
|
172
|
+
// Values and vouched-for tools are checked before anything starts.
|
|
173
|
+
let given = {};
|
|
174
|
+
let assumeReadOnly = [];
|
|
175
|
+
try {
|
|
176
|
+
if (values['values-file'])
|
|
177
|
+
given = await loadValuesFile(values['values-file']);
|
|
178
|
+
for (const v of values.value ?? []) {
|
|
179
|
+
const [key, value] = parseValueFlag(v);
|
|
180
|
+
given[key] = value;
|
|
181
|
+
}
|
|
182
|
+
assumeReadOnly = parseAssumeReadOnly(values['assume-read-only'] ?? []);
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
throw new UsageError(error instanceof Error ? error.message : String(error));
|
|
186
|
+
}
|
|
310
187
|
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
311
188
|
const waited = { ms: 0 };
|
|
312
189
|
const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs, undefined, waited).catch((error) => {
|
|
313
190
|
throw refusedForTooMany(error, waited.ms);
|
|
314
191
|
});
|
|
315
|
-
const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
|
|
192
|
+
const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls, values: given, assumeReadOnly });
|
|
316
193
|
if (savePath)
|
|
317
194
|
await writeFile(savePath, scenarioYaml(plan, menu.server.name));
|
|
318
195
|
if (values.plan) {
|
|
319
196
|
process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
|
|
320
197
|
return 0;
|
|
321
198
|
}
|
|
322
|
-
const
|
|
323
|
-
result
|
|
199
|
+
const auto = { called: plan.called, skipped: plan.skipped, assumed: plan.assumed, withValues: plan.withValues, ignored: plan.ignored, maxCalls };
|
|
200
|
+
const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto', unionOut: !!values['union-out'], auto });
|
|
201
|
+
result.auto ??= auto;
|
|
324
202
|
if (values['union-out'])
|
|
325
203
|
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
326
204
|
const output = formatSession(result, format);
|
|
@@ -329,6 +207,8 @@ export async function main(argv) {
|
|
|
329
207
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
330
208
|
}
|
|
331
209
|
if (sub === 'session') {
|
|
210
|
+
if (values.value || values['values-file'] || values['assume-read-only'])
|
|
211
|
+
throw new UsageError('--value, --values-file and --assume-read-only go with --auto. In a scenario, write the values into the steps\' args, and list tools under assume_read_only.');
|
|
332
212
|
if (!values.scenario)
|
|
333
213
|
throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
|
|
334
214
|
let scenario;
|
|
@@ -384,17 +264,20 @@ async function authCommand(args, values) {
|
|
|
384
264
|
const [action, url, ...extra] = args;
|
|
385
265
|
if (action === 'list') {
|
|
386
266
|
const logins = await listLogins();
|
|
387
|
-
|
|
267
|
+
const width = Math.max(0, ...logins.map((l) => l.serverUrl.length));
|
|
268
|
+
process.stdout.write(logins.length
|
|
269
|
+
? [`${logins.length} login${logins.length === 1 ? '' : 's'}, stored in ${authDir()}:`, ...logins.map((l) => ` ${l.serverUrl.padEnd(width)} issuer ${l.issuer ?? 'not recorded'} · saved ${l.savedAt?.slice(0, 10) ?? '?'}`)].join('\n') + '\n'
|
|
270
|
+
: `No logins stored in ${authDir()}.\n→ Next: toolmenu auth login <url> for a server that asks for OAuth.\n`);
|
|
388
271
|
return 0;
|
|
389
272
|
}
|
|
390
273
|
if (action !== 'login' && action !== 'logout')
|
|
391
|
-
throw new UsageError('auth
|
|
274
|
+
throw new UsageError(`${action ? `auth has no action "${action}"` : 'auth needs an action'}: login <url>, logout <url> or list. See toolmenu auth --help.`);
|
|
392
275
|
if (!url || extra.length)
|
|
393
276
|
throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
|
|
394
277
|
if (!/^https?:\/\//.test(url))
|
|
395
278
|
throw new UsageError(`"${url}" isn't an http(s) URL.`);
|
|
396
279
|
if (action === 'logout') {
|
|
397
|
-
process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
|
|
280
|
+
process.stdout.write((await logout(url)) ? `Logged out of ${url}: its tokens and client registration are removed from ${authDir()}.\n` : `No login stored for ${url} (in ${authDir()}), so nothing to remove. toolmenu auth list shows the stored ones.\n`);
|
|
398
281
|
return 0;
|
|
399
282
|
}
|
|
400
283
|
const port = values.port ? Number(values.port) : undefined;
|
|
@@ -403,7 +286,7 @@ async function authCommand(args, values) {
|
|
|
403
286
|
if (values['client-secret'] && !values['client-id'])
|
|
404
287
|
throw new UsageError('--client-secret goes with --client-id');
|
|
405
288
|
const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
|
|
406
|
-
process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools
|
|
289
|
+
process.stdout.write(`Logged in to ${result.name ?? url} (${url}): it lists ${result.tools} tool${result.tools === 1 ? '' : 's'} with this login.\nsnapshot and session use it from now on (--no-auth to skip it); stored in ${authDir()}.\n`);
|
|
407
290
|
return 0;
|
|
408
291
|
}
|
|
409
292
|
function parsePairs(items, separator, flag) {
|
|
@@ -434,10 +317,37 @@ for (const method of ['log', 'info', 'debug']) {
|
|
|
434
317
|
console[method] = (...args) => console.error(...args);
|
|
435
318
|
}
|
|
436
319
|
main(process.argv.slice(2)).then(exitWhenFlushed, (error) => {
|
|
437
|
-
|
|
438
|
-
process.stderr.write(`toolmenu: ${message}\n`);
|
|
320
|
+
process.stderr.write(`toolmenu: ${failureMessage(error)}\n`);
|
|
439
321
|
exitWhenFlushed(2);
|
|
440
322
|
});
|
|
323
|
+
/** What stopped the command. A mistyped option gets the nearest real one. */
|
|
324
|
+
function failureMessage(error) {
|
|
325
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
326
|
+
const code = error?.code;
|
|
327
|
+
if (code === 'ERR_PARSE_ARGS_UNKNOWN_OPTION') {
|
|
328
|
+
const option = /Unknown option '([^']+)'/.exec(message)?.[1] ?? '';
|
|
329
|
+
const known = Object.keys(CLI_OPTIONS).map((name) => `--${name}`);
|
|
330
|
+
const guess = known.map((k) => [k, editDistance(option, k)]).sort((a, b) => a[1] - b[1])[0];
|
|
331
|
+
const hint = guess && guess[1] <= 2 ? ` Did you mean ${guess[0]}?` : '';
|
|
332
|
+
return `Unknown option ${option}.${hint}\n → Next: toolmenu --help lists every option. (A server's own flags go after "--".)`;
|
|
333
|
+
}
|
|
334
|
+
if (typeof code === 'string' && code.startsWith('ERR_PARSE_ARGS_'))
|
|
335
|
+
return `${message}\n → Next: toolmenu --help lists every option.`;
|
|
336
|
+
return message;
|
|
337
|
+
}
|
|
338
|
+
function editDistance(a, b) {
|
|
339
|
+
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
340
|
+
for (let i = 1; i <= a.length; i++) {
|
|
341
|
+
let diagonal = row[0];
|
|
342
|
+
row[0] = i;
|
|
343
|
+
for (let j = 1; j <= b.length; j++) {
|
|
344
|
+
const above = row[j];
|
|
345
|
+
row[j] = Math.min(row[j] + 1, row[j - 1] + 1, diagonal + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
346
|
+
diagonal = above;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return row[b.length];
|
|
350
|
+
}
|
|
441
351
|
function parseRelease(value) {
|
|
442
352
|
const m = /^(.+?)\.\.(.+)$/.exec(value);
|
|
443
353
|
if (!m)
|
package/dist/config.d.ts
CHANGED
|
@@ -16,9 +16,13 @@ export interface Config {
|
|
|
16
16
|
fullDescriptions?: string[];
|
|
17
17
|
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
18
18
|
processes?: number;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* snapshot --catalog: which search tool and which queries (default: detected,
|
|
21
|
+
* derived from the menu), or which command routers to list (default: detected).
|
|
22
|
+
*/
|
|
20
23
|
catalog?: {
|
|
21
24
|
tool?: string;
|
|
25
|
+
routers?: string[];
|
|
22
26
|
queries?: string[];
|
|
23
27
|
pauseMs?: number;
|
|
24
28
|
crawl?: boolean;
|
package/dist/config.js
CHANGED
|
@@ -26,5 +26,19 @@ export async function loadConfig(path) {
|
|
|
26
26
|
if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
|
|
27
27
|
throw new Error(`${file}: processes must be a whole number, 1 or more`);
|
|
28
28
|
}
|
|
29
|
+
const catalog = config.catalog;
|
|
30
|
+
if (catalog !== undefined) {
|
|
31
|
+
const names = (v) => Array.isArray(v) && v.length > 0 && v.every((x) => typeof x === 'string' && x.length > 0);
|
|
32
|
+
if (typeof catalog !== 'object' || catalog === null || Array.isArray(catalog))
|
|
33
|
+
throw new Error(`${file}: catalog must be an object, like { "routers": ["keyvault"] } or { "tool": "search_tools" }`);
|
|
34
|
+
if (catalog.tool !== undefined && !(typeof catalog.tool === 'string' && catalog.tool))
|
|
35
|
+
throw new Error(`${file}: catalog.tool must be the name of the search tool, a string`);
|
|
36
|
+
if (catalog.routers !== undefined && !names(catalog.routers))
|
|
37
|
+
throw new Error(`${file}: catalog.routers must be a list of tool names, like ["keyvault", "storage"]`);
|
|
38
|
+
if (catalog.tool !== undefined && catalog.routers !== undefined)
|
|
39
|
+
throw new Error(`${file}: catalog names both a search tool (tool) and routers (routers); a menu file keeps one catalog, so keep the one the operations are behind`);
|
|
40
|
+
if (catalog.queries !== undefined && !names(catalog.queries))
|
|
41
|
+
throw new Error(`${file}: catalog.queries must be a list of search phrases, like ["list issues"]`);
|
|
42
|
+
}
|
|
29
43
|
return config;
|
|
30
44
|
}
|
package/dist/connect.d.ts
CHANGED
|
@@ -47,6 +47,10 @@ export interface Connection {
|
|
|
47
47
|
usedAuth: boolean;
|
|
48
48
|
stderr: () => string;
|
|
49
49
|
close: () => Promise<void>;
|
|
50
|
+
/** An error from a request on this connection, said with what the connection saw (exit code, stderr, stray stdout). */
|
|
51
|
+
explain?: (error: unknown, stage: string) => unknown;
|
|
52
|
+
/** Resolves once a server that just failed a request has had its say on stderr (all of it, if it exited). */
|
|
53
|
+
stderrSettled?: () => Promise<void>;
|
|
50
54
|
}
|
|
51
55
|
export interface ConnectOptions {
|
|
52
56
|
timeoutMs?: number;
|
package/dist/connect.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
2
|
import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
|
|
3
3
|
import { hasLogin, StoredOAuthProvider } from './auth.js';
|
|
4
|
+
import { explain, Trace } from './explain.js';
|
|
4
5
|
import { eraOf } from './menu.js';
|
|
5
6
|
import { VERSION } from './version.js';
|
|
6
7
|
/** Every response the server sent, matched to the request method, as it came off the wire. */
|
|
@@ -50,6 +51,8 @@ export async function connect(target, options = {}) {
|
|
|
50
51
|
let transport;
|
|
51
52
|
let stderrText = '';
|
|
52
53
|
let oauth = false;
|
|
54
|
+
const trace = new Trace();
|
|
55
|
+
let stdioStderr = null;
|
|
53
56
|
if (target.kind === 'stdio') {
|
|
54
57
|
const stdio = new StdioClientTransport({
|
|
55
58
|
command: target.command,
|
|
@@ -59,8 +62,12 @@ export async function connect(target, options = {}) {
|
|
|
59
62
|
stderr: 'pipe',
|
|
60
63
|
});
|
|
61
64
|
stdio.stderr?.on('data', (chunk) => {
|
|
62
|
-
|
|
65
|
+
const text = chunk.toString();
|
|
66
|
+
stderrText = (stderrText + text).slice(-4000);
|
|
67
|
+
trace.stderr(text);
|
|
63
68
|
});
|
|
69
|
+
stdioStderr = stdio.stderr;
|
|
70
|
+
watchProcess(stdio, trace);
|
|
64
71
|
transport = stdio;
|
|
65
72
|
}
|
|
66
73
|
else {
|
|
@@ -70,23 +77,33 @@ export async function connect(target, options = {}) {
|
|
|
70
77
|
oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
|
|
71
78
|
transport = new StreamableHTTPClientTransport(new URL(target.url), {
|
|
72
79
|
requestInit: { headers: target.headers ?? {} },
|
|
80
|
+
fetch: tracedFetch(trace),
|
|
73
81
|
...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
|
|
74
82
|
});
|
|
75
83
|
}
|
|
84
|
+
const context = { target, timeoutMs: options.timeoutMs, oauth };
|
|
76
85
|
try {
|
|
77
86
|
await client.connect(transport, { timeout: options.timeoutMs });
|
|
78
87
|
}
|
|
79
88
|
catch (error) {
|
|
80
|
-
//
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
if (
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
89
|
+
// What failed is fixed now: the exit that follows toolmenu's own SIGTERM isn't the server's doing.
|
|
90
|
+
trace.failedAt = Date.now();
|
|
91
|
+
// A server that never answered is stopped now, not after close()'s grace period for a clean exit.
|
|
92
|
+
if (trace.process && !trace.exit) {
|
|
93
|
+
trace.stopped = true;
|
|
94
|
+
trace.process.kill('SIGTERM');
|
|
95
|
+
}
|
|
96
|
+
// Read stderr to its end: the last lines usually say why the server exited. A
|
|
97
|
+
// process that has exited closes its end of the pipe, so wait for all of it (a
|
|
98
|
+
// loaded CI runner took over a second for 300 lines, and the reason is last);
|
|
99
|
+
// one toolmenu just stopped gets a moment to exit first.
|
|
100
|
+
if (target.kind === 'stdio') {
|
|
101
|
+
if (!trace.exit)
|
|
102
|
+
await exited(trace, 1000);
|
|
103
|
+
await stderrEnded(stdioStderr, trace.exit ? STDERR_AFTER_EXIT_MS : 1000);
|
|
87
104
|
}
|
|
88
105
|
await client.close().catch(() => { });
|
|
89
|
-
throw error;
|
|
106
|
+
throw explain(error, trace, context);
|
|
90
107
|
}
|
|
91
108
|
const wire = new WireLog();
|
|
92
109
|
wire.tap(transport);
|
|
@@ -103,22 +120,126 @@ export async function connect(target, options = {}) {
|
|
|
103
120
|
(oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
|
|
104
121
|
stderr: () => stderrText,
|
|
105
122
|
close: () => client.close(),
|
|
123
|
+
explain: (error, stage) => explain(error, trace, { ...context, stage }),
|
|
124
|
+
stderrSettled: async () => {
|
|
125
|
+
if (target.kind !== 'stdio')
|
|
126
|
+
return;
|
|
127
|
+
// A server still running gets a moment to exit; one that exited is read to the end of its stderr.
|
|
128
|
+
if (!trace.exit)
|
|
129
|
+
await exited(trace, 200);
|
|
130
|
+
if (trace.exit)
|
|
131
|
+
await stderrEnded(stdioStderr, STDERR_AFTER_EXIT_MS);
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/** How long to wait for an exited server's stderr to end: its pipe is closing, so this is only a ceiling for a stuck one. */
|
|
136
|
+
const STDERR_AFTER_EXIT_MS = 10_000;
|
|
137
|
+
/** Resolves when the traced process has exited, or after `maxMs`. */
|
|
138
|
+
function exited(trace, maxMs) {
|
|
139
|
+
const child = trace.process;
|
|
140
|
+
if (!child || trace.exit)
|
|
141
|
+
return Promise.resolve();
|
|
142
|
+
return new Promise((resolve) => {
|
|
143
|
+
const timer = setTimeout(resolve, maxMs);
|
|
144
|
+
child.once('exit', () => {
|
|
145
|
+
clearTimeout(timer);
|
|
146
|
+
// The exit listener that records trace.exit runs first: it was added at spawn.
|
|
147
|
+
resolve();
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
/** Resolves when `stream` has ended, or after `maxMs`. */
|
|
152
|
+
function stderrEnded(stream, maxMs) {
|
|
153
|
+
if (!stream || maxMs <= 0 || stream.readableEnded)
|
|
154
|
+
return Promise.resolve();
|
|
155
|
+
return new Promise((resolve) => {
|
|
156
|
+
const timer = setTimeout(resolve, maxMs);
|
|
157
|
+
stream.once('end', () => {
|
|
158
|
+
clearTimeout(timer);
|
|
159
|
+
resolve();
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Watch the stdio process toolmenu keeps (not the throwaway one the SDK starts
|
|
165
|
+
* for the server/discover probe): when it started, what it was asked, what it
|
|
166
|
+
* answered, its exit, and stdout lines that aren't JSON-RPC. The hooks go on this
|
|
167
|
+
* instance only: the SDK clones the transport's class for the probe.
|
|
168
|
+
*/
|
|
169
|
+
function watchProcess(stdio, trace) {
|
|
170
|
+
const start = stdio.start.bind(stdio);
|
|
171
|
+
stdio.start = async () => {
|
|
172
|
+
trace.sessionStartedAt = Date.now();
|
|
173
|
+
const onmessage = stdio.onmessage;
|
|
174
|
+
const ids = new Map();
|
|
175
|
+
const t = stdio;
|
|
176
|
+
t.onmessage = (message, extra) => {
|
|
177
|
+
if (!('method' in message) && 'id' in message && message.id !== undefined) {
|
|
178
|
+
const method = ids.get(message.id);
|
|
179
|
+
if (method)
|
|
180
|
+
trace.answered.add(method);
|
|
181
|
+
}
|
|
182
|
+
onmessage?.(message, extra);
|
|
183
|
+
};
|
|
184
|
+
const send = t.send.bind(stdio);
|
|
185
|
+
t.send = (message, options) => {
|
|
186
|
+
if ('method' in message && 'id' in message && message.id !== undefined) {
|
|
187
|
+
ids.set(message.id, message.method);
|
|
188
|
+
trace.send(message.method);
|
|
189
|
+
}
|
|
190
|
+
return send(message, options);
|
|
191
|
+
};
|
|
192
|
+
await start();
|
|
193
|
+
const child = stdio._process;
|
|
194
|
+
if (child) {
|
|
195
|
+
trace.process = child;
|
|
196
|
+
child.once('exit', (code, signal) => (trace.exit = { code, signal, afterStop: trace.stopped }));
|
|
197
|
+
child.stdout?.on('data', (chunk) => trace.stdout(chunk.toString()));
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/** fetch, noting each JSON-RPC request's method and the answer's status, content type and final URL. */
|
|
202
|
+
function tracedFetch(trace) {
|
|
203
|
+
return async (input, init) => {
|
|
204
|
+
let method;
|
|
205
|
+
if (typeof init?.body === 'string') {
|
|
206
|
+
try {
|
|
207
|
+
const body = JSON.parse(init.body);
|
|
208
|
+
if (typeof body.method === 'string' && body.id !== undefined)
|
|
209
|
+
method = body.method;
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
// Not JSON: not a request to trace.
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (method)
|
|
216
|
+
trace.send(method);
|
|
217
|
+
const response = await fetch(input, init);
|
|
218
|
+
if (method) {
|
|
219
|
+
trace.http.set(method, { status: response.status, statusText: response.statusText, contentType: response.headers.get('content-type') ?? '', url: response.url });
|
|
220
|
+
}
|
|
221
|
+
return response;
|
|
106
222
|
};
|
|
107
223
|
}
|
|
108
224
|
/** One full `tools/list`, bypassing the SDK's cache, read from the wire rather than the SDK's parsed copy. */
|
|
109
225
|
export async function listTools(connection, options = {}) {
|
|
110
226
|
const mark = connection.wire.responses.length;
|
|
111
227
|
let clientError;
|
|
228
|
+
let caught;
|
|
112
229
|
try {
|
|
113
230
|
await connection.client.listTools(undefined, { cacheMode: 'bypass', timeout: options.timeoutMs });
|
|
114
231
|
}
|
|
115
232
|
catch (error) {
|
|
233
|
+
caught = error;
|
|
116
234
|
clientError = error instanceof Error ? error.message : String(error);
|
|
117
235
|
}
|
|
118
236
|
const responses = connection.wire.since(mark, 'tools/list');
|
|
119
237
|
const pages = responses.flatMap((r) => (r.result ? [r.result] : []));
|
|
120
|
-
if (clientError && pages.length === 0)
|
|
121
|
-
|
|
238
|
+
if (clientError && pages.length === 0) {
|
|
239
|
+
// Let stderr arrive first, as for connect: the reason is usually its last lines.
|
|
240
|
+
await connection.stderrSettled?.();
|
|
241
|
+
throw connection.explain?.(caught, 'tools/list') ?? new Error(clientError);
|
|
242
|
+
}
|
|
122
243
|
const first = pages[0] ?? {};
|
|
123
244
|
return {
|
|
124
245
|
tools: pages.flatMap((p) => (Array.isArray(p.tools) ? p.tools : [])),
|
package/dist/diff.d.ts
CHANGED
|
@@ -3,8 +3,14 @@ export type ChangeClass = 'breaking' | 'minor' | 'notice';
|
|
|
3
3
|
export type Bump = 'major' | 'minor' | 'patch' | 'none';
|
|
4
4
|
export interface DiffFinding extends Finding {
|
|
5
5
|
class?: ChangeClass;
|
|
6
|
-
/** Every place one change was found, when it's reached from several (a shared definition). */
|
|
6
|
+
/** Every place one change was found, when it's reached from several (a shared definition, or several tools). */
|
|
7
7
|
places?: string[];
|
|
8
|
+
/**
|
|
9
|
+
* Every tool the change was found in, when it's the same change in several
|
|
10
|
+
* (same rule, same parameter path, same schema before and after). `tool` is
|
|
11
|
+
* then left out. Counts are per change: one finding, however many tools.
|
|
12
|
+
*/
|
|
13
|
+
tools?: string[];
|
|
8
14
|
}
|
|
9
15
|
interface DiffRule {
|
|
10
16
|
severity: Severity;
|
|
@@ -24,13 +30,29 @@ export interface TokenChange {
|
|
|
24
30
|
delta: number;
|
|
25
31
|
}[];
|
|
26
32
|
}
|
|
33
|
+
/** Per class: how many changes (findings), and how many tools they touch. */
|
|
34
|
+
export type ClassCounts = Record<ChangeClass, {
|
|
35
|
+
changes: number;
|
|
36
|
+
tools: number;
|
|
37
|
+
}>;
|
|
27
38
|
export interface DiffResult {
|
|
28
39
|
before: Menu['server'];
|
|
29
40
|
after: Menu['server'];
|
|
30
41
|
findings: DiffFinding[];
|
|
31
42
|
tokens: TokenChange;
|
|
43
|
+
/** Breaking, minor and notice changes: counted per change, with the tools they touch. */
|
|
44
|
+
classes: ClassCounts;
|
|
45
|
+
/** What the changes call for by semver: breaking → major, new → minor, other → patch. */
|
|
32
46
|
suggestedBump: Bump;
|
|
47
|
+
/**
|
|
48
|
+
* The smallest bump the release line needs (with a release to check): the
|
|
49
|
+
* suggested one, one step lower under 1.0.0 (npm's caret: ^0.2.3 accepts
|
|
50
|
+
* 0.2.x), none under 0.1.0.
|
|
51
|
+
*/
|
|
52
|
+
requiredBump?: Bump;
|
|
33
53
|
actualBump?: Bump;
|
|
54
|
+
/** When the actual bump is too small: the version to release instead (2.0.0). */
|
|
55
|
+
releaseAs?: string;
|
|
34
56
|
/** The versions the bump was checked against, and where they came from. */
|
|
35
57
|
release?: {
|
|
36
58
|
before: string;
|