toolmenu 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +76 -20
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +53 -17
  6. package/dist/auto.d.ts +70 -6
  7. package/dist/auto.js +248 -17
  8. package/dist/breakdown.d.ts +2 -0
  9. package/dist/breakdown.js +62 -19
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +80 -170
  13. package/dist/config.d.ts +5 -1
  14. package/dist/config.js +14 -0
  15. package/dist/connect.d.ts +4 -0
  16. package/dist/connect.js +132 -11
  17. package/dist/diff.d.ts +23 -1
  18. package/dist/diff.js +526 -114
  19. package/dist/difference.d.ts +2 -0
  20. package/dist/difference.js +13 -12
  21. package/dist/explain.d.ts +103 -0
  22. package/dist/explain.js +434 -0
  23. package/dist/failures.d.ts +9 -0
  24. package/dist/failures.js +17 -0
  25. package/dist/help.d.ts +9 -0
  26. package/dist/help.js +198 -0
  27. package/dist/history.d.ts +13 -0
  28. package/dist/history.js +86 -6
  29. package/dist/init.d.ts +92 -2
  30. package/dist/init.js +507 -20
  31. package/dist/menu.js +10 -2
  32. package/dist/options.d.ts +135 -0
  33. package/dist/options.js +48 -0
  34. package/dist/report.d.ts +1 -1
  35. package/dist/report.js +229 -99
  36. package/dist/rules/description.d.ts +7 -2
  37. package/dist/rules/description.js +85 -42
  38. package/dist/rules/determinism.d.ts +3 -1
  39. package/dist/rules/determinism.js +145 -11
  40. package/dist/rules/ids.d.ts +1 -1
  41. package/dist/rules/ids.js +9 -4
  42. package/dist/rules/index.d.ts +6 -0
  43. package/dist/rules/index.js +12 -4
  44. package/dist/rules/naming.d.ts +1 -1
  45. package/dist/rules/naming.js +66 -19
  46. package/dist/rules/rule.d.ts +18 -0
  47. package/dist/rules/rule.js +25 -1
  48. package/dist/rules/spec.js +33 -9
  49. package/dist/rules/write.d.ts +1 -1
  50. package/dist/rules/write.js +42 -10
  51. package/dist/session.d.ts +18 -8
  52. package/dist/session.js +484 -92
  53. package/dist/snapshot.d.ts +2 -2
  54. package/dist/snapshot.js +17 -2
  55. package/dist/types.d.ts +14 -0
  56. package/dist/words.d.ts +5 -0
  57. package/dist/words.js +14 -0
  58. package/package.json +1 -1
package/dist/cli.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { mkdir, writeFile } from 'node:fs/promises';
3
- import { dirname, join } from 'node:path';
2
+ import { writeFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
4
  import { parseArgs } from 'node:util';
5
5
  import { loadConfig } from './config.js';
6
6
  import { diffMenus } from './diff.js';
@@ -15,99 +15,13 @@ import { existsSync } from 'node:fs';
15
15
  import { loadRoutes } from './routes.js';
16
16
  import { snapshot } from './snapshot.js';
17
17
  import { connectPatiently, MAIN_SEED, probeMenu, refusedForTooMany, seeded } from './probe.js';
18
- import { autoScenario, scenarioYaml } from './auto.js';
18
+ import { autoScenario, loadValuesFile, parseAssumeReadOnly, parseValueFlag, scenarioYaml } from './auto.js';
19
19
  import { authDir, listLogins, login, logout } from './auth.js';
20
- import { detectProject, secretsNeeded, workflowYaml } from './init.js';
20
+ import { existingMessage, initReport, planInit, writeInit } from './init.js';
21
21
  import { SEVERITY_RANK } from './types.js';
22
22
  import { VERSION } from './version.js';
23
- const HELP = `toolmenu ${VERSION}
24
- Lint your MCP server's tool menu for changes that confuse agents or break caches.
25
-
26
- Usage:
27
- toolmenu init [options] -- <command> | <url> set up CI: a baseline and a workflow
28
- toolmenu snapshot [options] -- <command> [args...] stdio server
29
- toolmenu snapshot [options] <url> Streamable HTTP server
30
- toolmenu diff [options] <old.json> <new.json> compare two snapshots
31
- toolmenu history [options] <npm-package> snapshot and diff published versions
32
- toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
33
- toolmenu session --scenario <file> [options] -- <command> | <url>
34
- run a scripted session, watch the menu
35
- toolmenu session --auto [options] -- <command> | <url>
36
- the same, with steps built from the menu
37
-
38
- Commands:
39
- init snapshot the server into menu.json and write .github/workflows/toolmenu.yml
40
- snapshot establish the menu: list tools twice, write menu.json, run the menu rules
41
- diff compare releases: breaking changes, token change, semver bump
42
- session observe the menu changing while a scripted session runs
43
- history research release history: install, snapshot and diff published npm versions
44
- auth log in to an OAuth-protected server once; snapshot and session then use it
45
-
46
- Options:
47
- --out <path> where to write the menu (default: menu.json)
48
- --no-write don't write the menu file
49
- --routes <path> routes.yml: pin which words route to which tools
50
- --config <path> config file (default: toolmenu.config.json, if present)
51
- --format <fmt> text (default), json, github (annotations) or markdown (PR comments)
52
- --json same as --format json
53
- --fail-on <level> error (default), warn or info
54
- --header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
55
- --env <K=V> environment variable for a stdio server, repeatable. The server
56
- only gets a minimal environment (PATH, HOME, ...) plus these
57
- --no-auth don't use a stored OAuth login for this server
58
- --timeout <ms> per-request timeout (default: 30000)
59
- --catalog also read the operations behind a search tool (search and execute:
60
- discover, search_*_tools) and keep them in the menu file for diff
61
- --processes <n> server processes (stdio) or connections (HTTP) to compare,
62
- the main one included (default: 2; 1 opens no second one, for
63
- session's scope check either: for servers with one session per client)
64
- -h, --help show this help
65
- -v, --version show the version
66
-
67
- diff options:
68
- --release <old>..<new> the release versions (npm, git tag) to check the bump
69
- against, e.g. 1.4.0..1.5.0. Without it, the bump isn't
70
- checked: serverInfo.version is often not the release
71
- --server-version-is-release check the bump against serverInfo.version instead
72
-
73
- session options:
74
- --scenario <path> scenario.yml: the steps to run (list, call, wait_for)
75
- --plan print the steps without connecting or running anything
76
- --init write a starter scenario from the server's menu (to --scenario,
77
- default scenario.yml; never overwrites)
78
- --auto build the steps from the menu: every read-only tool whose
79
- required arguments the schema can fill (const, default,
80
- examples, enum, type), then the first call again
81
- --open-world with --auto, also call read-only tools marked openWorldHint:
82
- true (web search, fetch, scraping): they may cost API credits
83
- --max-calls <n> with --auto, at most n calls (default: 20)
84
- --save-scenario <p> with --auto, write the steps it ran as a scenario file
85
- --union-out <path> write every tool the session saw as a menu file, to commit as
86
- the baseline for diff (tools behind unlocks included)
87
-
88
- history options (best effort; installs and runs third-party code, so use a container):
89
- --versions <n|all> number of published versions to inspect (default: 10)
90
- --include-prereleases include versions like 1.2.0-beta.1
91
- --arg <value> argument for the server's bin, repeatable (use --arg=--flag for flags)
92
- --bin <name> which bin to run when the package has several
93
- --cmd <template> command instead of the bin; {bin} and {dir} are filled in
94
- --allow-scripts run install scripts (off by default)
95
- --install-timeout <ms> per-version install timeout (default: 180000)
96
- --csv <path> also write the dataset as CSV
97
- --keep-installs keep each version's install directory
98
- --out <dir> where menus and history.json go (default: toolmenu-history/<package>)
99
-
100
- init options:
101
- --with-session the workflow also runs session --auto (calls read-only tools)
102
-
103
- auth options:
104
- --port <n> loopback port for the login redirect (default: 33418)
105
- --scope <scopes> scopes to ask for (default: the server's)
106
- --client-id <id> a pre-registered client, for servers without dynamic registration
107
- --client-secret <s> its secret, if it has one
108
-
109
- Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
110
- `;
23
+ import { CLI_OPTIONS } from './options.js';
24
+ import { commandHelp, isCommand, overview, unknownCommand } from './help.js';
111
25
  class UsageError extends Error {
112
26
  }
113
27
  export async function main(argv) {
@@ -117,59 +31,26 @@ export async function main(argv) {
117
31
  const { values, positionals } = parseArgs({
118
32
  args: before,
119
33
  allowPositionals: true,
120
- options: {
121
- out: { type: 'string' },
122
- 'no-write': { type: 'boolean' },
123
- routes: { type: 'string' },
124
- config: { type: 'string' },
125
- format: { type: 'string' },
126
- json: { type: 'boolean' },
127
- 'fail-on': { type: 'string' },
128
- header: { type: 'string', multiple: true },
129
- env: { type: 'string', multiple: true },
130
- timeout: { type: 'string' },
131
- 'no-auth': { type: 'boolean' },
132
- port: { type: 'string' },
133
- scope: { type: 'string' },
134
- 'client-id': { type: 'string' },
135
- 'client-secret': { type: 'string' },
136
- processes: { type: 'string' },
137
- versions: { type: 'string' },
138
- release: { type: 'string' },
139
- 'server-version-is-release': { type: 'boolean' },
140
- 'include-prereleases': { type: 'boolean' },
141
- arg: { type: 'string', multiple: true },
142
- bin: { type: 'string' },
143
- cmd: { type: 'string' },
144
- 'allow-scripts': { type: 'boolean' },
145
- 'install-timeout': { type: 'string' },
146
- csv: { type: 'string' },
147
- 'keep-installs': { type: 'boolean' },
148
- scenario: { type: 'string' },
149
- plan: { type: 'boolean' },
150
- init: { type: 'boolean' },
151
- 'union-out': { type: 'string' },
152
- auto: { type: 'boolean' },
153
- 'with-session': { type: 'boolean' },
154
- catalog: { type: 'boolean' },
155
- 'open-world': { type: 'boolean' },
156
- 'max-calls': { type: 'string' },
157
- 'save-scenario': { type: 'string' },
158
- help: { type: 'boolean', short: 'h' },
159
- version: { type: 'boolean', short: 'v' },
160
- },
34
+ options: CLI_OPTIONS,
161
35
  });
162
36
  if (values.version) {
163
37
  process.stdout.write(`${VERSION}\n`);
164
38
  return 0;
165
39
  }
166
40
  const [sub, ...rest] = positionals;
41
+ // toolmenu help [command] reads like toolmenu [command] --help.
42
+ if (sub === 'help' && rest.length <= 1) {
43
+ if (rest[0] && !isCommand(rest[0]))
44
+ throw new UsageError(unknownCommand(rest[0]));
45
+ process.stdout.write(rest[0] && isCommand(rest[0]) ? commandHelp(rest[0]) : overview());
46
+ return 0;
47
+ }
48
+ if (sub && !isCommand(sub))
49
+ throw new UsageError(unknownCommand(sub));
167
50
  if (values.help || !sub) {
168
- process.stdout.write(HELP);
51
+ process.stdout.write(sub && isCommand(sub) ? commandHelp(sub) : overview());
169
52
  return sub || values.help ? 0 : 2;
170
53
  }
171
- if (!['init', 'snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
172
- throw new UsageError(`Unknown command "${sub}". Try --help.`);
173
54
  const format = (values.json ? 'json' : values.format ?? 'text');
174
55
  if (!['text', 'json', 'github', 'markdown'].includes(format))
175
56
  throw new UsageError(`--format must be text, json, github or markdown`);
@@ -232,7 +113,7 @@ export async function main(argv) {
232
113
  });
233
114
  if (values.csv)
234
115
  await writeFile(values.csv, historyCsv(result));
235
- process.stdout.write(formatHistory(result, format, outDir) + '\n');
116
+ process.stdout.write(formatHistory(result, format, outDir, values.csv) + '\n');
236
117
  return result.rows.length > 0 && result.rows.every((r) => r.status === 'failed') ? 1 : 0;
237
118
  }
238
119
  if (sub === 'session' && values.init) {
@@ -270,32 +151,13 @@ export async function main(argv) {
270
151
  return 0;
271
152
  }
272
153
  if (sub === 'init') {
273
- const workflow = join('.github', 'workflows', 'toolmenu.yml');
274
- const baseline = values.out ?? 'menu.json';
275
- for (const f of [workflow, baseline])
276
- if (existsSync(f))
277
- throw new UsageError(`${f} already exists; init never overwrites. Remove it, or set things up by hand (README: "In CI").`);
278
154
  const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
155
+ const plan = planInit({ cwd: process.cwd(), target, out: values.out, session: values['with-session'] });
156
+ if (plan.existing.length)
157
+ throw new UsageError(existingMessage(plan));
279
158
  const { menu, findings } = await snapshot(target, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
280
- await writeFile(baseline, JSON.stringify(menu, null, 2) + '\n');
281
- await mkdir(dirname(workflow), { recursive: true });
282
- const project = detectProject('.');
283
- await writeFile(workflow, workflowYaml({ target, project, session: values['with-session'], baseline }));
284
- const c = counts(findings);
285
- const secrets = secretsNeeded(target);
286
- const lines = [
287
- `toolmenu init ${menu.server.name ?? 'server'} · ${menu.tools.length} tools · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
288
- ` wrote ${baseline}: the baseline every pull request is compared with`,
289
- ` wrote ${workflow}${project.kind === 'unknown' ? ' (fill in the TODO: how CI builds your server)' : ` (${project.kind} project)`}`,
290
- ` today's menu: ${c.error} errors, ${c.warn} warnings, ${c.info} info (run toolmenu snapshot to see them)`,
291
- ...(secrets.length ? ['', `Add these repository secrets (Settings → Secrets and variables → Actions): ${secrets.join(', ')}`] : []),
292
- '',
293
- 'Next:',
294
- ` git add ${baseline} ${workflow} && git commit -m "Check the MCP tool menu on every PR"`,
295
- ' Then open a pull request: toolmenu comments on it. Refresh the baseline when a change is intended:',
296
- ` toolmenu snapshot --out ${baseline} ${target.kind === 'stdio' ? '-- ' + [target.command, ...target.args].join(' ') : target.url}`,
297
- ];
298
- process.stdout.write(lines.join('\n') + '\n');
159
+ await writeInit(plan, JSON.stringify(menu, null, 2) + '\n');
160
+ process.stdout.write(initReport(plan, { server: menu.server.name, version: menu.server.version, tools: menu.tools.length, tokens: menu.totalTokens, counts: counts(findings) }, target));
299
161
  return 0;
300
162
  }
301
163
  if (sub === 'session' && values.auto) {
@@ -307,20 +169,36 @@ export async function main(argv) {
307
169
  const savePath = values['save-scenario'];
308
170
  if (savePath && existsSync(savePath))
309
171
  throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
172
+ // Values and vouched-for tools are checked before anything starts.
173
+ let given = {};
174
+ let assumeReadOnly = [];
175
+ try {
176
+ if (values['values-file'])
177
+ given = await loadValuesFile(values['values-file']);
178
+ for (const v of values.value ?? []) {
179
+ const [key, value] = parseValueFlag(v);
180
+ given[key] = value;
181
+ }
182
+ assumeReadOnly = parseAssumeReadOnly(values['assume-read-only'] ?? []);
183
+ }
184
+ catch (error) {
185
+ throw new UsageError(error instanceof Error ? error.message : String(error));
186
+ }
310
187
  const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
311
188
  const waited = { ms: 0 };
312
189
  const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs, undefined, waited).catch((error) => {
313
190
  throw refusedForTooMany(error, waited.ms);
314
191
  });
315
- const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
192
+ const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls, values: given, assumeReadOnly });
316
193
  if (savePath)
317
194
  await writeFile(savePath, scenarioYaml(plan, menu.server.name));
318
195
  if (values.plan) {
319
196
  process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
320
197
  return 0;
321
198
  }
322
- const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto', unionOut: !!values['union-out'], auto: { called: plan.called, skipped: plan.skipped } });
323
- result.auto = { called: plan.called, skipped: plan.skipped };
199
+ const auto = { called: plan.called, skipped: plan.skipped, assumed: plan.assumed, withValues: plan.withValues, ignored: plan.ignored, maxCalls };
200
+ const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto', unionOut: !!values['union-out'], auto });
201
+ result.auto ??= auto;
324
202
  if (values['union-out'])
325
203
  await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
326
204
  const output = formatSession(result, format);
@@ -329,6 +207,8 @@ export async function main(argv) {
329
207
  return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
330
208
  }
331
209
  if (sub === 'session') {
210
+ if (values.value || values['values-file'] || values['assume-read-only'])
211
+ throw new UsageError('--value, --values-file and --assume-read-only go with --auto. In a scenario, write the values into the steps\' args, and list tools under assume_read_only.');
332
212
  if (!values.scenario)
333
213
  throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
334
214
  let scenario;
@@ -384,17 +264,20 @@ async function authCommand(args, values) {
384
264
  const [action, url, ...extra] = args;
385
265
  if (action === 'list') {
386
266
  const logins = await listLogins();
387
- process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
267
+ const width = Math.max(0, ...logins.map((l) => l.serverUrl.length));
268
+ process.stdout.write(logins.length
269
+ ? [`${logins.length} login${logins.length === 1 ? '' : 's'}, stored in ${authDir()}:`, ...logins.map((l) => ` ${l.serverUrl.padEnd(width)} issuer ${l.issuer ?? 'not recorded'} · saved ${l.savedAt?.slice(0, 10) ?? '?'}`)].join('\n') + '\n'
270
+ : `No logins stored in ${authDir()}.\n→ Next: toolmenu auth login <url> for a server that asks for OAuth.\n`);
388
271
  return 0;
389
272
  }
390
273
  if (action !== 'login' && action !== 'logout')
391
- throw new UsageError('auth takes login <url>, logout <url> or list.');
274
+ throw new UsageError(`${action ? `auth has no action "${action}"` : 'auth needs an action'}: login <url>, logout <url> or list. See toolmenu auth --help.`);
392
275
  if (!url || extra.length)
393
276
  throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
394
277
  if (!/^https?:\/\//.test(url))
395
278
  throw new UsageError(`"${url}" isn't an http(s) URL.`);
396
279
  if (action === 'logout') {
397
- process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
280
+ process.stdout.write((await logout(url)) ? `Logged out of ${url}: its tokens and client registration are removed from ${authDir()}.\n` : `No login stored for ${url} (in ${authDir()}), so nothing to remove. toolmenu auth list shows the stored ones.\n`);
398
281
  return 0;
399
282
  }
400
283
  const port = values.port ? Number(values.port) : undefined;
@@ -403,7 +286,7 @@ async function authCommand(args, values) {
403
286
  if (values['client-secret'] && !values['client-id'])
404
287
  throw new UsageError('--client-secret goes with --client-id');
405
288
  const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
406
- process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
289
+ process.stdout.write(`Logged in to ${result.name ?? url} (${url}): it lists ${result.tools} tool${result.tools === 1 ? '' : 's'} with this login.\nsnapshot and session use it from now on (--no-auth to skip it); stored in ${authDir()}.\n`);
407
290
  return 0;
408
291
  }
409
292
  function parsePairs(items, separator, flag) {
@@ -434,10 +317,37 @@ for (const method of ['log', 'info', 'debug']) {
434
317
  console[method] = (...args) => console.error(...args);
435
318
  }
436
319
  main(process.argv.slice(2)).then(exitWhenFlushed, (error) => {
437
- const message = error instanceof Error ? error.message : String(error);
438
- process.stderr.write(`toolmenu: ${message}\n`);
320
+ process.stderr.write(`toolmenu: ${failureMessage(error)}\n`);
439
321
  exitWhenFlushed(2);
440
322
  });
323
+ /** What stopped the command. A mistyped option gets the nearest real one. */
324
+ function failureMessage(error) {
325
+ const message = error instanceof Error ? error.message : String(error);
326
+ const code = error?.code;
327
+ if (code === 'ERR_PARSE_ARGS_UNKNOWN_OPTION') {
328
+ const option = /Unknown option '([^']+)'/.exec(message)?.[1] ?? '';
329
+ const known = Object.keys(CLI_OPTIONS).map((name) => `--${name}`);
330
+ const guess = known.map((k) => [k, editDistance(option, k)]).sort((a, b) => a[1] - b[1])[0];
331
+ const hint = guess && guess[1] <= 2 ? ` Did you mean ${guess[0]}?` : '';
332
+ return `Unknown option ${option}.${hint}\n → Next: toolmenu --help lists every option. (A server's own flags go after "--".)`;
333
+ }
334
+ if (typeof code === 'string' && code.startsWith('ERR_PARSE_ARGS_'))
335
+ return `${message}\n → Next: toolmenu --help lists every option.`;
336
+ return message;
337
+ }
338
+ function editDistance(a, b) {
339
+ const row = Array.from({ length: b.length + 1 }, (_, i) => i);
340
+ for (let i = 1; i <= a.length; i++) {
341
+ let diagonal = row[0];
342
+ row[0] = i;
343
+ for (let j = 1; j <= b.length; j++) {
344
+ const above = row[j];
345
+ row[j] = Math.min(row[j] + 1, row[j - 1] + 1, diagonal + (a[i - 1] === b[j - 1] ? 0 : 1));
346
+ diagonal = above;
347
+ }
348
+ }
349
+ return row[b.length];
350
+ }
441
351
  function parseRelease(value) {
442
352
  const m = /^(.+?)\.\.(.+)$/.exec(value);
443
353
  if (!m)
package/dist/config.d.ts CHANGED
@@ -16,9 +16,13 @@ export interface Config {
16
16
  fullDescriptions?: string[];
17
17
  /** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
18
18
  processes?: number;
19
- /** snapshot --catalog: which search tool and which queries (default: detected, derived from the menu). */
19
+ /**
20
+ * snapshot --catalog: which search tool and which queries (default: detected,
21
+ * derived from the menu), or which command routers to list (default: detected).
22
+ */
20
23
  catalog?: {
21
24
  tool?: string;
25
+ routers?: string[];
22
26
  queries?: string[];
23
27
  pauseMs?: number;
24
28
  crawl?: boolean;
package/dist/config.js CHANGED
@@ -26,5 +26,19 @@ export async function loadConfig(path) {
26
26
  if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
27
27
  throw new Error(`${file}: processes must be a whole number, 1 or more`);
28
28
  }
29
+ const catalog = config.catalog;
30
+ if (catalog !== undefined) {
31
+ const names = (v) => Array.isArray(v) && v.length > 0 && v.every((x) => typeof x === 'string' && x.length > 0);
32
+ if (typeof catalog !== 'object' || catalog === null || Array.isArray(catalog))
33
+ throw new Error(`${file}: catalog must be an object, like { "routers": ["keyvault"] } or { "tool": "search_tools" }`);
34
+ if (catalog.tool !== undefined && !(typeof catalog.tool === 'string' && catalog.tool))
35
+ throw new Error(`${file}: catalog.tool must be the name of the search tool, a string`);
36
+ if (catalog.routers !== undefined && !names(catalog.routers))
37
+ throw new Error(`${file}: catalog.routers must be a list of tool names, like ["keyvault", "storage"]`);
38
+ if (catalog.tool !== undefined && catalog.routers !== undefined)
39
+ throw new Error(`${file}: catalog names both a search tool (tool) and routers (routers); a menu file keeps one catalog, so keep the one the operations are behind`);
40
+ if (catalog.queries !== undefined && !names(catalog.queries))
41
+ throw new Error(`${file}: catalog.queries must be a list of search phrases, like ["list issues"]`);
42
+ }
29
43
  return config;
30
44
  }
package/dist/connect.d.ts CHANGED
@@ -47,6 +47,10 @@ export interface Connection {
47
47
  usedAuth: boolean;
48
48
  stderr: () => string;
49
49
  close: () => Promise<void>;
50
+ /** An error from a request on this connection, said with what the connection saw (exit code, stderr, stray stdout). */
51
+ explain?: (error: unknown, stage: string) => unknown;
52
+ /** Resolves once a server that just failed a request has had its say on stderr (all of it, if it exited). */
53
+ stderrSettled?: () => Promise<void>;
50
54
  }
51
55
  export interface ConnectOptions {
52
56
  timeoutMs?: number;
package/dist/connect.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
2
2
  import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
3
3
  import { hasLogin, StoredOAuthProvider } from './auth.js';
4
+ import { explain, Trace } from './explain.js';
4
5
  import { eraOf } from './menu.js';
5
6
  import { VERSION } from './version.js';
6
7
  /** Every response the server sent, matched to the request method, as it came off the wire. */
@@ -50,6 +51,8 @@ export async function connect(target, options = {}) {
50
51
  let transport;
51
52
  let stderrText = '';
52
53
  let oauth = false;
54
+ const trace = new Trace();
55
+ let stdioStderr = null;
53
56
  if (target.kind === 'stdio') {
54
57
  const stdio = new StdioClientTransport({
55
58
  command: target.command,
@@ -59,8 +62,12 @@ export async function connect(target, options = {}) {
59
62
  stderr: 'pipe',
60
63
  });
61
64
  stdio.stderr?.on('data', (chunk) => {
62
- stderrText = (stderrText + chunk.toString()).slice(-4000);
65
+ const text = chunk.toString();
66
+ stderrText = (stderrText + text).slice(-4000);
67
+ trace.stderr(text);
63
68
  });
69
+ stdioStderr = stdio.stderr;
70
+ watchProcess(stdio, trace);
64
71
  transport = stdio;
65
72
  }
66
73
  else {
@@ -70,23 +77,33 @@ export async function connect(target, options = {}) {
70
77
  oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
71
78
  transport = new StreamableHTTPClientTransport(new URL(target.url), {
72
79
  requestInit: { headers: target.headers ?? {} },
80
+ fetch: tracedFetch(trace),
73
81
  ...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
74
82
  });
75
83
  }
84
+ const context = { target, timeoutMs: options.timeoutMs, oauth };
76
85
  try {
77
86
  await client.connect(transport, { timeout: options.timeoutMs });
78
87
  }
79
88
  catch (error) {
80
- // Give stderr a moment to arrive: it usually says why the server exited.
81
- await new Promise((resolve) => setTimeout(resolve, 100));
82
- const stderr = stderrText.trim();
83
- if (stderr && error instanceof Error)
84
- error.message += `\nserver stderr:\n${stderr}`;
85
- if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
86
- error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
89
+ // What failed is fixed now: the exit that follows toolmenu's own SIGTERM isn't the server's doing.
90
+ trace.failedAt = Date.now();
91
+ // A server that never answered is stopped now, not after close()'s grace period for a clean exit.
92
+ if (trace.process && !trace.exit) {
93
+ trace.stopped = true;
94
+ trace.process.kill('SIGTERM');
95
+ }
96
+ // Read stderr to its end: the last lines usually say why the server exited. A
97
+ // process that has exited closes its end of the pipe, so wait for all of it (a
98
+ // loaded CI runner took over a second for 300 lines, and the reason is last);
99
+ // one toolmenu just stopped gets a moment to exit first.
100
+ if (target.kind === 'stdio') {
101
+ if (!trace.exit)
102
+ await exited(trace, 1000);
103
+ await stderrEnded(stdioStderr, trace.exit ? STDERR_AFTER_EXIT_MS : 1000);
87
104
  }
88
105
  await client.close().catch(() => { });
89
- throw error;
106
+ throw explain(error, trace, context);
90
107
  }
91
108
  const wire = new WireLog();
92
109
  wire.tap(transport);
@@ -103,22 +120,126 @@ export async function connect(target, options = {}) {
103
120
  (oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
104
121
  stderr: () => stderrText,
105
122
  close: () => client.close(),
123
+ explain: (error, stage) => explain(error, trace, { ...context, stage }),
124
+ stderrSettled: async () => {
125
+ if (target.kind !== 'stdio')
126
+ return;
127
+ // A server still running gets a moment to exit; one that exited is read to the end of its stderr.
128
+ if (!trace.exit)
129
+ await exited(trace, 200);
130
+ if (trace.exit)
131
+ await stderrEnded(stdioStderr, STDERR_AFTER_EXIT_MS);
132
+ },
133
+ };
134
+ }
135
+ /** How long to wait for an exited server's stderr to end: its pipe is closing, so this is only a ceiling for a stuck one. */
136
+ const STDERR_AFTER_EXIT_MS = 10_000;
137
+ /** Resolves when the traced process has exited, or after `maxMs`. */
138
+ function exited(trace, maxMs) {
139
+ const child = trace.process;
140
+ if (!child || trace.exit)
141
+ return Promise.resolve();
142
+ return new Promise((resolve) => {
143
+ const timer = setTimeout(resolve, maxMs);
144
+ child.once('exit', () => {
145
+ clearTimeout(timer);
146
+ // The exit listener that records trace.exit runs first: it was added at spawn.
147
+ resolve();
148
+ });
149
+ });
150
+ }
151
+ /** Resolves when `stream` has ended, or after `maxMs`. */
152
+ function stderrEnded(stream, maxMs) {
153
+ if (!stream || maxMs <= 0 || stream.readableEnded)
154
+ return Promise.resolve();
155
+ return new Promise((resolve) => {
156
+ const timer = setTimeout(resolve, maxMs);
157
+ stream.once('end', () => {
158
+ clearTimeout(timer);
159
+ resolve();
160
+ });
161
+ });
162
+ }
163
+ /**
164
+ * Watch the stdio process toolmenu keeps (not the throwaway one the SDK starts
165
+ * for the server/discover probe): when it started, what it was asked, what it
166
+ * answered, its exit, and stdout lines that aren't JSON-RPC. The hooks go on this
167
+ * instance only: the SDK clones the transport's class for the probe.
168
+ */
169
+ function watchProcess(stdio, trace) {
170
+ const start = stdio.start.bind(stdio);
171
+ stdio.start = async () => {
172
+ trace.sessionStartedAt = Date.now();
173
+ const onmessage = stdio.onmessage;
174
+ const ids = new Map();
175
+ const t = stdio;
176
+ t.onmessage = (message, extra) => {
177
+ if (!('method' in message) && 'id' in message && message.id !== undefined) {
178
+ const method = ids.get(message.id);
179
+ if (method)
180
+ trace.answered.add(method);
181
+ }
182
+ onmessage?.(message, extra);
183
+ };
184
+ const send = t.send.bind(stdio);
185
+ t.send = (message, options) => {
186
+ if ('method' in message && 'id' in message && message.id !== undefined) {
187
+ ids.set(message.id, message.method);
188
+ trace.send(message.method);
189
+ }
190
+ return send(message, options);
191
+ };
192
+ await start();
193
+ const child = stdio._process;
194
+ if (child) {
195
+ trace.process = child;
196
+ child.once('exit', (code, signal) => (trace.exit = { code, signal, afterStop: trace.stopped }));
197
+ child.stdout?.on('data', (chunk) => trace.stdout(chunk.toString()));
198
+ }
199
+ };
200
+ }
201
+ /** fetch, noting each JSON-RPC request's method and the answer's status, content type and final URL. */
202
+ function tracedFetch(trace) {
203
+ return async (input, init) => {
204
+ let method;
205
+ if (typeof init?.body === 'string') {
206
+ try {
207
+ const body = JSON.parse(init.body);
208
+ if (typeof body.method === 'string' && body.id !== undefined)
209
+ method = body.method;
210
+ }
211
+ catch {
212
+ // Not JSON: not a request to trace.
213
+ }
214
+ }
215
+ if (method)
216
+ trace.send(method);
217
+ const response = await fetch(input, init);
218
+ if (method) {
219
+ trace.http.set(method, { status: response.status, statusText: response.statusText, contentType: response.headers.get('content-type') ?? '', url: response.url });
220
+ }
221
+ return response;
106
222
  };
107
223
  }
108
224
  /** One full `tools/list`, bypassing the SDK's cache, read from the wire rather than the SDK's parsed copy. */
109
225
  export async function listTools(connection, options = {}) {
110
226
  const mark = connection.wire.responses.length;
111
227
  let clientError;
228
+ let caught;
112
229
  try {
113
230
  await connection.client.listTools(undefined, { cacheMode: 'bypass', timeout: options.timeoutMs });
114
231
  }
115
232
  catch (error) {
233
+ caught = error;
116
234
  clientError = error instanceof Error ? error.message : String(error);
117
235
  }
118
236
  const responses = connection.wire.since(mark, 'tools/list');
119
237
  const pages = responses.flatMap((r) => (r.result ? [r.result] : []));
120
- if (clientError && pages.length === 0)
121
- throw new Error(clientError);
238
+ if (clientError && pages.length === 0) {
239
+ // Let stderr arrive first, as for connect: the reason is usually its last lines.
240
+ await connection.stderrSettled?.();
241
+ throw connection.explain?.(caught, 'tools/list') ?? new Error(clientError);
242
+ }
122
243
  const first = pages[0] ?? {};
123
244
  return {
124
245
  tools: pages.flatMap((p) => (Array.isArray(p.tools) ? p.tools : [])),
package/dist/diff.d.ts CHANGED
@@ -3,8 +3,14 @@ export type ChangeClass = 'breaking' | 'minor' | 'notice';
3
3
  export type Bump = 'major' | 'minor' | 'patch' | 'none';
4
4
  export interface DiffFinding extends Finding {
5
5
  class?: ChangeClass;
6
- /** Every place one change was found, when it's reached from several (a shared definition). */
6
+ /** Every place one change was found, when it's reached from several (a shared definition, or several tools). */
7
7
  places?: string[];
8
+ /**
9
+ * Every tool the change was found in, when it's the same change in several
10
+ * (same rule, same parameter path, same schema before and after). `tool` is
11
+ * then left out. Counts are per change: one finding, however many tools.
12
+ */
13
+ tools?: string[];
8
14
  }
9
15
  interface DiffRule {
10
16
  severity: Severity;
@@ -24,13 +30,29 @@ export interface TokenChange {
24
30
  delta: number;
25
31
  }[];
26
32
  }
33
+ /** Per class: how many changes (findings), and how many tools they touch. */
34
+ export type ClassCounts = Record<ChangeClass, {
35
+ changes: number;
36
+ tools: number;
37
+ }>;
27
38
  export interface DiffResult {
28
39
  before: Menu['server'];
29
40
  after: Menu['server'];
30
41
  findings: DiffFinding[];
31
42
  tokens: TokenChange;
43
+ /** Breaking, minor and notice changes: counted per change, with the tools they touch. */
44
+ classes: ClassCounts;
45
+ /** What the changes call for by semver: breaking → major, new → minor, other → patch. */
32
46
  suggestedBump: Bump;
47
+ /**
48
+ * The smallest bump the release line needs (with a release to check): the
49
+ * suggested one, one step lower under 1.0.0 (npm's caret: ^0.2.3 accepts
50
+ * 0.2.x), none under 0.1.0.
51
+ */
52
+ requiredBump?: Bump;
33
53
  actualBump?: Bump;
54
+ /** When the actual bump is too small: the version to release instead (2.0.0). */
55
+ releaseAs?: string;
34
56
  /** The versions the bump was checked against, and where they came from. */
35
57
  release?: {
36
58
  before: string;