toolmenu 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -1,37 +1,46 @@
1
1
  #!/usr/bin/env node
2
- import { writeFile } from 'node:fs/promises';
3
- import { join } from 'node:path';
2
+ import { mkdir, writeFile } from 'node:fs/promises';
3
+ import { dirname, join } from 'node:path';
4
4
  import { parseArgs } from 'node:util';
5
5
  import { loadConfig } from './config.js';
6
6
  import { diffMenus } from './diff.js';
7
7
  import { loadMenu } from './menu.js';
8
8
  import { history, historyCsv } from './history.js';
9
- import { formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
9
+ import { counts, formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
10
10
  import { loadScenario, session, starterScenario } from './session.js';
11
11
  import { connect, listTools } from './connect.js';
12
12
  import { buildMenu } from './menu.js';
13
13
  import { existsSync } from 'node:fs';
14
14
  import { loadRoutes } from './routes.js';
15
15
  import { snapshot } from './snapshot.js';
16
- import { MAIN_SEED, seeded } from './probe.js';
16
+ import { MAIN_SEED, probeMenu, seeded } from './probe.js';
17
+ import { autoScenario, scenarioYaml } from './auto.js';
18
+ import { authDir, listLogins, login, logout } from './auth.js';
19
+ import { detectProject, secretsNeeded, workflowYaml } from './init.js';
17
20
  import { SEVERITY_RANK } from './types.js';
18
21
  import { VERSION } from './version.js';
19
22
  const HELP = `toolmenu ${VERSION}
20
23
  Lint your MCP server's tool menu for changes that confuse agents or break caches.
21
24
 
22
25
  Usage:
26
+ toolmenu init [options] -- <command> | <url> set up CI: a baseline and a workflow
23
27
  toolmenu snapshot [options] -- <command> [args...] stdio server
24
28
  toolmenu snapshot [options] <url> Streamable HTTP server
25
29
  toolmenu diff [options] <old.json> <new.json> compare two snapshots
26
30
  toolmenu history [options] <npm-package> snapshot and diff published versions
31
+ toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
27
32
  toolmenu session --scenario <file> [options] -- <command> | <url>
28
33
  run a scripted session, watch the menu
34
+ toolmenu session --auto [options] -- <command> | <url>
35
+ the same, with steps built from the menu
29
36
 
30
37
  Commands:
38
+ init snapshot the server into menu.json and write .github/workflows/toolmenu.yml
31
39
  snapshot establish the menu: list tools twice, write menu.json, run the menu rules
32
40
  diff compare releases: breaking changes, token change, semver bump
33
41
  session observe the menu changing while a scripted session runs
34
42
  history research release history: install, snapshot and diff published npm versions
43
+ auth log in to an OAuth-protected server once; snapshot and session then use it
35
44
 
36
45
  Options:
37
46
  --out <path> where to write the menu (default: menu.json)
@@ -44,7 +53,10 @@ Options:
44
53
  --header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
45
54
  --env <K=V> environment variable for a stdio server, repeatable. The server
46
55
  only gets a minimal environment (PATH, HOME, ...) plus these
56
+ --no-auth don't use a stored OAuth login for this server
47
57
  --timeout <ms> per-request timeout (default: 30000)
58
+ --catalog also read the operations behind a search tool (search and execute:
59
+ discover, search_*_tools) and keep them in the menu file for diff
48
60
  --processes <n> server processes (stdio) or connections (HTTP) to compare,
49
61
  the main one included (default: 2; 1 turns the check off)
50
62
  -h, --help show this help
@@ -61,6 +73,15 @@ session options:
61
73
  --plan print the steps without connecting or running anything
62
74
  --init write a starter scenario from the server's menu (to --scenario,
63
75
  default scenario.yml; never overwrites)
76
+ --auto build the steps from the menu: every read-only tool whose
77
+ required arguments the schema can fill (const, default,
78
+ examples, enum, type), then the first call again
79
+ --open-world with --auto, also call read-only tools marked openWorldHint:
80
+ true (web search, fetch, scraping): they may cost API credits
81
+ --max-calls <n> with --auto, at most n calls (default: 20)
82
+ --save-scenario <p> with --auto, write the steps it ran as a scenario file
83
+ --union-out <path> write every tool the session saw as a menu file, to commit as
84
+ the baseline for diff (tools behind unlocks included)
64
85
 
65
86
  history options (best effort; installs and runs third-party code, so use a container):
66
87
  --versions <n|all> number of published versions to inspect (default: 10)
@@ -74,6 +95,15 @@ history options (best effort; installs and runs third-party code, so use a conta
74
95
  --keep-installs keep each version's install directory
75
96
  --out <dir> where menus and history.json go (default: toolmenu-history/<package>)
76
97
 
98
+ init options:
99
+ --with-session the workflow also runs session --auto (calls read-only tools)
100
+
101
+ auth options:
102
+ --port <n> loopback port for the login redirect (default: 33418)
103
+ --scope <scopes> scopes to ask for (default: the server's)
104
+ --client-id <id> a pre-registered client, for servers without dynamic registration
105
+ --client-secret <s> its secret, if it has one
106
+
77
107
  Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
78
108
  `;
79
109
  class UsageError extends Error {
@@ -96,6 +126,11 @@ export async function main(argv) {
96
126
  header: { type: 'string', multiple: true },
97
127
  env: { type: 'string', multiple: true },
98
128
  timeout: { type: 'string' },
129
+ 'no-auth': { type: 'boolean' },
130
+ port: { type: 'string' },
131
+ scope: { type: 'string' },
132
+ 'client-id': { type: 'string' },
133
+ 'client-secret': { type: 'string' },
99
134
  processes: { type: 'string' },
100
135
  versions: { type: 'string' },
101
136
  release: { type: 'string' },
@@ -111,6 +146,13 @@ export async function main(argv) {
111
146
  scenario: { type: 'string' },
112
147
  plan: { type: 'boolean' },
113
148
  init: { type: 'boolean' },
149
+ 'union-out': { type: 'string' },
150
+ auto: { type: 'boolean' },
151
+ 'with-session': { type: 'boolean' },
152
+ catalog: { type: 'boolean' },
153
+ 'open-world': { type: 'boolean' },
154
+ 'max-calls': { type: 'string' },
155
+ 'save-scenario': { type: 'string' },
114
156
  help: { type: 'boolean', short: 'h' },
115
157
  version: { type: 'boolean', short: 'v' },
116
158
  },
@@ -124,7 +166,7 @@ export async function main(argv) {
124
166
  process.stdout.write(HELP);
125
167
  return sub || values.help ? 0 : 2;
126
168
  }
127
- if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
169
+ if (!['init', 'snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
128
170
  throw new UsageError(`Unknown command "${sub}". Try --help.`);
129
171
  const format = (values.json ? 'json' : values.format ?? 'text');
130
172
  if (!['text', 'json', 'github', 'markdown'].includes(format))
@@ -135,6 +177,8 @@ export async function main(argv) {
135
177
  const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
136
178
  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
137
179
  throw new UsageError(`--timeout must be a positive number of milliseconds`);
180
+ if (sub === 'auth')
181
+ return authCommand(rest, values);
138
182
  const config = await loadConfig(values.config);
139
183
  const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
140
184
  if (!Number.isInteger(processes) || processes < 1)
@@ -193,7 +237,7 @@ export async function main(argv) {
193
237
  const path = values.scenario ?? 'scenario.yml';
194
238
  if (existsSync(path))
195
239
  throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
196
- const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
240
+ const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
197
241
  const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
198
242
  try {
199
243
  const list = await listTools(conn, { timeoutMs });
@@ -206,6 +250,62 @@ export async function main(argv) {
206
250
  }
207
251
  return 0;
208
252
  }
253
+ if (sub === 'init') {
254
+ const workflow = join('.github', 'workflows', 'toolmenu.yml');
255
+ const baseline = values.out ?? 'menu.json';
256
+ for (const f of [workflow, baseline])
257
+ if (existsSync(f))
258
+ throw new UsageError(`${f} already exists; init never overwrites. Remove it, or set things up by hand (README: "In CI").`);
259
+ const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
260
+ const { menu, findings } = await snapshot(target, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
261
+ await writeFile(baseline, JSON.stringify(menu, null, 2) + '\n');
262
+ await mkdir(dirname(workflow), { recursive: true });
263
+ const project = detectProject('.');
264
+ await writeFile(workflow, workflowYaml({ target, project, session: values['with-session'], baseline }));
265
+ const c = counts(findings);
266
+ const secrets = secretsNeeded(target);
267
+ const lines = [
268
+ `toolmenu init ${menu.server.name ?? 'server'} · ${menu.tools.length} tools · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
269
+ ` wrote ${baseline}: the baseline every pull request is compared with`,
270
+ ` wrote ${workflow}${project.kind === 'unknown' ? ' (fill in the TODO: how CI builds your server)' : ` (${project.kind} project)`}`,
271
+ ` today's menu: ${c.error} errors, ${c.warn} warnings, ${c.info} info (run toolmenu snapshot to see them)`,
272
+ ...(secrets.length ? ['', `Add these repository secrets (Settings → Secrets and variables → Actions): ${secrets.join(', ')}`] : []),
273
+ '',
274
+ 'Next:',
275
+ ` git add ${baseline} ${workflow} && git commit -m "Check the MCP tool menu on every PR"`,
276
+ ' Then open a pull request: toolmenu comments on it. Refresh the baseline when a change is intended:',
277
+ ` toolmenu snapshot --out ${baseline} ${target.kind === 'stdio' ? '-- ' + [target.command, ...target.args].join(' ') : target.url}`,
278
+ ];
279
+ process.stdout.write(lines.join('\n') + '\n');
280
+ return 0;
281
+ }
282
+ if (sub === 'session' && values.auto) {
283
+ if (values.scenario)
284
+ throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
285
+ const maxCalls = values['max-calls'] ? Number(values['max-calls']) : 20;
286
+ if (!Number.isInteger(maxCalls) || maxCalls < 1)
287
+ throw new UsageError('--max-calls must be a whole number, 1 or more');
288
+ const savePath = values['save-scenario'];
289
+ if (savePath && existsSync(savePath))
290
+ throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
291
+ const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
292
+ const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
293
+ const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
294
+ if (savePath)
295
+ await writeFile(savePath, scenarioYaml(plan, menu.server.name));
296
+ if (values.plan) {
297
+ process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
298
+ return 0;
299
+ }
300
+ const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
301
+ result.auto = { called: plan.called, skipped: plan.skipped };
302
+ if (values['union-out'])
303
+ await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
304
+ const output = formatSession(result, format);
305
+ if (output)
306
+ process.stdout.write(output + '\n');
307
+ return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
308
+ }
209
309
  if (sub === 'session') {
210
310
  if (!values.scenario)
211
311
  throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
@@ -220,17 +320,19 @@ export async function main(argv) {
220
320
  process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
221
321
  return 0;
222
322
  }
223
- const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
323
+ const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
224
324
  const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
325
+ if (values['union-out'])
326
+ await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
225
327
  const output = formatSession(result, format);
226
328
  if (output)
227
329
  process.stdout.write(output + '\n');
228
330
  return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
229
331
  }
230
- const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
332
+ const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
231
333
  const routesPath = values.routes ?? config.routes;
232
334
  const routes = routesPath ? await loadRoutes(routesPath) : undefined;
233
- const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
335
+ const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
234
336
  const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
235
337
  if (outPath)
236
338
  await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
@@ -239,7 +341,7 @@ export async function main(argv) {
239
341
  process.stdout.write(output + '\n');
240
342
  return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
241
343
  }
242
- function parseTarget(positionals, command, headers, env) {
344
+ function parseTarget(positionals, command, headers, env, noAuth = false) {
243
345
  if (command) {
244
346
  if (command.length === 0)
245
347
  throw new UsageError('Nothing after "--": give the command that starts the server.');
@@ -254,7 +356,33 @@ function parseTarget(positionals, command, headers, env) {
254
356
  throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
255
357
  if (!/^https?:\/\//.test(url))
256
358
  throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
257
- return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
359
+ return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
360
+ }
361
+ async function authCommand(args, values) {
362
+ const [action, url, ...extra] = args;
363
+ if (action === 'list') {
364
+ const logins = await listLogins();
365
+ process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
366
+ return 0;
367
+ }
368
+ if (action !== 'login' && action !== 'logout')
369
+ throw new UsageError('auth takes login <url>, logout <url> or list.');
370
+ if (!url || extra.length)
371
+ throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
372
+ if (!/^https?:\/\//.test(url))
373
+ throw new UsageError(`"${url}" isn't an http(s) URL.`);
374
+ if (action === 'logout') {
375
+ process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
376
+ return 0;
377
+ }
378
+ const port = values.port ? Number(values.port) : undefined;
379
+ if (port !== undefined && !(Number.isInteger(port) && port > 0 && port < 65536))
380
+ throw new UsageError('--port must be a port number');
381
+ if (values['client-secret'] && !values['client-id'])
382
+ throw new UsageError('--client-secret goes with --client-id');
383
+ const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
384
+ process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
385
+ return 0;
258
386
  }
259
387
  function parsePairs(items, separator, flag) {
260
388
  const pairs = {};
package/dist/config.d.ts CHANGED
@@ -16,6 +16,12 @@ export interface Config {
16
16
  fullDescriptions?: string[];
17
17
  /** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
18
18
  processes?: number;
19
+ /** snapshot --catalog: which search tool and which queries (default: detected, derived from the menu). */
20
+ catalog?: {
21
+ tool?: string;
22
+ queries?: string[];
23
+ pauseMs?: number;
24
+ };
19
25
  }
20
26
  /** Optional: no config file is fine. */
21
27
  export declare function loadConfig(path?: string): Promise<Config>;
package/dist/connect.d.ts CHANGED
@@ -11,6 +11,7 @@ export type Target = {
11
11
  kind: 'http';
12
12
  url: string;
13
13
  headers?: Record<string, string>;
14
+ noAuth?: boolean;
14
15
  };
15
16
  export interface WireResponse {
16
17
  method: string;
package/dist/connect.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
2
2
  import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
3
+ import { hasLogin, StoredOAuthProvider } from './auth.js';
3
4
  import { eraOf } from './menu.js';
4
5
  import { VERSION } from './version.js';
5
6
  /** Every response the server sent, matched to the request method, as it came off the wire. */
@@ -48,6 +49,7 @@ export async function connect(target, options = {}) {
48
49
  const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
49
50
  let transport;
50
51
  let stderrText = '';
52
+ let oauth = false;
51
53
  if (target.kind === 'stdio') {
52
54
  const stdio = new StdioClientTransport({
53
55
  command: target.command,
@@ -62,8 +64,13 @@ export async function connect(target, options = {}) {
62
64
  transport = stdio;
63
65
  }
64
66
  else {
67
+ // A stored OAuth login (toolmenu auth login) is used unless the request brings
68
+ // its own Authorization header, or --no-auth.
69
+ const ownAuth = Object.keys(target.headers ?? {}).some((h) => h.toLowerCase() === 'authorization');
70
+ oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
65
71
  transport = new StreamableHTTPClientTransport(new URL(target.url), {
66
72
  requestInit: { headers: target.headers ?? {} },
73
+ ...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
67
74
  });
68
75
  }
69
76
  try {
@@ -75,6 +82,9 @@ export async function connect(target, options = {}) {
75
82
  const stderr = stderrText.trim();
76
83
  if (stderr && error instanceof Error)
77
84
  error.message += `\nserver stderr:\n${stderr}`;
85
+ if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
86
+ error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
87
+ }
78
88
  await client.close().catch(() => { });
79
89
  throw error;
80
90
  }
@@ -90,7 +100,7 @@ export async function connect(target, options = {}) {
90
100
  server: { name: server.name, version: server.version },
91
101
  capabilities: (client.getServerCapabilities() ?? {}),
92
102
  usedAuth: target.kind === 'http' &&
93
- Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h)),
103
+ (oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
94
104
  stderr: () => stderrText,
95
105
  close: () => client.close(),
96
106
  };
package/dist/diff.js CHANGED
@@ -24,6 +24,12 @@ export const DIFF_RULES = {
24
24
  'diff/version-bump': { severity: 'warn' },
25
25
  'diff/version-backwards': { severity: 'warn' },
26
26
  'diff/token-budget': { severity: 'error' },
27
+ // Operations behind a search tool (snapshot --catalog). A search returns its top
28
+ // matches, so one not found this time may be ranked out, not removed: a notice,
29
+ // no bump. Changes to operations found in both runs use the rules above.
30
+ 'diff/catalog-missing': { severity: 'info' },
31
+ 'diff/catalog-added': { severity: 'info' },
32
+ 'diff/catalog-queries': { severity: 'info' },
27
33
  };
28
34
  export function diffMenus(before, after, options = {}) {
29
35
  const raw = [];
@@ -62,6 +68,7 @@ export function diffMenus(before, after, options = {}) {
62
68
  message: `Tool order changed (${moved.map((m) => m.tool).join(', ')}). Only a notice between releases: prompt caches are short-lived, so a deploy costs roughly one rewrite. It matters within a session.`,
63
69
  });
64
70
  }
71
+ raw.push(...compareCatalogs(before.catalog, after.catalog, ignored));
65
72
  // Ignored tools leave the token counts and the budget too.
66
73
  const tokens = tokenChange(oldTools, newTools);
67
74
  if (options.tokenBudget !== undefined && tokens.after > options.tokenBudget) {
@@ -207,6 +214,42 @@ function compareTool(old, t) {
207
214
  }
208
215
  return out.concat(compareRest(old, t));
209
216
  }
217
+ /** Operations behind a search tool, compared like tools where both runs found them. */
218
+ function compareCatalogs(before, after, ignored) {
219
+ if (!before && !after)
220
+ return [];
221
+ if (!before || !after) {
222
+ return [{ rule: 'diff/catalog-queries', message: `Only the ${before ? 'older' : 'newer'} snapshot has a catalog (snapshot --catalog), so the operations behind ${(before ?? after).tool} weren't compared.` }];
223
+ }
224
+ const out = [];
225
+ const where = `behind ${after.tool}`;
226
+ if (JSON.stringify(before.queries) !== JSON.stringify(after.queries)) {
227
+ out.push({ rule: 'diff/catalog-queries', message: `The two catalogs were read with different queries, so operations found on one side only say little. Pin them with catalog.queries in the config.` });
228
+ }
229
+ const partial = [before, after].filter((c) => c.failed?.length);
230
+ if (partial.length) {
231
+ out.push({ rule: 'diff/catalog-queries', message: `${partial.length === 2 ? 'Both catalogs are' : `The ${partial[0] === before ? 'older' : 'newer'} catalog is`} partial: some queries failed (rate limits or errors), so operations found on one side only say even less.` });
232
+ }
233
+ const oldOps = new Map(before.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
234
+ const newOps = new Map(after.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
235
+ for (const [name, op] of newOps) {
236
+ const old = oldOps.get(name);
237
+ if (!old) {
238
+ out.push({ rule: 'diff/catalog-added', tool: name, message: `${name} (${where}) is new, or newly found by the same queries.` });
239
+ continue;
240
+ }
241
+ for (const r of compareTool(old, op))
242
+ out.push({ ...r, message: `${r.message.replace(/\.$/, '')} (an operation ${where}).` });
243
+ }
244
+ const missing = [...oldOps.keys()].filter((n) => !newOps.has(n));
245
+ if (missing.length) {
246
+ out.push({
247
+ rule: 'diff/catalog-missing',
248
+ message: `${missing.length} operation${missing.length === 1 ? '' : 's'} ${where} weren't returned by the same queries this time: removed, renamed, or ranked lower. Check before relying on them: ${missing.slice(0, 8).join(', ')}${missing.length > 8 ? ', …' : ''}.`,
249
+ });
250
+ }
251
+ return out;
252
+ }
210
253
  const TOOL_FIELDS = new Set(['name', 'description', 'inputSchema', 'outputSchema', 'annotations', 'tokens']);
211
254
  /** Output schema, annotations and any other fields (title, icons, _meta…). */
212
255
  function compareRest(old, t) {
package/dist/init.d.ts ADDED
@@ -0,0 +1,22 @@
1
+ import type { Target } from './connect.js';
2
+ export type ProjectKind = 'node' | 'python' | 'go' | 'rust' | 'unknown';
3
+ export interface Project {
4
+ kind: ProjectKind;
5
+ /** The steps that install and build the server in CI, as YAML list items. */
6
+ setup: string[];
7
+ }
8
+ /** What the repository in `dir` looks like, and how CI gets it ready to run. */
9
+ export declare function detectProject(dir: string): Project;
10
+ /** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
11
+ export declare function looksSecret(key: string): boolean;
12
+ export interface WorkflowOptions {
13
+ target: Target;
14
+ project: Project;
15
+ /** Also run `session --auto` (calls read-only tools with the job's credentials). */
16
+ session?: boolean;
17
+ baseline?: string;
18
+ }
19
+ /** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
20
+ export declare function workflowYaml(opts: WorkflowOptions): string;
21
+ /** The secret names the workflow refers to, to tell the user what to add. */
22
+ export declare function secretsNeeded(target: Target): string[];
package/dist/init.js ADDED
@@ -0,0 +1,105 @@
1
+ import { existsSync, readFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { VERSION } from './version.js';
4
+ /** What the repository in `dir` looks like, and how CI gets it ready to run. */
5
+ export function detectProject(dir) {
6
+ const has = (f) => existsSync(join(dir, f));
7
+ if (has('package.json')) {
8
+ let scripts = {};
9
+ try {
10
+ scripts = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')).scripts ?? {};
11
+ }
12
+ catch {
13
+ // an unreadable package.json still means node
14
+ }
15
+ const pm = has('pnpm-lock.yaml') ? 'pnpm' : has('yarn.lock') ? 'yarn' : 'npm';
16
+ const install = pm === 'pnpm' ? 'pnpm install --frozen-lockfile' : pm === 'yarn' ? 'yarn install --frozen-lockfile' : has('package-lock.json') ? 'npm ci' : 'npm install';
17
+ const run = pm === 'npm' ? 'npm run' : pm;
18
+ return {
19
+ kind: 'node',
20
+ setup: [
21
+ ...(pm === 'pnpm' ? ['- uses: pnpm/action-setup@v6'] : []),
22
+ '- uses: actions/setup-node@v7',
23
+ ` with: { node-version: 22${pm === 'npm' ? '' : `, cache: ${pm}`} }`,
24
+ `- run: ${install}${scripts.build ? ` && ${run} build` : ''}`,
25
+ ],
26
+ };
27
+ }
28
+ if (has('pyproject.toml') || has('requirements.txt')) {
29
+ const uv = has('uv.lock');
30
+ return {
31
+ kind: 'python',
32
+ setup: uv
33
+ ? ['- uses: astral-sh/setup-uv@v10', '- run: uv sync']
34
+ : ['- uses: actions/setup-python@v7', " with: { python-version: '3.12' }", `- run: pip install ${has('pyproject.toml') ? '.' : '-r requirements.txt'}`],
35
+ };
36
+ }
37
+ if (has('go.mod'))
38
+ return { kind: 'go', setup: ['- uses: actions/setup-go@v7', " with: { go-version-file: go.mod }", '- run: go build ./...'] };
39
+ if (has('Cargo.toml'))
40
+ return { kind: 'rust', setup: ['- run: cargo build --release'] };
41
+ return { kind: 'unknown', setup: ['# TODO: install and build your server here'] };
42
+ }
43
+ /** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
44
+ export function looksSecret(key) {
45
+ return /token|secret|password|passwd|api[_-]?key|(^|_)key$|credential|auth/i.test(key);
46
+ }
47
+ /** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
48
+ export function workflowYaml(opts) {
49
+ const t = opts.target;
50
+ const lines = [
51
+ '# Written by `toolmenu init`. On every pull request: snapshot the tool menu this',
52
+ '# change produces, diff it against the committed baseline, and post one comment.',
53
+ '# https://github.com/niksa90/toolmenu',
54
+ 'name: toolmenu',
55
+ 'on: pull_request',
56
+ 'permissions:',
57
+ ' contents: read',
58
+ ' pull-requests: write',
59
+ 'jobs:',
60
+ ' toolmenu:',
61
+ ' runs-on: ubuntu-latest',
62
+ ' steps:',
63
+ ' - uses: actions/checkout@v7',
64
+ ...(t.kind === 'stdio' ? opts.project.setup.map((l) => ` ${l}`) : []),
65
+ ` - uses: niksa90/toolmenu@v${VERSION}`,
66
+ ' with:',
67
+ ];
68
+ if (t.kind === 'stdio') {
69
+ lines.push(` command: ${yamlString([t.command, ...t.args].map(shellWord).join(' '))}`);
70
+ const env = Object.keys(t.env ?? {});
71
+ if (env.length) {
72
+ lines.push(' env: |');
73
+ for (const k of env)
74
+ lines.push(` ${k}=${looksSecret(k) ? `\${{ secrets.${k} }}` : t.env[k]}`);
75
+ }
76
+ }
77
+ else {
78
+ lines.push(` url: ${yamlString(t.url)} # checks the server at this URL, not this PR's code: see docs/github-action.md`);
79
+ const headers = Object.keys(t.headers ?? {});
80
+ if (headers.length) {
81
+ lines.push(' headers: |');
82
+ for (const h of headers)
83
+ lines.push(` ${h}: \${{ secrets.${secretName(h)} }}`);
84
+ }
85
+ }
86
+ lines.push(` baseline: ${opts.baseline ?? 'menu.json'}`);
87
+ if (opts.session)
88
+ lines.push(' scenario: auto # also call read-only tools and watch the menu');
89
+ return lines.join('\n') + '\n';
90
+ }
91
+ /** The secret names the workflow refers to, to tell the user what to add. */
92
+ export function secretsNeeded(target) {
93
+ if (target.kind === 'stdio')
94
+ return Object.keys(target.env ?? {}).filter(looksSecret);
95
+ return Object.keys(target.headers ?? {}).map(secretName);
96
+ }
97
+ function secretName(header) {
98
+ return 'MCP_' + header.toUpperCase().replace(/[^A-Z0-9]+/g, '_');
99
+ }
100
+ function shellWord(w) {
101
+ return /^[\w./:=@%+-]+$/.test(w) ? w : `'${w.replace(/'/g, `'\\''`)}'`;
102
+ }
103
+ function yamlString(s) {
104
+ return /^[\w./:@%+ =-]+$/.test(s) && !/^[-?:,[\]{}#&*!|>'"%@`]/.test(s) && !/: |\s#/.test(s) ? s : JSON.stringify(s);
105
+ }
package/dist/menu.js CHANGED
@@ -61,6 +61,8 @@ export async function loadMenu(path) {
61
61
  t.tokens = toolTokens(t);
62
62
  }
63
63
  menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
64
+ for (const op of menu.catalog?.operations ?? [])
65
+ op.tokens = toolTokens(op);
64
66
  menu.server ??= {};
65
67
  return menu;
66
68
  }
package/dist/report.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { stepLabel } from './session.js';
2
+ import { breakdown, breakdownLines } from './breakdown.js';
2
3
  const LABEL = { error: 'ERROR', warn: 'WARN ', info: 'INFO ' };
3
4
  export function counts(findings) {
4
5
  const c = { error: 0, warn: 0, info: 0 };
@@ -17,6 +18,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
17
18
  tools: menu.tools.length,
18
19
  totalTokens: menu.totalTokens,
19
20
  listMeta: menu.listMeta,
21
+ breakdown: breakdown(menu),
20
22
  written: outPath ?? null,
21
23
  counts: counts(findings),
22
24
  findings,
@@ -32,6 +34,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
32
34
  `Protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
33
35
  '',
34
36
  ...mdFindings(findings),
37
+ ...mdBreakdown(menu),
35
38
  ].join('\n');
36
39
  }
37
40
  const s = menu.server;
@@ -42,10 +45,19 @@ export function formatSnapshot(menu, findings, format, outPath) {
42
45
  if (outPath)
43
46
  lines.push(` wrote ${outPath}`);
44
47
  lines.push('');
48
+ const where = breakdownLines(breakdown(menu), menu.tools.length);
49
+ if (where.length)
50
+ lines.push(...where, '');
45
51
  lines.push(...findingLines(findings));
46
52
  lines.push(summaryLine(findings));
47
53
  return lines.join('\n');
48
54
  }
55
+ function mdBreakdown(menu) {
56
+ const where = breakdownLines(breakdown(menu), menu.tools.length);
57
+ if (!where.length)
58
+ return [];
59
+ return ['', `<details><summary>${where[0]}</summary>`, '', '```', ...where.slice(1).map((l) => l.trimStart()), '```', '', '</details>'];
60
+ }
49
61
  function findingLines(findings) {
50
62
  const lines = [];
51
63
  for (const f of findings) {
@@ -197,8 +209,11 @@ export function formatHistory(h, format, outDir) {
197
209
  return lines.join('\n');
198
210
  }
199
211
  export function formatSession(s, format) {
200
- if (format === 'json')
201
- return JSON.stringify({ command: 'session', ...s, counts: counts(s.findings) }, null, 2);
212
+ if (format === 'json') {
213
+ // The union menu itself goes to --union-out; the report says how big it is.
214
+ const { union, ...rest } = s;
215
+ return JSON.stringify({ command: 'session', ...rest, union: { tools: union.tools.length, totalTokens: union.totalTokens }, counts: counts(s.findings) }, null, 2);
216
+ }
202
217
  if (format === 'github') {
203
218
  return githubLines(s.findings.map((f) => ({ ...f, message: f.step ? `step ${f.step} (${s.steps[f.step - 1]?.label}): ${f.message}` : f.message }))).join('\n');
204
219
  }
@@ -208,6 +223,7 @@ export function formatSession(s, format) {
208
223
  '',
209
224
  `\`${s.server.name ?? 'server'}\` ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport} · ${s.baseline.tools} → ${s.final.tools} tools · ~${s.baseline.tokens.toLocaleString('en-US')} → ~${s.final.tokens.toLocaleString('en-US')} tokens (estimate)`,
210
225
  '',
226
+ ...(autoLine(s) ? [autoLine(s), ''] : []),
211
227
  '| Step | Menu | list_changed | Scope |',
212
228
  '|---|---|---|---|',
213
229
  ...s.steps.map((st) => `| ${st.index}. ${mdCell(st.label)} | ${st.status !== 'ok' ? st.status : st.changed ? `changed (${st.tools} tools)` : 'no change'} | ${st.changed ? (st.listChanged ? 'received' : '**missing**') : ''} | ${st.scope ?? ''} |`),
@@ -221,6 +237,7 @@ export function formatSession(s, format) {
221
237
  `toolmenu session ${s.scenario}`,
222
238
  ` ${s.server.name ?? 'server'} ${s.server.version ?? ''} · protocol ${s.server.protocolVersion ?? '?'} · ${s.transport}`,
223
239
  ` baseline: ${plural(s.baseline.tools, 'tool')} · ${tok(s.baseline.tokens)} tokens (estimate) · listening for list_changed: ${s.listening ? 'yes' : 'no'}`,
240
+ ...(autoLine(s) ? [` ${autoLine(s)}`] : []),
224
241
  ` fresh-${s.transport === 'stdio' ? 'process' : 'connection'} check: ${s.connectionCheck === undefined ? 'not checked' : s.connectionCheck === 'same' ? 'the same menu' : 'a DIFFERENT menu'}`,
225
242
  '',
226
243
  ];
@@ -264,6 +281,21 @@ export function formatPlan(scenario, name) {
264
281
  lines.push('', 'After every step the menu is listed and compared with the one before. Nothing was run.');
265
282
  return lines.join('\n');
266
283
  }
284
+ /** One line on what --auto called and what it left out, and how to reach more. */
285
+ function autoLine(s) {
286
+ if (!s.auto)
287
+ return undefined;
288
+ const by = (r) => s.auto.skipped.filter((x) => x.reason === r);
289
+ const needs = by('needs values');
290
+ const params = [...new Set(needs.flatMap((x) => x.missing ?? []))];
291
+ const parts = [
292
+ by('not read-only').length && `${by('not read-only').length} not read-only`,
293
+ by('open world').length && `${by('open world').length} marked openWorldHint: true (--open-world to call them)`,
294
+ needs.length && `${needs.length} need values the schema doesn't give (${params.slice(0, 6).join(', ')}${params.length > 6 ? ', …' : ''}: write a scenario, or --save-scenario and fill them in)`,
295
+ by('over the call budget').length && `${by('over the call budget').length} over --max-calls`,
296
+ ].filter(Boolean);
297
+ return `auto: called ${s.auto.called.length} tool${s.auto.called.length === 1 ? '' : 's'}${parts.length ? ` · skipped ${parts.join(', ')}` : ''}`;
298
+ }
267
299
  function mdCell(text) {
268
300
  return text.replace(/\|/g, '\\|').replace(/\n/g, ' ');
269
301
  }