toolmenu 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/auto.js ADDED
@@ -0,0 +1,81 @@
1
+ import { synthesizeArgs } from './args.js';
2
+ import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf } from './words.js';
3
+ /**
4
+ * A scenario built from the menu, no hand-written steps: every read-only tool
5
+ * whose required arguments the schema can fill, cheapest first (no arguments,
6
+ * then lists and searches, then single lookups, then the rest), then the first
7
+ * call again, which should change nothing. Never a tool that isn't marked
8
+ * readOnlyHint, whatever allow_writes says.
9
+ */
10
+ export function autoScenario(menu, options = {}) {
11
+ const maxCalls = options.maxCalls ?? 20;
12
+ const skipped = [];
13
+ const candidates = [];
14
+ for (const tool of menu.tools) {
15
+ const a = tool.annotations ?? {};
16
+ if (a.readOnlyHint !== true) {
17
+ skipped.push({ tool: tool.name, reason: 'not read-only' });
18
+ continue;
19
+ }
20
+ // readOnlyHint is the server's own claim, and a read-only search still spends
21
+ // credits on a paid API. Servers that mark a tool openWorldHint: true are the
22
+ // web search, fetch and scraping ones (Exa, Firecrawl, Context7, Playwright):
23
+ // opt-in. Unmarked tools are called: on GitHub, Atlassian or Notion that's
24
+ // every read, of the account the credentials belong to (FINDINGS F13).
25
+ if (a.openWorldHint === true && !options.openWorld) {
26
+ skipped.push({ tool: tool.name, reason: 'open world' });
27
+ continue;
28
+ }
29
+ const filled = synthesizeArgs(tool.inputSchema);
30
+ if (!filled.ok) {
31
+ skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
32
+ continue;
33
+ }
34
+ candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args) });
35
+ }
36
+ candidates.sort((x, y) => x.rank - y.rank);
37
+ const chosen = candidates.slice(0, maxCalls);
38
+ for (const c of candidates.slice(maxCalls))
39
+ skipped.push({ tool: c.tool.name, reason: 'over the call budget' });
40
+ const steps = [{ kind: 'list' }];
41
+ for (const c of chosen)
42
+ steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
43
+ if (chosen.length)
44
+ steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
45
+ return { scenario: { allowWrites: false, steps }, called: chosen.map((c) => c.tool.name), skipped };
46
+ }
47
+ function rank(tool, args) {
48
+ if (Object.keys(args).length === 0)
49
+ return 0;
50
+ const verb = verbOf(tool.name);
51
+ if (verb && COLLECTION_VERBS.has(verb))
52
+ return 1;
53
+ if (verb && LOOKUP_VERBS.has(verb))
54
+ return 2;
55
+ return 3;
56
+ }
57
+ /** The plan as a scenario file, to commit and run again with --scenario. */
58
+ export function scenarioYaml(plan, serverName) {
59
+ const lines = [
60
+ `# Written by \`toolmenu session --auto\`${serverName ? ` for ${serverName}` : ''}: every read-only tool whose`,
61
+ '# required arguments the schema could fill. Edit freely.',
62
+ 'allow_writes: false',
63
+ 'steps:',
64
+ ];
65
+ for (const step of plan.scenario.steps) {
66
+ if (step.kind === 'list')
67
+ lines.push(' - list');
68
+ else if (step.kind === 'call') {
69
+ lines.push(` - call: ${step.tool}`);
70
+ if (Object.keys(step.args).length)
71
+ lines.push(` args: ${JSON.stringify(step.args)}`);
72
+ }
73
+ }
74
+ const needs = plan.skipped.filter((s) => s.reason === 'needs values');
75
+ if (needs.length) {
76
+ lines.push('', ' # Read-only tools that need values the schema doesn\'t give:');
77
+ for (const s of needs.slice(0, 15))
78
+ lines.push(` # - call: ${s.tool}`, ` # args: { ${s.missing.map((m) => `${m}: TODO`).join(', ')} }`);
79
+ }
80
+ return lines.join('\n') + '\n';
81
+ }
package/dist/cli.js CHANGED
@@ -13,6 +13,9 @@ import { buildMenu } from './menu.js';
13
13
  import { existsSync } from 'node:fs';
14
14
  import { loadRoutes } from './routes.js';
15
15
  import { snapshot } from './snapshot.js';
16
+ import { MAIN_SEED, probeMenu, seeded } from './probe.js';
17
+ import { autoScenario, scenarioYaml } from './auto.js';
18
+ import { authDir, listLogins, login, logout } from './auth.js';
16
19
  import { SEVERITY_RANK } from './types.js';
17
20
  import { VERSION } from './version.js';
18
21
  const HELP = `toolmenu ${VERSION}
@@ -23,14 +26,18 @@ Usage:
23
26
  toolmenu snapshot [options] <url> Streamable HTTP server
24
27
  toolmenu diff [options] <old.json> <new.json> compare two snapshots
25
28
  toolmenu history [options] <npm-package> snapshot and diff published versions
29
+ toolmenu auth login <url> | logout <url> | list OAuth logins for HTTP servers
26
30
  toolmenu session --scenario <file> [options] -- <command> | <url>
27
31
  run a scripted session, watch the menu
32
+ toolmenu session --auto [options] -- <command> | <url>
33
+ the same, with steps built from the menu
28
34
 
29
35
  Commands:
30
36
  snapshot establish the menu: list tools twice, write menu.json, run the menu rules
31
37
  diff compare releases: breaking changes, token change, semver bump
32
38
  session observe the menu changing while a scripted session runs
33
39
  history research release history: install, snapshot and diff published npm versions
40
+ auth log in to an OAuth-protected server once; snapshot and session then use it
34
41
 
35
42
  Options:
36
43
  --out <path> where to write the menu (default: menu.json)
@@ -43,7 +50,10 @@ Options:
43
50
  --header <k: v> HTTP header, repeatable (e.g. "Authorization: Bearer ...")
44
51
  --env <K=V> environment variable for a stdio server, repeatable. The server
45
52
  only gets a minimal environment (PATH, HOME, ...) plus these
53
+ --no-auth don't use a stored OAuth login for this server
46
54
  --timeout <ms> per-request timeout (default: 30000)
55
+ --processes <n> server processes (stdio) or connections (HTTP) to compare,
56
+ the main one included (default: 2; 1 turns the check off)
47
57
  -h, --help show this help
48
58
  -v, --version show the version
49
59
 
@@ -58,6 +68,15 @@ session options:
58
68
  --plan print the steps without connecting or running anything
59
69
  --init write a starter scenario from the server's menu (to --scenario,
60
70
  default scenario.yml; never overwrites)
71
+ --auto build the steps from the menu: every read-only tool whose
72
+ required arguments the schema can fill (const, default,
73
+ examples, enum, type), then the first call again
74
+ --open-world with --auto, also call read-only tools marked openWorldHint:
75
+ true (web search, fetch, scraping): they may cost API credits
76
+ --max-calls <n> with --auto, at most n calls (default: 20)
77
+ --save-scenario <p> with --auto, write the steps it ran as a scenario file
78
+ --union-out <path> write every tool the session saw as a menu file, to commit as
79
+ the baseline for diff (tools behind unlocks included)
61
80
 
62
81
  history options (best effort; installs and runs third-party code, so use a container):
63
82
  --versions <n|all> number of published versions to inspect (default: 10)
@@ -71,6 +90,12 @@ history options (best effort; installs and runs third-party code, so use a conta
71
90
  --keep-installs keep each version's install directory
72
91
  --out <dir> where menus and history.json go (default: toolmenu-history/<package>)
73
92
 
93
+ auth options:
94
+ --port <n> loopback port for the login redirect (default: 33418)
95
+ --scope <scopes> scopes to ask for (default: the server's)
96
+ --client-id <id> a pre-registered client, for servers without dynamic registration
97
+ --client-secret <s> its secret, if it has one
98
+
74
99
  Exit codes: 0 clean · 1 findings at or above --fail-on · 2 couldn't connect or bad usage
75
100
  `;
76
101
  class UsageError extends Error {
@@ -93,6 +118,12 @@ export async function main(argv) {
93
118
  header: { type: 'string', multiple: true },
94
119
  env: { type: 'string', multiple: true },
95
120
  timeout: { type: 'string' },
121
+ 'no-auth': { type: 'boolean' },
122
+ port: { type: 'string' },
123
+ scope: { type: 'string' },
124
+ 'client-id': { type: 'string' },
125
+ 'client-secret': { type: 'string' },
126
+ processes: { type: 'string' },
96
127
  versions: { type: 'string' },
97
128
  release: { type: 'string' },
98
129
  'server-version-is-release': { type: 'boolean' },
@@ -107,6 +138,11 @@ export async function main(argv) {
107
138
  scenario: { type: 'string' },
108
139
  plan: { type: 'boolean' },
109
140
  init: { type: 'boolean' },
141
+ 'union-out': { type: 'string' },
142
+ auto: { type: 'boolean' },
143
+ 'open-world': { type: 'boolean' },
144
+ 'max-calls': { type: 'string' },
145
+ 'save-scenario': { type: 'string' },
110
146
  help: { type: 'boolean', short: 'h' },
111
147
  version: { type: 'boolean', short: 'v' },
112
148
  },
@@ -120,7 +156,7 @@ export async function main(argv) {
120
156
  process.stdout.write(HELP);
121
157
  return sub || values.help ? 0 : 2;
122
158
  }
123
- if (!['snapshot', 'diff', 'history', 'session'].includes(sub))
159
+ if (!['snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
124
160
  throw new UsageError(`Unknown command "${sub}". Try --help.`);
125
161
  const format = (values.json ? 'json' : values.format ?? 'text');
126
162
  if (!['text', 'json', 'github', 'markdown'].includes(format))
@@ -131,7 +167,12 @@ export async function main(argv) {
131
167
  const timeoutMs = values.timeout ? Number(values.timeout) : 30_000;
132
168
  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
133
169
  throw new UsageError(`--timeout must be a positive number of milliseconds`);
170
+ if (sub === 'auth')
171
+ return authCommand(rest, values);
134
172
  const config = await loadConfig(values.config);
173
+ const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
174
+ if (!Number.isInteger(processes) || processes < 1)
175
+ throw new UsageError('--processes must be a whole number, 1 or more');
135
176
  if (sub === 'diff') {
136
177
  if (command)
137
178
  throw new UsageError('diff compares two menu files; it doesn\'t start a server. Run snapshot first.');
@@ -186,8 +227,8 @@ export async function main(argv) {
186
227
  const path = values.scenario ?? 'scenario.yml';
187
228
  if (existsSync(path))
188
229
  throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
189
- const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
190
- const conn = await connect(initTarget, { timeoutMs });
230
+ const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
231
+ const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
191
232
  try {
192
233
  const list = await listTools(conn, { timeoutMs });
193
234
  const menu = buildMenu(list.tools, { name: conn.server.name, version: conn.server.version, protocolVersion: conn.protocolVersion });
@@ -199,6 +240,33 @@ export async function main(argv) {
199
240
  }
200
241
  return 0;
201
242
  }
243
+ if (sub === 'session' && values.auto) {
244
+ if (values.scenario)
245
+ throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
246
+ const maxCalls = values['max-calls'] ? Number(values['max-calls']) : 20;
247
+ if (!Number.isInteger(maxCalls) || maxCalls < 1)
248
+ throw new UsageError('--max-calls must be a whole number, 1 or more');
249
+ const savePath = values['save-scenario'];
250
+ if (savePath && existsSync(savePath))
251
+ throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
252
+ const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
253
+ const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
254
+ const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
255
+ if (savePath)
256
+ await writeFile(savePath, scenarioYaml(plan, menu.server.name));
257
+ if (values.plan) {
258
+ process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
259
+ return 0;
260
+ }
261
+ const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
262
+ result.auto = { called: plan.called, skipped: plan.skipped };
263
+ if (values['union-out'])
264
+ await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
265
+ const output = formatSession(result, format);
266
+ if (output)
267
+ process.stdout.write(output + '\n');
268
+ return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
269
+ }
202
270
  if (sub === 'session') {
203
271
  if (!values.scenario)
204
272
  throw new UsageError('session needs --scenario <file>. Run session --init for a starter, or see https://github.com/niksa90/toolmenu#session-the-menu-changing-while-the-agent-works for the format.');
@@ -213,17 +281,19 @@ export async function main(argv) {
213
281
  process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
214
282
  return 0;
215
283
  }
216
- const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? []);
217
- const result = await session(sessionTarget, scenario, { timeoutMs, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
284
+ const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
285
+ const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
286
+ if (values['union-out'])
287
+ await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
218
288
  const output = formatSession(result, format);
219
289
  if (output)
220
290
  process.stdout.write(output + '\n');
221
291
  return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
222
292
  }
223
- const target = parseTarget(rest, command, values.header ?? [], values.env ?? []);
293
+ const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
224
294
  const routesPath = values.routes ?? config.routes;
225
295
  const routes = routesPath ? await loadRoutes(routesPath) : undefined;
226
- const { menu, findings } = await snapshot(target, { routes, timeoutMs, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
296
+ const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
227
297
  const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
228
298
  if (outPath)
229
299
  await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
@@ -232,7 +302,7 @@ export async function main(argv) {
232
302
  process.stdout.write(output + '\n');
233
303
  return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
234
304
  }
235
- function parseTarget(positionals, command, headers, env) {
305
+ function parseTarget(positionals, command, headers, env, noAuth = false) {
236
306
  if (command) {
237
307
  if (command.length === 0)
238
308
  throw new UsageError('Nothing after "--": give the command that starts the server.');
@@ -247,7 +317,33 @@ function parseTarget(positionals, command, headers, env) {
247
317
  throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
248
318
  if (!/^https?:\/\//.test(url))
249
319
  throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
250
- return { kind: 'http', url, headers: parsePairs(headers, ':', '--header') };
320
+ return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
321
+ }
322
+ async function authCommand(args, values) {
323
+ const [action, url, ...extra] = args;
324
+ if (action === 'list') {
325
+ const logins = await listLogins();
326
+ process.stdout.write(logins.length ? logins.map((l) => `${l.serverUrl} (issuer ${l.issuer ?? '?'}, saved ${l.savedAt?.slice(0, 10) ?? '?'})`).join('\n') + '\n' : `No logins. Stored in ${authDir()}.\n`);
327
+ return 0;
328
+ }
329
+ if (action !== 'login' && action !== 'logout')
330
+ throw new UsageError('auth takes login <url>, logout <url> or list.');
331
+ if (!url || extra.length)
332
+ throw new UsageError(`auth ${action} needs one server URL: toolmenu auth ${action} https://example.com/mcp`);
333
+ if (!/^https?:\/\//.test(url))
334
+ throw new UsageError(`"${url}" isn't an http(s) URL.`);
335
+ if (action === 'logout') {
336
+ process.stdout.write((await logout(url)) ? `Logged out of ${url}.\n` : `No login stored for ${url}.\n`);
337
+ return 0;
338
+ }
339
+ const port = values.port ? Number(values.port) : undefined;
340
+ if (port !== undefined && !(Number.isInteger(port) && port > 0 && port < 65536))
341
+ throw new UsageError('--port must be a port number');
342
+ if (values['client-secret'] && !values['client-id'])
343
+ throw new UsageError('--client-secret goes with --client-id');
344
+ const result = await login(url, { port, scope: values.scope, clientId: values['client-id'], clientSecret: values['client-secret'] });
345
+ process.stdout.write(`Logged in to ${result.name ?? url}: ${result.tools} tools. snapshot and session use this login from now on (--no-auth to skip it).\n`);
346
+ return 0;
251
347
  }
252
348
  function parsePairs(items, separator, flag) {
253
349
  const pairs = {};
package/dist/config.d.ts CHANGED
@@ -14,6 +14,8 @@ export interface Config {
14
14
  descriptionLimit?: number | string;
15
15
  /** Tools (names or globs) your client sends with the full description. */
16
16
  fullDescriptions?: string[];
17
+ /** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
18
+ processes?: number;
17
19
  }
18
20
  /** Optional: no config file is fine. */
19
21
  export declare function loadConfig(path?: string): Promise<Config>;
package/dist/config.js CHANGED
@@ -23,5 +23,8 @@ export async function loadConfig(path) {
23
23
  if (limit !== undefined && !(typeof limit === 'number' && limit > 0) && !(typeof limit === 'string' && limit in CLIENT_LIMITS)) {
24
24
  throw new Error(`${file}: descriptionLimit must be a positive number or one of ${Object.keys(CLIENT_LIMITS).join(', ')}`);
25
25
  }
26
+ if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
27
+ throw new Error(`${file}: processes must be a whole number, 1 or more`);
28
+ }
26
29
  return config;
27
30
  }
package/dist/connect.d.ts CHANGED
@@ -11,6 +11,7 @@ export type Target = {
11
11
  kind: 'http';
12
12
  url: string;
13
13
  headers?: Record<string, string>;
14
+ noAuth?: boolean;
14
15
  };
15
16
  export interface WireResponse {
16
17
  method: string;
package/dist/connect.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
2
2
  import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
3
+ import { hasLogin, StoredOAuthProvider } from './auth.js';
3
4
  import { eraOf } from './menu.js';
4
5
  import { VERSION } from './version.js';
5
6
  /** Every response the server sent, matched to the request method, as it came off the wire. */
@@ -48,6 +49,7 @@ export async function connect(target, options = {}) {
48
49
  const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
49
50
  let transport;
50
51
  let stderrText = '';
52
+ let oauth = false;
51
53
  if (target.kind === 'stdio') {
52
54
  const stdio = new StdioClientTransport({
53
55
  command: target.command,
@@ -62,8 +64,13 @@ export async function connect(target, options = {}) {
62
64
  transport = stdio;
63
65
  }
64
66
  else {
67
+ // A stored OAuth login (toolmenu auth login) is used unless the request brings
68
+ // its own Authorization header, or --no-auth.
69
+ const ownAuth = Object.keys(target.headers ?? {}).some((h) => h.toLowerCase() === 'authorization');
70
+ oauth = !target.noAuth && !ownAuth && hasLogin(target.url);
65
71
  transport = new StreamableHTTPClientTransport(new URL(target.url), {
66
72
  requestInit: { headers: target.headers ?? {} },
73
+ ...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
67
74
  });
68
75
  }
69
76
  try {
@@ -75,6 +82,9 @@ export async function connect(target, options = {}) {
75
82
  const stderr = stderrText.trim();
76
83
  if (stderr && error instanceof Error)
77
84
  error.message += `\nserver stderr:\n${stderr}`;
85
+ if (target.kind === 'http' && !oauth && error instanceof Error && /\b401\b|unauthori[sz]ed/i.test(error.message)) {
86
+ error.message += `\nIf the server uses OAuth, log in once with: toolmenu auth login ${target.url}`;
87
+ }
78
88
  await client.close().catch(() => { });
79
89
  throw error;
80
90
  }
@@ -90,7 +100,7 @@ export async function connect(target, options = {}) {
90
100
  server: { name: server.name, version: server.version },
91
101
  capabilities: (client.getServerCapabilities() ?? {}),
92
102
  usedAuth: target.kind === 'http' &&
93
- Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h)),
103
+ (oauth || Object.keys(target.headers ?? {}).some((h) => /^(authorization|x-api-key|api-key|cookie)$/i.test(h))),
94
104
  stderr: () => stderrText,
95
105
  close: () => client.close(),
96
106
  };
@@ -0,0 +1,17 @@
1
+ import type { MenuTool } from './types.js';
2
+ export interface Difference {
3
+ /** JSON path of the first value that differs, from the tool (`inputSchema.properties.fields.default`). */
4
+ path: string;
5
+ before: unknown;
6
+ after: unknown;
7
+ /** Same items in a different order: a comma-separated string or an array. */
8
+ reordered: boolean;
9
+ }
10
+ /** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
11
+ export declare function firstDifference(a: unknown, b: unknown, path?: string): Difference | undefined;
12
+ /**
13
+ * One line saying what differs in a tool between two menus, for a finding's
14
+ * detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
15
+ * different order ("assignee,…" vs "issuetype,…")`.
16
+ */
17
+ export declare function describeToolDifference(before: MenuTool, after: MenuTool): string;
@@ -0,0 +1,70 @@
1
+ import { canonical } from './compare.js';
2
+ import { toolDefinition } from './menu.js';
3
+ /** The first leaf where two values differ, depth first, keys in sorted order. Undefined when equal. */
4
+ export function firstDifference(a, b, path = '') {
5
+ if (canonical(a) === canonical(b))
6
+ return undefined;
7
+ if (isObject(a) && isObject(b)) {
8
+ for (const key of [...new Set([...Object.keys(a), ...Object.keys(b)])].sort()) {
9
+ const found = firstDifference(a[key], b[key], join(path, key));
10
+ if (found)
11
+ return found;
12
+ }
13
+ }
14
+ if (Array.isArray(a) && Array.isArray(b) && a.length === b.length && !samePermutation(a, b)) {
15
+ for (let i = 0; i < a.length; i++) {
16
+ const found = firstDifference(a[i], b[i], `${path}[${i}]`);
17
+ if (found)
18
+ return found;
19
+ }
20
+ }
21
+ return { path, before: a, after: b, reordered: samePermutation(a, b) };
22
+ }
23
+ /** Different order, same items: arrays, or strings that are comma-separated lists. */
24
+ function samePermutation(a, b) {
25
+ const items = (v) => {
26
+ if (Array.isArray(v))
27
+ return v.map((x) => canonical(x));
28
+ if (typeof v === 'string' && v.includes(','))
29
+ return v.split(',').map((s) => s.trim());
30
+ return undefined;
31
+ };
32
+ const x = items(a);
33
+ const y = items(b);
34
+ if (!x || !y || x.length !== y.length || x.length < 2)
35
+ return false;
36
+ return canonical([...x].sort()) === canonical([...y].sort()) && canonical(x) !== canonical(y);
37
+ }
38
+ /**
39
+ * One line saying what differs in a tool between two menus, for a finding's
40
+ * detail: `jira_get_issue: inputSchema.properties.fields.default: same 11 items,
41
+ * different order ("assignee,…" vs "issuetype,…")`.
42
+ */
43
+ export function describeToolDifference(before, after) {
44
+ const a = toolDefinition(before);
45
+ const b = toolDefinition(after);
46
+ const d = firstDifference(a, b);
47
+ if (!d) {
48
+ return JSON.stringify(a) === JSON.stringify(b)
49
+ ? `${after.name}: identical`
50
+ : `${after.name}: same content, keys in a different order (the bytes differ)`;
51
+ }
52
+ const where = d.path || '(the whole tool)';
53
+ if (d.reordered) {
54
+ const n = Array.isArray(d.before) ? d.before.length : String(d.before).split(',').length;
55
+ return `${after.name}: ${where}: same ${n} items, different order (${show(d.before)} vs ${show(d.after)}). Sort them.`;
56
+ }
57
+ return `${after.name}: ${where}: ${show(d.before)} vs ${show(d.after)}`;
58
+ }
59
+ function show(value) {
60
+ if (value === undefined)
61
+ return '(absent)';
62
+ const text = JSON.stringify(value);
63
+ return text.length > 60 ? text.slice(0, 57) + '…' : text;
64
+ }
65
+ function isObject(v) {
66
+ return !!v && typeof v === 'object' && !Array.isArray(v);
67
+ }
68
+ function join(path, key) {
69
+ return /^[A-Za-z_$][\w$]*$/.test(key) ? (path ? `${path}.${key}` : key) : `${path}[${JSON.stringify(key)}]`;
70
+ }
@@ -0,0 +1,14 @@
1
+ export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'invalid-arguments' | 'other';
2
+ /** Failures of the setup (credentials, this machine, the network), not of the server. */
3
+ export declare const SETUP_FAILURES: Set<FailureClass>;
4
+ /**
5
+ * Classify a failed call. `code` is the JSON-RPC error code when the call threw.
6
+ * `hadArguments`: a 404 on a call that named nothing (no arguments) means the
7
+ * account or site behind the credentials isn't there (HubSpot answers a bad token
8
+ * with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
9
+ */
10
+ export declare function classifyFailure(text: string, options?: {
11
+ code?: number;
12
+ hadArguments?: boolean;
13
+ }): FailureClass;
14
+ export declare const FAILURE_LABELS: Record<FailureClass, string>;
@@ -0,0 +1,47 @@
1
+ /*
2
+ * Why a tool call failed, from its error text. The patterns come from real
3
+ * servers run with dummy credentials (FINDINGS F12): a failure that stops a call
4
+ * before the tool's own logic runs says nothing about the menu, and shouldn't read
5
+ * as a finding.
6
+ */
7
+ /** Failures of the setup (credentials, this machine, the network), not of the server. */
8
+ export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found']);
9
+ // Something this machine lacks: "Could not find Google Chrome executable"
10
+ // (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright).
11
+ const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b/i;
12
+ // "401 Bad credentials" (GitHub), "Authorization Expired" (Sentry), "Invalid
13
+ // token" (Firecrawl), "authentication token is not valid" (Apify), "Please provide
14
+ // a valid access token" (Supabase). Not a bare "token": plenty of errors mention one.
15
+ const AUTH = /\b(401|403)\b|unauthori[sz]ed|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|valid (access )?token/i;
16
+ const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
17
+ // "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
18
+ // "**Input Error**" (Sentry), JSON-RPC's invalid params.
19
+ const INVALID_ARGUMENTS = /invalid (params|parameters|arguments|input)|input error|validation (error|failed)|must be provided|is required\b|\beither\b[^.]{0,80}\bor\b[^.]{0,80}\b(must|required)/i;
20
+ const NOT_FOUND = /\b404\b|not found/i;
21
+ /**
22
+ * Classify a failed call. `code` is the JSON-RPC error code when the call threw.
23
+ * `hadArguments`: a 404 on a call that named nothing (no arguments) means the
24
+ * account or site behind the credentials isn't there (HubSpot answers a bad token
25
+ * with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
26
+ */
27
+ export function classifyFailure(text, options = {}) {
28
+ if (ENVIRONMENT.test(text))
29
+ return 'environment';
30
+ if (AUTH.test(text))
31
+ return 'auth';
32
+ if (NETWORK.test(text))
33
+ return 'network';
34
+ if (options.code === -32602 || INVALID_ARGUMENTS.test(text))
35
+ return 'invalid-arguments';
36
+ if (NOT_FOUND.test(text) && !options.hadArguments)
37
+ return 'not-found';
38
+ return 'other';
39
+ }
40
+ export const FAILURE_LABELS = {
41
+ auth: 'authentication',
42
+ environment: 'something missing on this machine',
43
+ network: 'the network',
44
+ 'not-found': 'a 404 on a call that named nothing (usually the account or site behind the credentials)',
45
+ 'invalid-arguments': 'the arguments',
46
+ other: 'other errors',
47
+ };
package/dist/history.js CHANGED
@@ -80,7 +80,9 @@ async function inspectVersion(pkg, v, options, source) {
80
80
  return { row: fail(row, 'no-bin', new Error('The package has no bin to start, and no --cmd was given.')) };
81
81
  let result;
82
82
  try {
83
- result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir }, { timeoutMs: options.timeoutMs ?? 30_000 });
83
+ result = await snapshot({ kind: 'stdio', command: command[0], args: [...command.slice(1), ...(options.args ?? [])], env: options.env, cwd: dir },
84
+ // One process per version: history already starts one for each, and runs long.
85
+ { timeoutMs: options.timeoutMs ?? 30_000, processes: 1 });
84
86
  }
85
87
  catch (error) {
86
88
  return { row: fail(row, classifyFailure(error), error) };
@@ -0,0 +1,27 @@
1
+ import { type Target } from './connect.js';
2
+ import type { Probe } from './rules/determinism.js';
3
+ import type { Menu, MenuTool } from './types.js';
4
+ /**
5
+ * The hash seed of the process whose menu toolmenu saves and compares. Python
6
+ * orders sets of strings by a per-process random seed; pinning it makes a Python
7
+ * server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
8
+ */
9
+ export declare const MAIN_SEED = "0";
10
+ /** The probe's seed: different from the main one, so a set-order bug shows every time. */
11
+ export declare const PROBE_SEED = "1";
12
+ /**
13
+ * The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
14
+ * a stdio server. Other runtimes ignore it.
15
+ */
16
+ export declare function seeded(target: Target, seed: string): Target;
17
+ /** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
18
+ export declare function isContainerWrapper(target: Target): boolean;
19
+ /** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
20
+ export declare function probeMenu(target: Target, timeoutMs: number): Promise<Menu>;
21
+ /**
22
+ * Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
23
+ * compare with the main one. stdio probes run with a different hash seed each. An
24
+ * HTTP probe that differs is retried once: a rolling deploy can serve two
25
+ * versions for a moment.
26
+ */
27
+ export declare function probeVariance(target: Target, main: MenuTool[], processes: number, timeoutMs: number): Promise<Probe[]>;
package/dist/probe.js ADDED
@@ -0,0 +1,58 @@
1
+ import { basename } from 'node:path';
2
+ import { connect, listTools } from './connect.js';
3
+ import { compareMenus } from './compare.js';
4
+ import { buildMenu } from './menu.js';
5
+ /**
6
+ * The hash seed of the process whose menu toolmenu saves and compares. Python
7
+ * orders sets of strings by a per-process random seed; pinning it makes a Python
8
+ * server's saved menu the same on every run (FINDINGS F12: mcp-atlassian).
9
+ */
10
+ export const MAIN_SEED = '0';
11
+ /** The probe's seed: different from the main one, so a set-order bug shows every time. */
12
+ export const PROBE_SEED = '1';
13
+ /**
14
+ * The target with PYTHONHASHSEED set, unless the user set it (--env) or it isn't
15
+ * a stdio server. Other runtimes ignore it.
16
+ */
17
+ export function seeded(target, seed) {
18
+ if (target.kind !== 'stdio' || target.env?.PYTHONHASHSEED !== undefined)
19
+ return target;
20
+ return { ...target, env: { ...target.env, PYTHONHASHSEED: seed } };
21
+ }
22
+ /** A command that starts the server in a container: the seed (and --env) don't reach it unless passed through. */
23
+ export function isContainerWrapper(target) {
24
+ return target.kind === 'stdio' && /^(docker|podman|nerdctl)(\.exe)?$/.test(basename(target.command));
25
+ }
26
+ /** The menu a fresh process (stdio) or connection (HTTP) serves: connect, list once, close. */
27
+ export async function probeMenu(target, timeoutMs) {
28
+ const c = await connect(target, { timeoutMs });
29
+ try {
30
+ const list = await listTools(c, { timeoutMs });
31
+ return buildMenu(list.tools, { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era });
32
+ }
33
+ finally {
34
+ await c.close().catch(() => { });
35
+ }
36
+ }
37
+ /**
38
+ * Menus from `processes - 1` fresh processes (stdio) or connections (HTTP), to
39
+ * compare with the main one. stdio probes run with a different hash seed each. An
40
+ * HTTP probe that differs is retried once: a rolling deploy can serve two
41
+ * versions for a moment.
42
+ */
43
+ export async function probeVariance(target, main, processes, timeoutMs) {
44
+ const probes = [];
45
+ for (let i = 0; i < processes - 1; i++) {
46
+ const seed = String(Number(PROBE_SEED) + i);
47
+ try {
48
+ let tools = (await probeMenu(seeded(target, seed), timeoutMs)).tools;
49
+ if (target.kind === 'http' && compareMenus(main, tools).length)
50
+ tools = (await probeMenu(target, timeoutMs)).tools;
51
+ probes.push({ tools });
52
+ }
53
+ catch (error) {
54
+ probes.push({ error: error instanceof Error ? error.message : String(error) });
55
+ }
56
+ }
57
+ return probes;
58
+ }