toolmenu 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/catalog.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { toolTokens } from './menu.js';
2
+ import { RATE_LIMITED, tooMany } from './failures.js';
2
3
  import { nouns, singular } from './words.js';
3
4
  const QUERY_PARAM = /^(q|query|search|keywords?|text)$/i;
4
5
  const SEARCH_NAME = /(search|find|list|discover)[\w-]*(tools?|operations?|actions?|capabilit)|^discover$|(tools?|operations?)[\w-]*(search|discover)/i;
@@ -35,13 +36,33 @@ export async function readCatalog(conn, tools, options = {}) {
35
36
  throw new Error(options.tool ? `--catalog: no tool named ${options.tool} in the menu.` : '--catalog: no catalog search tool in the menu (read-only, one required query, named like search_*_tools or discover). Name it with catalog.tool in the config.');
36
37
  const queryParam = tool.inputSchema.required[0];
37
38
  const limit = limitArg(tool.inputSchema);
38
- const queries = options.queries ?? defaultQueries(tools);
39
+ // Seeds, then the catalog's own words: every operation found or mentioned is
40
+ // searched for by name, which finds it and its neighbours. Breadth first, in the
41
+ // order found, so the same catalog gives the same queries.
42
+ const queue = [...new Set(options.queries ?? [...describedExamples(tool), ...defaultQueries(tools)])];
43
+ const crawl = options.crawl ?? true;
44
+ // Crawl until the catalog stops yielding: a small one ends early, a big one
45
+ // (Atlassian's ~300 operations) goes on, up to the cap.
46
+ const maxQueries = options.maxQueries ?? 200;
47
+ const stopAfter = options.stopAfter ?? 20;
48
+ let dry = 0;
49
+ const asked = [];
50
+ const queued = new Set(queue);
51
+ const enqueue = (name) => {
52
+ const q = phrase(name);
53
+ if (crawl && q && !queued.has(q)) {
54
+ queued.add(q);
55
+ queue.push(q);
56
+ }
57
+ };
39
58
  const found = new Map();
40
59
  const failed = [];
41
60
  const pause = options.pauseMs ?? 300;
42
- for (const [i, query] of queries.entries()) {
43
- if (i > 0)
61
+ while (queue.length && asked.length < maxQueries && dry < stopAfter) {
62
+ const query = queue.shift();
63
+ if (asked.length > 0)
44
64
  await sleep(pause);
65
+ asked.push(query);
45
66
  // A rate limit is waited out (2 s, 4 s, 8 s); anything else, or a limit that
46
67
  // persists, leaves this query out and the catalog partial, never the snapshot
47
68
  // failed: diff treats operations not found as notices.
@@ -49,21 +70,30 @@ export async function readCatalog(conn, tools, options = {}) {
49
70
  try {
50
71
  const result = await conn.client.callTool({ name: tool.name, arguments: { [queryParam]: query, ...limit } }, { timeout: options.timeoutMs ?? 30_000 });
51
72
  const text = result.isError ? resultText(result) : '';
52
- if (result.isError && RATE_LIMIT.test(text) && attempt < 3) {
73
+ if (result.isError && RATE_LIMITED.test(text) && attempt < 3) {
53
74
  await sleep(2000 * 2 ** attempt);
54
75
  continue;
55
76
  }
56
77
  if (result.isError)
57
78
  failed.push({ query, error: text.slice(0, 200) || 'the tool returned an error' });
58
- else
59
- for (const op of operationsIn(result))
60
- if (!found.has(op.name))
61
- found.set(op.name, op);
79
+ else {
80
+ const before = found.size;
81
+ for (const op of operationsIn(result)) {
82
+ if (found.has(op.name))
83
+ continue;
84
+ found.set(op.name, op);
85
+ enqueue(op.name);
86
+ }
87
+ for (const name of mentionedNames(result))
88
+ if (!found.has(name))
89
+ enqueue(name);
90
+ dry = found.size > before ? 0 : dry + 1;
91
+ }
62
92
  break;
63
93
  }
64
94
  catch (error) {
65
95
  const message = error instanceof Error ? error.message.split('\n')[0] : String(error);
66
- if (RATE_LIMIT.test(message) && attempt < 3) {
96
+ if (tooMany(error) && attempt < 3) {
67
97
  await sleep(2000 * 2 ** attempt);
68
98
  continue;
69
99
  }
@@ -73,9 +103,25 @@ export async function readCatalog(conn, tools, options = {}) {
73
103
  }
74
104
  }
75
105
  const operations = [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
76
- return { tool: tool.name, queries, operations, ...(failed.length ? { failed } : {}) };
106
+ return { tool: tool.name, queries: asked, operations, ...(failed.length ? { failed } : {}) };
107
+ }
108
+ /** An operation name as a search phrase: listJiraIssueWorklogs → "list jira issue worklogs". */
109
+ function phrase(name) {
110
+ return name
111
+ .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
112
+ .split(/[^A-Za-z0-9]+/)
113
+ .filter(Boolean)
114
+ .map((w) => w.toLowerCase())
115
+ .join(' ');
116
+ }
117
+ /**
118
+ * Example queries the search tool's own schema suggests: Atlassian's discover
119
+ * documents its query with "transition a jira issue", "list confluence spaces"…
120
+ */
121
+ function describedExamples(tool) {
122
+ const text = [tool.description ?? '', ...Object.values(tool.inputSchema?.properties ?? {}).map((p) => String(p.description ?? ''))].join(' ');
123
+ return [...text.matchAll(/["“]([a-z][a-z0-9 ,'-]{5,60})["”]/g)].map((m) => m[1]).filter((q) => q.includes(' '));
77
124
  }
78
- const RATE_LIMIT = /rate.?limit|too many requests|\b429\b/i;
79
125
  function sleep(ms) {
80
126
  return new Promise((resolve) => setTimeout(resolve, ms));
81
127
  }
@@ -108,12 +154,9 @@ export function operationsIn(result) {
108
154
  const text = part.text;
109
155
  if (typeof text !== 'string')
110
156
  continue;
111
- try {
112
- roots.push(JSON.parse(text));
113
- }
114
- catch {
115
- // prose, not a catalog
116
- }
157
+ const value = leadingJson(text);
158
+ if (value !== undefined)
159
+ roots.push(value);
117
160
  }
118
161
  const out = new Map();
119
162
  const walk = (v, depth) => {
@@ -132,6 +175,46 @@ export function operationsIn(result) {
132
175
  roots.forEach((r) => walk(r, 0));
133
176
  return [...out.values()];
134
177
  }
178
+ /**
179
+ * The JSON value a text starts with, ignoring what follows: Atlassian's discover
180
+ * returns its results as JSON and then a prose list of related operations, which
181
+ * makes the whole text invalid JSON.
182
+ */
183
+ export function leadingJson(text) {
184
+ const trimmed = text.trimStart();
185
+ if (!/^[[{]/.test(trimmed))
186
+ return undefined;
187
+ try {
188
+ return JSON.parse(trimmed);
189
+ }
190
+ catch (error) {
191
+ const at = /position (\d+)/.exec(error instanceof Error ? error.message : '');
192
+ if (!at)
193
+ return undefined;
194
+ try {
195
+ return JSON.parse(trimmed.slice(0, Number(at[1])));
196
+ }
197
+ catch {
198
+ return undefined;
199
+ }
200
+ }
201
+ }
202
+ /**
203
+ * Operation names a result mentions without defining them: lines like
204
+ * "createJiraBoard — Create a new company-managed…" after Atlassian's JSON. Each
205
+ * is worth a search of its own.
206
+ */
207
+ export function mentionedNames(result) {
208
+ const names = new Set();
209
+ for (const part of Array.isArray(result.content) ? result.content : []) {
210
+ const text = part.text;
211
+ if (typeof text !== 'string')
212
+ continue;
213
+ for (const m of text.matchAll(/^[ \t]*[-*]?[ \t]*`?([A-Za-z][A-Za-z0-9_.-]{2,80})`?[ \t]+(?:—|–|-{1,2}|:)[ \t]/gm))
214
+ names.add(m[1]);
215
+ }
216
+ return [...names];
217
+ }
135
218
  function asOperation(o) {
136
219
  if (typeof o.name !== 'string' || !o.name)
137
220
  return undefined;
package/dist/cli.js CHANGED
@@ -7,13 +7,14 @@ import { diffMenus } from './diff.js';
7
7
  import { loadMenu } from './menu.js';
8
8
  import { history, historyCsv } from './history.js';
9
9
  import { counts, formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
10
- import { loadScenario, session, starterScenario } from './session.js';
11
- import { connect, listTools } from './connect.js';
10
+ import { loadScenario, session, starterScenario, unlockListers, valuesFromListing } from './session.js';
11
+ import { listTools } from './connect.js';
12
+ import { patiently, tooMany } from './failures.js';
12
13
  import { buildMenu } from './menu.js';
13
14
  import { existsSync } from 'node:fs';
14
15
  import { loadRoutes } from './routes.js';
15
16
  import { snapshot } from './snapshot.js';
16
- import { MAIN_SEED, probeMenu, seeded } from './probe.js';
17
+ import { connectPatiently, MAIN_SEED, probeMenu, refusedForTooMany, seeded } from './probe.js';
17
18
  import { autoScenario, scenarioYaml } from './auto.js';
18
19
  import { authDir, listLogins, login, logout } from './auth.js';
19
20
  import { detectProject, secretsNeeded, workflowYaml } from './init.js';
@@ -58,7 +59,8 @@ Options:
58
59
  --catalog also read the operations behind a search tool (search and execute:
59
60
  discover, search_*_tools) and keep them in the menu file for diff
60
61
  --processes <n> server processes (stdio) or connections (HTTP) to compare,
61
- the main one included (default: 2; 1 turns the check off)
62
+ the main one included (default: 2; 1 opens no second one, for
63
+ session's scope check either: for servers with one session per client)
62
64
  -h, --help show this help
63
65
  -v, --version show the version
64
66
 
@@ -238,12 +240,29 @@ export async function main(argv) {
238
240
  if (existsSync(path))
239
241
  throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
240
242
  const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
241
- const conn = await connect(seeded(initTarget, MAIN_SEED), { timeoutMs });
243
+ const conn = await connectPatiently(seeded(initTarget, MAIN_SEED), { timeoutMs });
242
244
  try {
243
- const list = await listTools(conn, { timeoutMs });
245
+ const list = await patiently(() => listTools(conn, { timeoutMs }), { error: tooMany });
244
246
  const menu = buildMenu(list.tools, { name: conn.server.name, version: conn.server.version, protocolVersion: conn.protocolVersion });
245
- await writeFile(path, starterScenario(menu));
246
- process.stdout.write(`wrote ${path}: a starter scenario for ${menu.tools.length} tools. Fill in the TODOs, then run toolmenu session --scenario ${path}\n`);
247
+ // An unlock without an enum: its values come from the server's own read-only
248
+ // listing (list_toolsets), so the scenario can unlock every one.
249
+ const unlockValues = {};
250
+ const read = [];
251
+ for (const { unlock, lister } of unlockListers(menu.tools)) {
252
+ try {
253
+ const result = await conn.client.callTool({ name: lister, arguments: {} }, { timeout: timeoutMs });
254
+ const found = result.isError ? [] : valuesFromListing(result);
255
+ if (found.length) {
256
+ unlockValues[unlock] = found;
257
+ read.push(`${found.length} values for ${unlock} from ${lister}`);
258
+ }
259
+ }
260
+ catch {
261
+ // Left as TODO in the scenario.
262
+ }
263
+ }
264
+ await writeFile(path, starterScenario(menu, { values: unlockValues }));
265
+ process.stdout.write(`wrote ${path}: a starter scenario for ${menu.tools.length} tools${read.length ? ` (read ${read.join('; ')})` : ''}. Fill in the TODOs, then run toolmenu session --scenario ${path}\n`);
247
266
  }
248
267
  finally {
249
268
  await conn.close().catch(() => { });
@@ -289,7 +308,10 @@ export async function main(argv) {
289
308
  if (savePath && existsSync(savePath))
290
309
  throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
291
310
  const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
292
- const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs);
311
+ const waited = { ms: 0 };
312
+ const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs, undefined, waited).catch((error) => {
313
+ throw refusedForTooMany(error, waited.ms);
314
+ });
293
315
  const plan = autoScenario(menu, { openWorld: values['open-world'], maxCalls });
294
316
  if (savePath)
295
317
  await writeFile(savePath, scenarioYaml(plan, menu.server.name));
@@ -297,7 +319,7 @@ export async function main(argv) {
297
319
  process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
298
320
  return 0;
299
321
  }
300
- const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto' });
322
+ const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto', unionOut: !!values['union-out'], auto: { called: plan.called, skipped: plan.skipped } });
301
323
  result.auto = { called: plan.called, skipped: plan.skipped };
302
324
  if (values['union-out'])
303
325
  await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
@@ -321,7 +343,7 @@ export async function main(argv) {
321
343
  return 0;
322
344
  }
323
345
  const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
324
- const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario });
346
+ const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario, unionOut: !!values['union-out'] });
325
347
  if (values['union-out'])
326
348
  await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
327
349
  const output = formatSession(result, format);
@@ -405,6 +427,12 @@ function exitWhenFlushed(code) {
405
427
  }
406
428
  process.stdout.write('', () => process.stderr.write('', () => process.exit(code)));
407
429
  }
430
+ // stdout carries the report alone (--format json is parsed by CI). Libraries log
431
+ // with console: the MCP SDK prints a console.debug line to stdout when a server
432
+ // has no tools capability. Their lines go to stderr.
433
+ for (const method of ['log', 'info', 'debug']) {
434
+ console[method] = (...args) => console.error(...args);
435
+ }
408
436
  main(process.argv.slice(2)).then(exitWhenFlushed, (error) => {
409
437
  const message = error instanceof Error ? error.message : String(error);
410
438
  process.stderr.write(`toolmenu: ${message}\n`);
package/dist/compare.js CHANGED
@@ -35,18 +35,33 @@ export function toolDiff(before, after) {
35
35
  kinds.push('serialization');
36
36
  return kinds;
37
37
  }
38
+ /**
39
+ * Each tool's identity in a list: its name, and for a name that repeats, which
40
+ * occurrence it is. The second `search` pairs with the second `search`, so a
41
+ * duplicate (menu/duplicate-name) isn't also read as a reorder.
42
+ */
43
+ function identities(tools) {
44
+ const seen = new Map();
45
+ return tools.map((t) => {
46
+ const n = seen.get(t.name) ?? 0;
47
+ seen.set(t.name, n + 1);
48
+ return n === 0 ? t.name : `${t.name}\u0000${n}`;
49
+ });
50
+ }
38
51
  export function compareMenus(before, after) {
39
52
  const changes = [];
40
- const beforeByName = new Map(before.map((t, i) => [t.name, { tool: t, index: i }]));
41
- const afterNames = new Set(after.map((t) => t.name));
53
+ const beforeIds = identities(before);
54
+ const afterIds = identities(after);
55
+ const beforeById = new Map(before.map((t, i) => [beforeIds[i], { tool: t, index: i }]));
56
+ const afterIdSet = new Set(afterIds);
42
57
  before.forEach((t, i) => {
43
- if (!afterNames.has(t.name))
58
+ if (!afterIdSet.has(beforeIds[i]))
44
59
  changes.push({ kind: 'removed', tool: t.name, position: i });
45
60
  });
46
61
  // Tools present in both: the ones outside the longest run that kept its
47
62
  // relative order are the ones that moved.
48
63
  const common = after
49
- .map((t, i) => ({ name: t.name, afterIndex: i, beforeIndex: beforeByName.get(t.name)?.index }))
64
+ .map((t, i) => ({ name: t.name, afterIndex: i, beforeIndex: beforeById.get(afterIds[i])?.index }))
50
65
  .filter((c) => c.beforeIndex !== undefined);
51
66
  const kept = longestIncreasingRun(common.map((c) => c.beforeIndex));
52
67
  common.forEach((c, i) => {
@@ -54,7 +69,7 @@ export function compareMenus(before, after) {
54
69
  changes.push({ kind: 'moved', tool: c.name, position: c.afterIndex });
55
70
  });
56
71
  after.forEach((t, i) => {
57
- const old = beforeByName.get(t.name);
72
+ const old = beforeById.get(afterIds[i]);
58
73
  if (!old) {
59
74
  changes.push({ kind: 'added', tool: t.name, position: i });
60
75
  return;
package/dist/config.d.ts CHANGED
@@ -21,6 +21,9 @@ export interface Config {
21
21
  tool?: string;
22
22
  queries?: string[];
23
23
  pauseMs?: number;
24
+ crawl?: boolean;
25
+ maxQueries?: number;
26
+ stopAfter?: number;
24
27
  };
25
28
  }
26
29
  /** Optional: no config file is fine. */
package/dist/diff.d.ts CHANGED
@@ -1,8 +1,10 @@
1
- import type { Finding, Menu, Severity } from './types.js';
1
+ import type { Finding, JsonSchema, Menu, Severity } from './types.js';
2
2
  export type ChangeClass = 'breaking' | 'minor' | 'notice';
3
3
  export type Bump = 'major' | 'minor' | 'patch' | 'none';
4
4
  export interface DiffFinding extends Finding {
5
5
  class?: ChangeClass;
6
+ /** Every place one change was found, when it's reached from several (a shared definition). */
7
+ places?: string[];
6
8
  }
7
9
  interface DiffRule {
8
10
  severity: Severity;
@@ -54,5 +56,15 @@ export interface DiffOptions {
54
56
  serverVersionIsRelease?: boolean;
55
57
  }
56
58
  export declare function diffMenus(before: Menu, after: Menu, options?: DiffOptions): DiffResult;
59
+ /**
60
+ * The schema with local $refs (`#/$defs/…`, `#/definitions/…`) replaced by what
61
+ * they point to, and the root $defs dropped: two spellings of one schema compare
62
+ * equal. A recursive definition is unrolled MAX_UNROLL level, then left as a
63
+ * $ref: a change to it shows in the levels above, and its $defs entry needn't be
64
+ * kept (keeping it made a recursive schema compare unequal to its own refactor,
65
+ * with nothing reported). A $ref toolmenu can't follow (another document) stays as
66
+ * it is. Keywords next to a $ref (a description) win over the target's.
67
+ */
68
+ export declare function resolveRefs(schema: JsonSchema | undefined): JsonSchema | undefined;
57
69
  export declare function versionBump(before: string | undefined, after: string | undefined): Bump | undefined;
58
70
  export {};