toolmenu 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +150 -30
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +61 -18
  6. package/dist/auto.d.ts +71 -7
  7. package/dist/auto.js +267 -13
  8. package/dist/breakdown.d.ts +35 -6
  9. package/dist/breakdown.js +179 -23
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +116 -178
  13. package/dist/compare.js +20 -5
  14. package/dist/config.d.ts +5 -1
  15. package/dist/config.js +14 -0
  16. package/dist/connect.d.ts +4 -0
  17. package/dist/connect.js +132 -11
  18. package/dist/diff.d.ts +35 -1
  19. package/dist/diff.js +1132 -121
  20. package/dist/difference.d.ts +2 -0
  21. package/dist/difference.js +13 -12
  22. package/dist/explain.d.ts +103 -0
  23. package/dist/explain.js +434 -0
  24. package/dist/failures.d.ts +55 -0
  25. package/dist/failures.js +103 -0
  26. package/dist/help.d.ts +9 -0
  27. package/dist/help.js +198 -0
  28. package/dist/history.d.ts +13 -0
  29. package/dist/history.js +86 -6
  30. package/dist/init.d.ts +100 -5
  31. package/dist/init.js +520 -26
  32. package/dist/menu.js +10 -2
  33. package/dist/options.d.ts +135 -0
  34. package/dist/options.js +48 -0
  35. package/dist/probe.d.ts +15 -3
  36. package/dist/probe.js +35 -11
  37. package/dist/report.d.ts +1 -1
  38. package/dist/report.js +229 -99
  39. package/dist/rules/description.d.ts +7 -2
  40. package/dist/rules/description.js +89 -43
  41. package/dist/rules/determinism.d.ts +3 -1
  42. package/dist/rules/determinism.js +146 -11
  43. package/dist/rules/ids.d.ts +1 -1
  44. package/dist/rules/ids.js +9 -4
  45. package/dist/rules/index.d.ts +6 -0
  46. package/dist/rules/index.js +15 -5
  47. package/dist/rules/naming.d.ts +1 -1
  48. package/dist/rules/naming.js +66 -19
  49. package/dist/rules/rule.d.ts +18 -0
  50. package/dist/rules/rule.js +25 -1
  51. package/dist/rules/spec.d.ts +2 -0
  52. package/dist/rules/spec.js +61 -8
  53. package/dist/rules/write.d.ts +1 -1
  54. package/dist/rules/write.js +42 -10
  55. package/dist/semver.js +1 -1
  56. package/dist/session.d.ts +56 -7
  57. package/dist/session.js +800 -94
  58. package/dist/snapshot.d.ts +2 -2
  59. package/dist/snapshot.js +25 -7
  60. package/dist/types.d.ts +14 -0
  61. package/dist/words.d.ts +5 -0
  62. package/dist/words.js +14 -0
  63. package/package.json +1 -1
package/dist/breakdown.js CHANGED
@@ -2,8 +2,12 @@ import { canonical } from './compare.js';
2
2
  import { countTokens } from './menu.js';
3
3
  const TOP = 5;
4
4
  const BIG_ENUM = 20;
5
- const REPEATED_MIN_TOKENS = 30;
5
+ const REPEATED_MIN_TOKENS = 25;
6
+ const SCHEMA_KEYWORDS = new Set(['anyOf', 'oneOf', 'allOf', 'not', 'items', 'additionalProperties', 'prefixItems', 'then', 'else', 'if', '$defs', 'definitions', 'patternProperties']);
7
+ /** Across tools only, a block has to be in this many to count (the same small parameter in two tools is normal). */
6
8
  const REPEATED_MIN_TOOLS = 3;
9
+ /** What a `{"$ref": "#/$defs/Block"}` costs in place of a copy (estimate). */
10
+ const REF_TOKENS = countTokens(JSON.stringify({ $ref: '#/$defs/Block' }));
7
11
  /** Where a menu's tokens go. Estimates, like every count in toolmenu. */
8
12
  export function breakdown(menu) {
9
13
  const total = menu.totalTokens || 1;
@@ -18,25 +22,121 @@ export function breakdown(menu) {
18
22
  schema: t.inputSchema ? countTokens(JSON.stringify(t.inputSchema)) : 0,
19
23
  }));
20
24
  const enums = [];
21
- const seen = new Map();
22
25
  for (const tool of menu.tools) {
23
26
  for (const [param, schema] of Object.entries(tool.inputSchema?.properties ?? {})) {
24
27
  const values = enumValues(schema);
25
28
  if (values.length >= BIG_ENUM)
26
29
  enums.push({ tool: tool.name, param, values: values.length, tokens: countTokens(JSON.stringify(values)) });
27
- const key = `${param}\u0000${canonical(schema)}`;
28
- const entry = seen.get(key) ?? { param, tokens: countTokens(JSON.stringify({ [param]: schema })), tools: new Set() };
29
- entry.tools.add(tool.name);
30
- seen.set(key, entry);
31
30
  }
32
31
  }
33
- const repeated = [...seen.values()]
34
- .filter((e) => e.tokens >= REPEATED_MIN_TOKENS && e.tools.size >= REPEATED_MIN_TOOLS)
35
- .map((e) => ({ param: e.param, tokens: e.tokens, tools: e.tools.size, total: e.tokens * e.tools.size }))
36
- .sort((a, b) => b.total - a.total)
37
- .slice(0, 5);
38
32
  enums.sort((a, b) => b.tokens - a.tokens);
39
- return { total: menu.totalTokens, top, enums: enums.slice(0, 5), repeated };
33
+ return { total: menu.totalTokens, top, enums: enums.slice(0, 5), repeated: repeatedBlocks(menu), unusedDefs: unusedDefs(menu) };
34
+ }
35
+ function unusedDefs(menu) {
36
+ let tokens = 0;
37
+ const tools = [];
38
+ for (const tool of menu.tools) {
39
+ const schema = (tool.inputSchema ?? {});
40
+ const pools = ['$defs', 'definitions'].filter((k) => schema[k] && typeof schema[k] === 'object');
41
+ if (!pools.length)
42
+ continue;
43
+ const { $defs: _d, definitions: _df, ...rest } = schema;
44
+ const used = new Set();
45
+ const queue = [JSON.stringify(rest)];
46
+ while (queue.length) {
47
+ for (const m of queue.pop().matchAll(/"\$ref"\s*:\s*"#\/(\$defs|definitions)\/([^"/]+)/g)) {
48
+ const id = `${m[1]}/${decodeURIComponent(m[2]).replace(/~1/g, '/').replace(/~0/g, '~')}`;
49
+ if (used.has(id))
50
+ continue;
51
+ used.add(id);
52
+ const [pool, name] = [m[1], id.slice(m[1].length + 1)];
53
+ queue.push(JSON.stringify(schema[pool][name] ?? {}));
54
+ }
55
+ }
56
+ let unused = 0;
57
+ for (const pool of pools) {
58
+ for (const [name, def] of Object.entries(schema[pool])) {
59
+ if (!used.has(`${pool}/${name}`))
60
+ unused += countTokens(JSON.stringify({ [name]: def }));
61
+ }
62
+ }
63
+ if (unused > 0) {
64
+ tokens += unused;
65
+ tools.push(tool.name);
66
+ }
67
+ }
68
+ return { tools: tools.length, tokens, examples: tools.slice(0, 3) };
69
+ }
70
+ /** Every sub-schema of every tool's inputSchema, fingerprinted, and the ones that repeat. */
71
+ function repeatedBlocks(menu) {
72
+ const places = new Map();
73
+ const tokens = new Map();
74
+ /** `map`: a name → schema map (properties, $defs), not a schema itself. */
75
+ const visit = (tool, node, path, map, ancestors) => {
76
+ if (!node || typeof node !== 'object')
77
+ return;
78
+ if (Array.isArray(node)) {
79
+ node.forEach((x, i) => visit(tool, x, `${path}[${i}]`, false, ancestors));
80
+ return;
81
+ }
82
+ let inner = ancestors;
83
+ if (path !== '' && !map) {
84
+ const key = canonical(node);
85
+ if (!tokens.has(key))
86
+ tokens.set(key, countTokens(JSON.stringify(node)));
87
+ places.set(key, [...(places.get(key) ?? []), { tool, path, ancestors }]);
88
+ inner = [...ancestors, key];
89
+ }
90
+ for (const [k, v] of Object.entries(node)) {
91
+ if (map)
92
+ visit(tool, v, `${path}.${k}`, false, inner);
93
+ else if (k === 'properties')
94
+ visit(tool, v, path, true, inner);
95
+ else if (k === '$defs' || k === 'definitions' || k === 'patternProperties')
96
+ visit(tool, v, `${path}.${k}`, true, inner);
97
+ else if (k === 'items' && !Array.isArray(v))
98
+ visit(tool, v, `${path}[]`, false, inner);
99
+ else
100
+ visit(tool, v, `${path}.${k}`, false, inner);
101
+ }
102
+ };
103
+ for (const tool of menu.tools)
104
+ visit(tool.name, tool.inputSchema, '', false, []);
105
+ const counts = new Map([...places].map(([k, occ]) => [k, occ.length]));
106
+ const out = [];
107
+ for (const [key, occ] of places) {
108
+ const size = tokens.get(key);
109
+ if (occ.length < 2 || size < REPEATED_MIN_TOKENS)
110
+ continue;
111
+ // Inside a bigger block that repeats as often: that one is the finding.
112
+ if (occ.every((o) => o.ancestors.some((a) => (tokens.get(a) ?? 0) >= REPEATED_MIN_TOKENS && (counts.get(a) ?? 0) >= occ.length)))
113
+ continue;
114
+ const perTool = new Map();
115
+ for (const o of occ)
116
+ perTool.set(o.tool, (perTool.get(o.tool) ?? 0) + 1);
117
+ const [withinTool, within] = [...perTool].sort((a, b) => b[1] - a[1])[0];
118
+ if (within < 2 && perTool.size < REPEATED_MIN_TOOLS)
119
+ continue;
120
+ const saving = [...perTool.values()].filter((k) => k > 1).reduce((s, k) => s + Math.max(0, (k - 1) * size - k * REF_TOKENS), 0);
121
+ // The nearest property name: `.freshness.anyOf[1]` is freshness.
122
+ const label = occ[0].path.split(/[.[\]]+/).filter((s) => s && !/^\d+$/.test(s) && !SCHEMA_KEYWORDS.has(s)).pop() ?? occ[0].path;
123
+ out.push({
124
+ param: label,
125
+ tokens: size,
126
+ count: occ.length,
127
+ tools: perTool.size,
128
+ total: size * occ.length,
129
+ within,
130
+ withinTool,
131
+ saving,
132
+ where: occ.slice(0, 3).map((o) => `${o.tool}${o.path}`),
133
+ waste: size * (occ.length - 1),
134
+ });
135
+ }
136
+ return out
137
+ .sort((a, b) => b.waste - a.waste)
138
+ .slice(0, 5)
139
+ .map(({ waste: _w, ...r }) => r);
40
140
  }
41
141
  function enumValues(schema) {
42
142
  if (Array.isArray(schema.enum))
@@ -45,19 +145,75 @@ function enumValues(schema) {
45
145
  return schema.items.enum;
46
146
  return [];
47
147
  }
148
+ const tok = (x) => `~${x.toLocaleString('en-US')}`;
149
+ const pct = (x) => `${Math.round(x * 100)}%`;
150
+ const list = (names, total) => `${names.join(', ')}${total > names.length ? ', …' : ''}`;
151
+ /** A tool's schema can't refer to another tool's: said once, after the repeated blocks that span tools. */
152
+ const ACROSS_TOOLS = "A tool's schema can't $ref another tool's, so a block shared across tools is paid once per tool: only a smaller block saves tokens there.";
153
+ function items(b, tools) {
154
+ const u = b.unusedDefs;
155
+ const unused = u.tokens
156
+ ? [{
157
+ kind: 'unused',
158
+ text: `unused $defs: ${tok(u.tokens)} tokens${b.total ? ` (${pct(u.tokens / b.total)} of the menu)` : ''} in ${u.tools} tool${u.tools === 1 ? '' : 's'} (${list(u.examples, u.tools)})`,
159
+ why: 'Definitions nothing in their tool refers to: sent with every conversation, read by no one.',
160
+ fix: "Remove the unreferenced $defs entries from each tool's inputSchema.",
161
+ }]
162
+ : [];
163
+ const repeated = b.repeated.map((r) => {
164
+ if (r.tools === 1) {
165
+ const places = list(r.where.map((w) => w.slice(r.withinTool.length + 1)), r.count);
166
+ return { kind: 'repeated', text: `repeated: one ${tok(r.tokens)}-token block ×${r.count} in ${r.withinTool} (${places})${r.saving > 0 ? `: a $defs entry could save ${tok(r.saving)}` : ''}`, ...(r.saving > 0 ? { fix: `Move the block into ${r.withinTool}'s $defs once and $ref it.` } : {}) };
167
+ }
168
+ const inside = r.within > 1 ? `, ×${r.within} inside ${r.withinTool}${r.saving > 0 ? `: a $defs entry there could save ${tok(r.saving)}` : ''}` : '';
169
+ return { kind: 'repeated', text: `repeated: ${r.param} (${tok(r.tokens)} tokens) in ${r.tools} tools, ${tok(r.total)} in all${inside}` };
170
+ });
171
+ // A small menu gets no breakdown, but waste inside one tool is worth a line anyway.
172
+ if (tools < 5) {
173
+ const within = [...unused, ...repeated.filter((_, i) => b.repeated[i].within > 1)];
174
+ return within.length ? { title: 'Where the tokens go (estimate):', items: within } : undefined;
175
+ }
176
+ const topShare = b.top.reduce((s, t) => s + t.share, 0);
177
+ const width = Math.min(40, Math.max(...b.top.map((t) => t.name.length)));
178
+ const top = b.top.map((t) => ({ kind: 'top', text: `${pct(t.share).padStart(4)} ${t.name.padEnd(width)} ${tok(t.tokens).padStart(7)} tokens (description ${tok(t.description)}, schema ${tok(t.schema)})` }));
179
+ const enums = b.enums.map((e) => ({ kind: 'enum', text: `enum: ${e.tool}.${e.param} has ${e.values} values, ${tok(e.tokens)} tokens` }));
180
+ const note = b.repeated.some((r) => r.tools > 1) ? [{ kind: 'note', text: ACROSS_TOOLS }] : [];
181
+ return { title: `Where the tokens go (estimate): the top ${b.top.length} tools are ${pct(topShare)} of the menu`, items: [...top, ...unused, ...enums, ...repeated, ...note] };
182
+ }
48
183
  /** The breakdown as report lines, or none for a menu too small to need one. */
49
184
  export function breakdownLines(b, tools) {
50
- if (tools < 5)
185
+ const found = items(b, tools);
186
+ if (!found)
51
187
  return [];
52
- const n = (x) => `~${x.toLocaleString('en-US')}`;
53
- const pct = (x) => `${Math.round(x * 100)}%`;
54
- const topShare = b.top.reduce((s, t) => s + t.share, 0);
55
- const lines = [`Where the tokens go (estimate): the top ${b.top.length} tools are ${pct(topShare)} of the menu`];
56
- for (const t of b.top)
57
- lines.push(` ${pct(t.share).padStart(4)} ${t.name} ${n(t.tokens)} (description ${n(t.description)}, schema ${n(t.schema)})`);
58
- for (const e of b.enums)
59
- lines.push(` enum: ${e.tool}.${e.param} has ${e.values} values, ${n(e.tokens)} tokens`);
60
- for (const r of b.repeated)
61
- lines.push(` repeated: ${r.param} (${n(r.tokens)}) in ${r.tools} tools, ${n(r.total)} in all`);
188
+ const lines = [found.title];
189
+ for (const item of found.items) {
190
+ lines.push(item.kind === 'note' ? ` (${item.text})` : ` ${item.text}`);
191
+ if (item.why)
192
+ lines.push(` ${item.why}`);
193
+ if (item.fix)
194
+ lines.push(` → Next: ${item.fix}`);
195
+ }
196
+ return lines;
197
+ }
198
+ /** The breakdown for a PR comment: a fold, the biggest tools as a table, the waste as a list. */
199
+ export function breakdownMarkdown(b, tools) {
200
+ const found = items(b, tools);
201
+ if (!found)
202
+ return [];
203
+ const cell = (t) => t.replace(/\|/g, '\\|');
204
+ const code = (t) => t.replace(/\$defs/g, '`$defs`');
205
+ const lines = ['', `<details><summary>${found.title}</summary>`, ''];
206
+ if (tools >= 5) {
207
+ lines.push('| Share | Tool | Tokens | Description | Schema |', '|---:|---|---:|---:|---:|');
208
+ for (const t of b.top)
209
+ lines.push(`| ${pct(t.share)} | \`${cell(t.name)}\` | ${tok(t.tokens)} | ${tok(t.description)} | ${tok(t.schema)} |`);
210
+ lines.push('');
211
+ }
212
+ for (const item of found.items.filter((i) => i.kind !== 'top' && i.kind !== 'note')) {
213
+ lines.push(`- ${code(item.text)}${item.why ? `. ${code(item.why)}` : ''}${item.fix ? `<br>**→ Next:** ${code(item.fix)}` : ''}`);
214
+ }
215
+ for (const item of found.items.filter((i) => i.kind === 'note'))
216
+ lines.push('', `_${code(item.text)}_`);
217
+ lines.push('', '</details>');
62
218
  return lines;
63
219
  }
package/dist/catalog.d.ts CHANGED
@@ -1,27 +1,55 @@
1
1
  import type { Connection } from './connect.js';
2
- import type { MenuTool } from './types.js';
2
+ import type { Finding, MenuTool } from './types.js';
3
3
  /**
4
- * Operations a server keeps behind a search tool instead of in its menu (search
5
- * and execute: Sentry's search_sentry_tools, Atlassian's discover). `diff` can't
6
- * see them in the menu, so snapshot --catalog asks the search tool a fixed set of
7
- * queries and keeps what comes back.
4
+ * Operations a server keeps out of its menu, where `diff` can't see them:
5
+ *
6
+ * - behind a search tool (search and execute: Sentry's search_sentry_tools,
7
+ * Atlassian's discover). snapshot --catalog asks it a fixed set of queries and
8
+ * crawls by the names that come back.
9
+ * - behind command routers: one tool per area that takes a `command` and its
10
+ * `parameters`, and lists its commands when called with a flag like `learn: true`
11
+ * (Azure's namespace mode). snapshot --catalog calls each router once, in that
12
+ * listing mode only, and never with a command.
13
+ *
14
+ * Both keep operations in the same shape, so `diff` compares them the same way.
8
15
  */
9
16
  export interface Catalog {
10
- /** The search tool asked. */
17
+ /** The search tool asked; for routers, the router (one) or "the command routers" (several). */
11
18
  tool: string;
12
- /** The queries asked, in order: the same ones every run, so runs compare. */
19
+ /** The queries asked, in order: the same ones every run, so runs compare. Empty for routers: see `routers`. */
13
20
  queries: string[];
14
- /** Every operation found, by name. Not complete: a search returns its top matches. */
21
+ /** Every operation found, by name. From a search, not complete: a search returns its top matches. */
15
22
  operations: MenuTool[];
16
- /** Queries that failed (rate limits, errors): the catalog is partial where they would have looked. */
23
+ /** Queries (or routers, by name) that failed: the catalog is partial where they would have looked. */
17
24
  failed?: {
18
25
  query: string;
19
26
  error: string;
20
27
  }[];
28
+ /** Routers only: each router called, in menu order, the exact arguments sent, and how many operations it listed. */
29
+ routers?: {
30
+ tool: string;
31
+ arguments: Record<string, unknown>;
32
+ operations: number;
33
+ duplicates?: {
34
+ command: string;
35
+ kept: string[];
36
+ }[];
37
+ }[];
38
+ /** Routers only: tools that look like routers but weren't called, and why. */
39
+ skipped?: {
40
+ tool: string;
41
+ reason: string;
42
+ }[];
21
43
  }
22
44
  export interface CatalogOptions {
23
45
  /** The search tool, when detection picks the wrong one. */
24
46
  tool?: string;
47
+ /**
48
+ * The command routers to read, by name, when detection misses them: every router
49
+ * to call (detection is then off). Each still needs a listing flag and an optional
50
+ * command, or it isn't called.
51
+ */
52
+ routers?: string[];
25
53
  /** The queries to ask. Default: derived from the menu. */
26
54
  queries?: string[];
27
55
  timeoutMs?: number;
@@ -42,8 +70,41 @@ export declare function findCatalogTool(tools: MenuTool[]): MenuTool | undefined
42
70
  * "update" and "delete".
43
71
  */
44
72
  export declare function defaultQueries(tools: MenuTool[], max?: number): string[];
45
- /** Ask the search tool every query and collect the operations it returns. */
73
+ /**
74
+ * Read the operations behind the menu: from the search tool (named, or detected),
75
+ * else from the command routers (named, or detected).
76
+ */
46
77
  export declare function readCatalog(conn: Connection, tools: MenuTool[], options?: CatalogOptions): Promise<Catalog>;
78
+ /** What goes in a required free-text parameter (Azure's `intent`): a request to list, nothing to act on. */
79
+ export declare const LISTING_PHRASE = "List the available commands and their parameters.";
80
+ export interface RouterCall {
81
+ tool: MenuTool;
82
+ /** The exact arguments sent: the listing flag and fixed phrases, never a command. */
83
+ arguments: Record<string, unknown>;
84
+ }
85
+ type Shape = {
86
+ call: RouterCall;
87
+ } | {
88
+ reason: string;
89
+ };
90
+ /**
91
+ * How to list a router's commands safely, or why it can't be. `named`: the config
92
+ * named it, so the description's wording isn't needed. Undefined: not a router.
93
+ */
94
+ export declare function routerShape(tool: MenuTool, named?: boolean): Shape | undefined;
95
+ /** The routers in a menu (menu order), and those that look like routers but can't be listed safely. */
96
+ export declare function findRouters(tools: MenuTool[]): {
97
+ usable: RouterCall[];
98
+ skipped: {
99
+ tool: string;
100
+ reason: string;
101
+ }[];
102
+ };
103
+ /**
104
+ * Findings about the read itself: what was called (catalog/read), what failed
105
+ * (catalog/failed, one per cause) and what was left alone (catalog/skipped).
106
+ */
107
+ export declare function catalogFindings(catalog: Catalog): Finding[];
47
108
  /**
48
109
  * Operation definitions in a search result: objects with a `name` and either a
49
110
  * JSON Schema (`inputSchema`, `input_schema`, `parameters`) or a list of
@@ -53,13 +114,28 @@ export declare function readCatalog(conn: Connection, tools: MenuTool[], options
53
114
  export declare function operationsIn(result: {
54
115
  structuredContent?: unknown;
55
116
  content?: unknown;
117
+ }, options?: {
118
+ nameKeys?: string[];
119
+ embedded?: boolean;
120
+ keepAll?: boolean;
56
121
  }): MenuTool[];
122
+ /** A router's answer that is a readable, empty command list: `[]`, or `{ "commands": [] }`. */
123
+ export declare function listsNothing(result: {
124
+ structuredContent?: unknown;
125
+ content?: unknown;
126
+ }): boolean;
57
127
  /**
58
128
  * The JSON value a text starts with, ignoring what follows: Atlassian's discover
59
129
  * returns its results as JSON and then a prose list of related operations, which
60
130
  * makes the whole text invalid JSON.
61
131
  */
62
132
  export declare function leadingJson(text: string): unknown;
133
+ /**
134
+ * The first JSON value in a text that opens with prose: Azure's routers answer
135
+ * "Here are the available commands… \n[{"command":…}]". The text itself when it
136
+ * starts with JSON; else each line that starts with [ or {, in order.
137
+ */
138
+ export declare function embeddedJson(text: string): unknown;
63
139
  /**
64
140
  * Operation names a result mentions without defining them: lines like
65
141
  * "createJiraBoard — Create a new company-managed…" after Atlassian's JSON. Each
@@ -68,3 +144,4 @@ export declare function leadingJson(text: string): unknown;
68
144
  export declare function mentionedNames(result: {
69
145
  content?: unknown;
70
146
  }): string[];
147
+ export {};