toolmenu 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -23
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +59 -21
- package/dist/auto.d.ts +80 -6
- package/dist/auto.js +255 -19
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +64 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +170 -185
- package/dist/close.d.ts +8 -0
- package/dist/close.js +18 -0
- package/dist/config.d.ts +7 -1
- package/dist/config.js +17 -0
- package/dist/connect.d.ts +14 -0
- package/dist/connect.js +150 -16
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +574 -120
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +113 -0
- package/dist/explain.js +461 -0
- package/dist/failures.d.ts +15 -0
- package/dist/failures.js +59 -2
- package/dist/help.d.ts +11 -0
- package/dist/help.js +205 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +93 -2
- package/dist/init.js +508 -21
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +141 -0
- package/dist/options.js +50 -0
- package/dist/report.d.ts +2 -2
- package/dist/report.js +305 -101
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +86 -42
- package/dist/rules/determinism.d.ts +4 -1
- package/dist/rules/determinism.js +175 -11
- package/dist/rules/gated.d.ts +7 -0
- package/dist/rules/gated.js +77 -0
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +18 -6
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +91 -21
- package/dist/rules/rule.d.ts +25 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.d.ts +6 -0
- package/dist/rules/spec.js +56 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +64 -14
- package/dist/session.d.ts +34 -9
- package/dist/session.js +525 -94
- package/dist/snapshot.d.ts +8 -2
- package/dist/snapshot.js +33 -2
- package/dist/types.d.ts +21 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +2 -2
package/dist/catalog.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { toolTokens } from './menu.js';
|
|
2
|
-
import { RATE_LIMITED, tooMany } from './failures.js';
|
|
2
|
+
import { classifyFailure, RATE_LIMITED, tooMany } from './failures.js';
|
|
3
3
|
import { nouns, singular } from './words.js';
|
|
4
4
|
const QUERY_PARAM = /^(q|query|search|keywords?|text)$/i;
|
|
5
5
|
const SEARCH_NAME = /(search|find|list|discover)[\w-]*(tools?|operations?|actions?|capabilit)|^discover$|(tools?|operations?)[\w-]*(search|discover)/i;
|
|
@@ -29,12 +29,42 @@ export function defaultQueries(tools, max = 25) {
|
|
|
29
29
|
queries.push(`${verb} ${noun}`);
|
|
30
30
|
return queries.slice(0, max);
|
|
31
31
|
}
|
|
32
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Read the operations behind the menu: from the search tool (named, or detected),
|
|
34
|
+
* else from the command routers (named, or detected).
|
|
35
|
+
*/
|
|
33
36
|
export async function readCatalog(conn, tools, options = {}) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
if (options.tool && options.routers?.length) {
|
|
38
|
+
throw new Error(`--catalog: the config names both a search tool (catalog.tool: ${options.tool}) and routers (catalog.routers), and a menu file keeps one catalog.\n→ Next: Keep the one the server's operations are behind, and remove the other from toolmenu.config.json.`);
|
|
39
|
+
}
|
|
40
|
+
if (options.routers?.length)
|
|
41
|
+
return readRouters(conn, namedRouters(tools, options.routers), [], options);
|
|
42
|
+
if (options.tool) {
|
|
43
|
+
const named = tools.find((t) => t.name === options.tool);
|
|
44
|
+
if (!named)
|
|
45
|
+
throw new Error(`--catalog: the config names ${options.tool} as the search tool (catalog.tool), but the menu has no tool by that name (${tools.length} tools listed).${nearby(options.tool, tools)}\n→ Next: Fix the name in toolmenu.config.json; the menu file from a plain snapshot lists every tool name.`);
|
|
46
|
+
return readSearch(conn, tools, named, options);
|
|
47
|
+
}
|
|
48
|
+
const search = findCatalogTool(tools);
|
|
49
|
+
if (search)
|
|
50
|
+
return readSearch(conn, tools, search, options);
|
|
51
|
+
const { usable, skipped } = findRouters(tools);
|
|
52
|
+
if (usable.length)
|
|
53
|
+
return readRouters(conn, usable, skipped, options);
|
|
54
|
+
if (skipped.length) {
|
|
55
|
+
throw new Error(`--catalog: ${plural(skipped.length, 'tool')} look${skipped.length === 1 ? 's' : ''} like command routers, but none can list its commands without being given one, so toolmenu called nothing.\n` +
|
|
56
|
+
skipped.map((s) => ` ${s.tool}: ${s.reason}`).join('\n') +
|
|
57
|
+
`\n→ Next: Nothing to read safely here; if a router gains a listing mode that needs no command, name it in catalog.routers.`);
|
|
58
|
+
}
|
|
59
|
+
throw new Error(`--catalog: nothing to read in this menu of ${plural(tools.length, 'tool')}: no catalog search tool (read-only, one required query, named like search_*_tools or discover) and no command router (a command parameter plus a listing flag like learn, described as a router).\n` +
|
|
60
|
+
`→ Next: If the operations are behind one of these tools, name it in toolmenu.config.json: "catalog": { "tool": "<search tool>" } or "catalog": { "routers": ["<router>"] }.`);
|
|
61
|
+
}
|
|
62
|
+
/** Ask the search tool every query and collect the operations it returns. */
|
|
63
|
+
async function readSearch(conn, tools, tool, options) {
|
|
64
|
+
const queryParam = tool.inputSchema?.required?.[0];
|
|
65
|
+
if (!queryParam || !tool.inputSchema?.properties?.[queryParam]) {
|
|
66
|
+
throw new Error(`--catalog: ${tool.name} is named as the search tool, but its schema has no required query parameter, so there is nothing to put a search in.\n→ Next: Name the tool that takes the search query in catalog.tool, or catalog.routers if ${tool.name} is a command router.`);
|
|
67
|
+
}
|
|
38
68
|
const limit = limitArg(tool.inputSchema);
|
|
39
69
|
// Seeds, then the catalog's own words: every operation found or mentioned is
|
|
40
70
|
// searched for by name, which finds it and its neighbours. Breadth first, in the
|
|
@@ -63,48 +93,343 @@ export async function readCatalog(conn, tools, options = {}) {
|
|
|
63
93
|
if (asked.length > 0)
|
|
64
94
|
await sleep(pause);
|
|
65
95
|
asked.push(query);
|
|
66
|
-
// A rate limit is waited out
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
77
|
-
if (result.isError)
|
|
78
|
-
failed.push({ query, error: text.slice(0, 200) || 'the tool returned an error' });
|
|
79
|
-
else {
|
|
80
|
-
const before = found.size;
|
|
81
|
-
for (const op of operationsIn(result)) {
|
|
82
|
-
if (found.has(op.name))
|
|
83
|
-
continue;
|
|
84
|
-
found.set(op.name, op);
|
|
85
|
-
enqueue(op.name);
|
|
86
|
-
}
|
|
87
|
-
for (const name of mentionedNames(result))
|
|
88
|
-
if (!found.has(name))
|
|
89
|
-
enqueue(name);
|
|
90
|
-
dry = found.size > before ? 0 : dry + 1;
|
|
91
|
-
}
|
|
92
|
-
break;
|
|
93
|
-
}
|
|
94
|
-
catch (error) {
|
|
95
|
-
const message = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
96
|
-
if (tooMany(error) && attempt < 3) {
|
|
97
|
-
await sleep(2000 * 2 ** attempt);
|
|
96
|
+
// A rate limit is waited out; anything else, or a limit that persists, leaves
|
|
97
|
+
// this query out and the catalog partial, never the snapshot failed: diff
|
|
98
|
+
// treats operations not found as notices.
|
|
99
|
+
const answer = await callPatiently(conn, tool.name, { [queryParam]: query, ...limit }, options.timeoutMs);
|
|
100
|
+
if ('error' in answer)
|
|
101
|
+
failed.push({ query, error: answer.error });
|
|
102
|
+
else {
|
|
103
|
+
const before = found.size;
|
|
104
|
+
for (const op of operationsIn(answer.result)) {
|
|
105
|
+
if (found.has(op.name))
|
|
98
106
|
continue;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
break;
|
|
107
|
+
found.set(op.name, op);
|
|
108
|
+
enqueue(op.name);
|
|
102
109
|
}
|
|
110
|
+
for (const name of mentionedNames(answer.result))
|
|
111
|
+
if (!found.has(name))
|
|
112
|
+
enqueue(name);
|
|
113
|
+
dry = found.size > before ? 0 : dry + 1;
|
|
103
114
|
}
|
|
104
115
|
}
|
|
105
116
|
const operations = [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
106
117
|
return { tool: tool.name, queries: asked, operations, ...(failed.length ? { failed } : {}) };
|
|
107
118
|
}
|
|
119
|
+
// ── Command routers ────────────────────────────────────────────────────────────
|
|
120
|
+
//
|
|
121
|
+
// A router is one tool that fronts many commands: `{ command, parameters }` runs
|
|
122
|
+
// one, a boolean flag lists them all (Azure: "This tool is a hierarchical MCP
|
|
123
|
+
// command router… Set "learn=true" to discover available sub commands."). The
|
|
124
|
+
// heuristic, all four required for detection:
|
|
125
|
+
// 1. a string parameter named command (or subcommand, operation);
|
|
126
|
+
// 2. an object parameter for its arguments (parameters, params, arguments, args);
|
|
127
|
+
// 3. a boolean listing flag (learn, discover, help, list_commands);
|
|
128
|
+
// 4. the tool's or the flag's description says so: "router", "sub commands",
|
|
129
|
+
// "child tools".
|
|
130
|
+
// catalog.routers names routers detection misses; they need 1 and 3.
|
|
131
|
+
//
|
|
132
|
+
// Safety: the only call made is the listing call, `{ <flag>: true }` plus a fixed
|
|
133
|
+
// "list the commands" phrase in any other required free-text parameter. It never
|
|
134
|
+
// carries the command or its arguments, so there is nothing for the router to run.
|
|
135
|
+
// That holds whatever the annotations say: a router's annotations describe the
|
|
136
|
+
// commands behind it (Azure's carry no hints at all), not its listing mode. A router
|
|
137
|
+
// whose command or arguments are required, or that requires anything toolmenu would
|
|
138
|
+
// have to guess (an enum, a number), isn't called.
|
|
139
|
+
const COMMAND_PARAM = /^(command|sub_?command|operation)$/i;
|
|
140
|
+
const ARGS_PARAM = /^(parameters|params|arguments|args)$/i;
|
|
141
|
+
const LISTING_FLAG = /^(learn|discover|help|list_?commands)$/i;
|
|
142
|
+
const ROUTER_WORDS = /\brouter\b|\bsub[- ]?commands?\b|\bchild (tools|commands)\b/i;
|
|
143
|
+
/** What goes in a required free-text parameter (Azure's `intent`): a request to list, nothing to act on. */
|
|
144
|
+
export const LISTING_PHRASE = 'List the available commands and their parameters.';
|
|
145
|
+
function hasType(schema, type) {
|
|
146
|
+
const t = schema?.type;
|
|
147
|
+
return Array.isArray(t) ? t.includes(type) : t === type;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* How to list a router's commands safely, or why it can't be. `named`: the config
|
|
151
|
+
* named it, so the description's wording isn't needed. Undefined: not a router.
|
|
152
|
+
*/
|
|
153
|
+
export function routerShape(tool, named = false) {
|
|
154
|
+
const props = tool.inputSchema?.properties ?? {};
|
|
155
|
+
const keys = Object.keys(props);
|
|
156
|
+
const command = keys.find((k) => COMMAND_PARAM.test(k) && hasType(props[k], 'string'));
|
|
157
|
+
const args = keys.find((k) => ARGS_PARAM.test(k) && hasType(props[k], 'object'));
|
|
158
|
+
const flag = keys.find((k) => LISTING_FLAG.test(k) && hasType(props[k], 'boolean'));
|
|
159
|
+
const words = ROUTER_WORDS.test(`${tool.description ?? ''} ${flag ? String(props[flag].description ?? '') : ''}`);
|
|
160
|
+
if (!named && !(command && args && flag && words))
|
|
161
|
+
return undefined;
|
|
162
|
+
if (!flag)
|
|
163
|
+
return { reason: `it has no boolean listing flag (learn, discover, help or list_commands), so there's no way to ask for its commands without running one` };
|
|
164
|
+
if (!command)
|
|
165
|
+
return { reason: `it has no command parameter (command, subcommand or operation), so it doesn't look like a router to call in listing mode` };
|
|
166
|
+
const required = tool.inputSchema?.required ?? [];
|
|
167
|
+
const arguments_ = { [flag]: true };
|
|
168
|
+
for (const r of required) {
|
|
169
|
+
if (r === flag)
|
|
170
|
+
continue;
|
|
171
|
+
// By name, whatever the type: a required `args` string or a second command
|
|
172
|
+
// (`subcommand` beside `command`) is still a command or its arguments.
|
|
173
|
+
if (COMMAND_PARAM.test(r) || ARGS_PARAM.test(r))
|
|
174
|
+
return { reason: `its ${r} parameter is required, so even a listing call would have to name ${COMMAND_PARAM.test(r) ? 'a command' : 'arguments for one'}` };
|
|
175
|
+
const p = props[r];
|
|
176
|
+
if (!p || !hasType(p, 'string') || p.enum || p.const !== undefined || p.pattern || p.format) {
|
|
177
|
+
return { reason: `it requires ${r}, which isn't free text, and toolmenu won't guess a value in a call to a tool that runs commands` };
|
|
178
|
+
}
|
|
179
|
+
arguments_[r] = LISTING_PHRASE;
|
|
180
|
+
}
|
|
181
|
+
return { call: { tool, arguments: arguments_ } };
|
|
182
|
+
}
|
|
183
|
+
/** The routers in a menu (menu order), and those that look like routers but can't be listed safely. */
|
|
184
|
+
export function findRouters(tools) {
|
|
185
|
+
const usable = [];
|
|
186
|
+
const skipped = [];
|
|
187
|
+
for (const t of tools) {
|
|
188
|
+
const shape = routerShape(t);
|
|
189
|
+
if (!shape)
|
|
190
|
+
continue;
|
|
191
|
+
if ('call' in shape)
|
|
192
|
+
usable.push(shape.call);
|
|
193
|
+
else
|
|
194
|
+
skipped.push({ tool: t.name, reason: shape.reason });
|
|
195
|
+
}
|
|
196
|
+
return { usable, skipped };
|
|
197
|
+
}
|
|
198
|
+
function namedRouters(tools, names) {
|
|
199
|
+
return [...new Set(names)].map((name) => {
|
|
200
|
+
const t = tools.find((x) => x.name === name);
|
|
201
|
+
if (!t)
|
|
202
|
+
throw new Error(`--catalog: the config names ${name} as a command router (catalog.routers), but the menu has no tool by that name (${plural(tools.length, 'tool')} listed).${nearby(name, tools)}\n→ Next: Fix the name in toolmenu.config.json; the menu file from a plain snapshot lists every tool name.`);
|
|
203
|
+
const shape = routerShape(t, true);
|
|
204
|
+
if ('reason' in shape)
|
|
205
|
+
throw new Error(`--catalog: ${name} is named as a command router (catalog.routers), but toolmenu won't call it: ${shape.reason}.\n→ Next: Remove ${name} from catalog.routers; if its operations sit behind a search tool instead, name that in catalog.tool.`);
|
|
206
|
+
return shape.call;
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Call each router once in its listing mode, paced like the search crawl, and keep
|
|
211
|
+
* the operations it lists. A router that fails leaves its operations out, never the
|
|
212
|
+
* snapshot failed.
|
|
213
|
+
*/
|
|
214
|
+
async function readRouters(conn, routers, skipped, options) {
|
|
215
|
+
const pause = options.pauseMs ?? 300;
|
|
216
|
+
const found = new Map();
|
|
217
|
+
const failed = [];
|
|
218
|
+
const called = [];
|
|
219
|
+
for (const [i, { tool, arguments: args }] of routers.entries()) {
|
|
220
|
+
if (i > 0)
|
|
221
|
+
await sleep(pause);
|
|
222
|
+
// The listing call. Checked here too, so no change above can make it run a
|
|
223
|
+
// command: routerShape never lets such a call through, so reaching this is a bug.
|
|
224
|
+
const carried = Object.keys(args).find((k) => COMMAND_PARAM.test(k) || ARGS_PARAM.test(k) || (args[k] !== true && args[k] !== LISTING_PHRASE));
|
|
225
|
+
if (carried)
|
|
226
|
+
throw new Error(`--catalog: stopped before calling ${tool.name}: its listing call would have carried ${carried}, and a listing call only ever carries the flag and the fixed phrase. Nothing was sent. This is a toolmenu bug.\n→ Next: Report it at https://github.com/niksa90/toolmenu/issues with the tool's inputSchema.`);
|
|
227
|
+
const answer = await callPatiently(conn, tool.name, args, options.timeoutMs);
|
|
228
|
+
let ops = [];
|
|
229
|
+
if ('error' in answer)
|
|
230
|
+
failed.push({ query: tool.name, error: answer.error });
|
|
231
|
+
else {
|
|
232
|
+
// Keyed by command first: it's what an agent passes. Every definition is
|
|
233
|
+
// kept, even two under one name.
|
|
234
|
+
ops = operationsIn(answer.result, { nameKeys: ['command', 'name'], embedded: true, keepAll: true });
|
|
235
|
+
if (!ops.length && !listsNothing(answer.result))
|
|
236
|
+
failed.push({ query: tool.name, error: `${UNREADABLE} toolmenu could read: ${quote(resultText(answer.result))}` });
|
|
237
|
+
}
|
|
238
|
+
// Every operation is kept as router.command, however many routers there are
|
|
239
|
+
// and whichever answered this run, so its name is the same from run to run
|
|
240
|
+
// and a router that fails doesn't rename another's operations. A command a
|
|
241
|
+
// router lists twice with different definitions is kept twice, the second
|
|
242
|
+
// as router.command#2, and said so.
|
|
243
|
+
const seen = new Map();
|
|
244
|
+
const duplicates = new Map();
|
|
245
|
+
for (const op of ops) {
|
|
246
|
+
let name = `${tool.name}.${op.name}`;
|
|
247
|
+
const n = (seen.get(op.name) ?? 0) + 1;
|
|
248
|
+
seen.set(op.name, n);
|
|
249
|
+
if (n > 1) {
|
|
250
|
+
let k = n;
|
|
251
|
+
while (found.has(`${name}#${k}`))
|
|
252
|
+
k++;
|
|
253
|
+
name = `${name}#${k}`;
|
|
254
|
+
duplicates.set(op.name, [...(duplicates.get(op.name) ?? [`${tool.name}.${op.name}`]), name]);
|
|
255
|
+
}
|
|
256
|
+
found.set(name, { ...op, name, tokens: toolTokens({ ...op, name }) });
|
|
257
|
+
}
|
|
258
|
+
called.push({ tool: tool.name, arguments: args, operations: ops.length, ...(duplicates.size ? { duplicates: [...duplicates].map(([command, kept]) => ({ command, kept })) } : {}) });
|
|
259
|
+
}
|
|
260
|
+
const operations = [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
261
|
+
return {
|
|
262
|
+
tool: routers.length === 1 ? routers[0].tool.name : 'the command routers',
|
|
263
|
+
queries: [],
|
|
264
|
+
operations,
|
|
265
|
+
...(failed.length ? { failed } : {}),
|
|
266
|
+
routers: called,
|
|
267
|
+
...(skipped.length ? { skipped } : {}),
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
/** A tool call, sent again after a wait while it's refused for being too many (2, 4, 8 s). */
|
|
271
|
+
async function callPatiently(conn, name, args, timeoutMs = 30_000) {
|
|
272
|
+
for (let attempt = 0;; attempt++) {
|
|
273
|
+
try {
|
|
274
|
+
const result = await conn.client.callTool({ name, arguments: args }, { timeout: timeoutMs });
|
|
275
|
+
const text = result.isError ? resultText(result) : '';
|
|
276
|
+
if (result.isError && RATE_LIMITED.test(text) && attempt < 3) {
|
|
277
|
+
await sleep(2000 * 2 ** attempt);
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
return result.isError ? { error: text.slice(0, 200) || 'the tool returned an error' } : { result };
|
|
281
|
+
}
|
|
282
|
+
catch (error) {
|
|
283
|
+
if (tooMany(error) && attempt < 3) {
|
|
284
|
+
await sleep(2000 * 2 ** attempt);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
return { error: (error instanceof Error ? error.message.split('\n')[0] : String(error)).slice(0, 200) };
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const FAILED_AS = {
|
|
292
|
+
auth: {
|
|
293
|
+
what: ['wanted credentials before it would list its commands', 'wanted credentials before they would list their commands'],
|
|
294
|
+
fix: "Rerun with the server's credentials (--env for a stdio server, --header for HTTP); if these routers aren't yours to watch, ignore this.",
|
|
295
|
+
},
|
|
296
|
+
environment: {
|
|
297
|
+
what: ["needs something this machine doesn't have", "need something this machine doesn't have"],
|
|
298
|
+
fix: 'Install what the error names where toolmenu starts the server (the same container or CI image), then rerun.',
|
|
299
|
+
},
|
|
300
|
+
network: {
|
|
301
|
+
what: ["couldn't reach the network", "couldn't reach the network"],
|
|
302
|
+
fix: 'Rerun where the server can reach its upstream services.',
|
|
303
|
+
},
|
|
304
|
+
'not-found': {
|
|
305
|
+
what: ['answered "not found"', 'answered "not found"'],
|
|
306
|
+
fix: 'Check the server is set up for this account or site, then rerun.',
|
|
307
|
+
},
|
|
308
|
+
'invalid-arguments': {
|
|
309
|
+
what: ["rejected the listing call's arguments", "rejected the listing call's arguments"],
|
|
310
|
+
fix: "Read the router's own words above; if it lists its commands another way, name the right tool in catalog.tool or catalog.routers.",
|
|
311
|
+
},
|
|
312
|
+
unreadable: {
|
|
313
|
+
what: ['answered, but not with a command list toolmenu could read', 'answered, but not with a command list toolmenu could read'],
|
|
314
|
+
fix: 'If the router lists its commands in another format, open an issue at https://github.com/niksa90/toolmenu/issues with its answer; toolmenu reads JSON objects with a name or command and an inputSchema.',
|
|
315
|
+
},
|
|
316
|
+
other: {
|
|
317
|
+
what: ['answered the listing call with an error', 'answered the listing call with an error'],
|
|
318
|
+
fix: "Read the server's words above and rerun; each failure leaves out only that router's part of the catalog.",
|
|
319
|
+
},
|
|
320
|
+
};
|
|
321
|
+
// Credentials in words failures.ts doesn't know: Azure's "ChainedTokenCredential
|
|
322
|
+
// failed to retrieve a token from the included credentials".
|
|
323
|
+
const CREDENTIALS = /\bcredentials?\b|failed to (retrieve|acquire|get|obtain) (a |an )?(access )?token|\b(sign|log)[ -]?in (is )?required\b|az(ure)? login/i;
|
|
324
|
+
const UNREADABLE = 'answered without a command list';
|
|
325
|
+
function failureClass(error) {
|
|
326
|
+
if (error.startsWith(UNREADABLE))
|
|
327
|
+
return 'unreadable';
|
|
328
|
+
const c = classifyFailure(error);
|
|
329
|
+
return c === 'other' && CREDENTIALS.test(error) ? 'auth' : c;
|
|
330
|
+
}
|
|
331
|
+
/** A server's error on one line, at most 160 characters. */
|
|
332
|
+
function oneLine(text) {
|
|
333
|
+
const t = text.replace(/\s+/g, ' ').trim();
|
|
334
|
+
return t.length > 160 ? t.slice(0, 159) + '…' : t;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Findings about the read itself: what was called (catalog/read), what failed
|
|
338
|
+
* (catalog/failed, one per cause) and what was left alone (catalog/skipped).
|
|
339
|
+
*/
|
|
340
|
+
export function catalogFindings(catalog) {
|
|
341
|
+
const out = [];
|
|
342
|
+
const n = catalog.operations.length;
|
|
343
|
+
const failed = catalog.failed ?? [];
|
|
344
|
+
if (!catalog.routers) {
|
|
345
|
+
out.push({
|
|
346
|
+
rule: 'catalog/read',
|
|
347
|
+
severity: 'info',
|
|
348
|
+
tool: catalog.tool,
|
|
349
|
+
message: `Read ${plural(n, 'operation')} behind ${catalog.tool} with ${plural(catalog.queries.length, 'search', 'searches')}, saved in the menu file for diff. A search returns its top matches, so this is what the searches reached, not a guaranteed full list.`,
|
|
350
|
+
});
|
|
351
|
+
if (failed.length) {
|
|
352
|
+
out.push({
|
|
353
|
+
rule: 'catalog/failed',
|
|
354
|
+
severity: 'info',
|
|
355
|
+
tool: catalog.tool,
|
|
356
|
+
message: `${plural(failed.length, 'search', 'searches')} of ${catalog.queries.length} failed, so the catalog is partial where they would have looked.`,
|
|
357
|
+
detail: failed.slice(0, 5).map((f) => `"${f.query}": ${oneLine(f.error)}`).concat(failed.length > 5 ? [`… and ${failed.length - 5} more`] : []),
|
|
358
|
+
fix: tooManyIn(failed) ? 'Rerun later, or raise catalog.pauseMs so the searches stay under the rate limit.' : 'Rerun; if the same searches fail again, the server\'s words above say why.',
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
return out;
|
|
362
|
+
}
|
|
363
|
+
const routers = catalog.routers;
|
|
364
|
+
const listing = routers.map((r) => JSON.stringify(r.arguments));
|
|
365
|
+
const oneCall = listing.every((a) => a === listing[0]);
|
|
366
|
+
out.push({
|
|
367
|
+
rule: 'catalog/read',
|
|
368
|
+
severity: 'info',
|
|
369
|
+
...(routers.length === 1 ? { tool: routers[0].tool } : {}),
|
|
370
|
+
message: `Read ${plural(n, 'operation')} behind ${routers.length === 1 ? `the command router ${routers[0].tool}` : `${routers.length} command routers`}, saved in the menu file for diff. Each router was called once, in its listing mode, and never with a command.`,
|
|
371
|
+
detail: [
|
|
372
|
+
oneCall ? `Called with ${listing[0]}` : `Called with: ${routers.map((r, i) => `${r.tool} ${listing[i]}`).join('; ')}`,
|
|
373
|
+
listedLine(routers),
|
|
374
|
+
...routers.flatMap((r) => (r.duplicates ?? []).map((d) => `${r.tool} listed ${d.command} ${d.kept.length === 2 ? 'twice' : `${d.kept.length} times`}, with different definitions; kept each, as ${d.kept.join(', ')}`)),
|
|
375
|
+
],
|
|
376
|
+
});
|
|
377
|
+
const byClass = new Map();
|
|
378
|
+
for (const f of failed) {
|
|
379
|
+
const c = failureClass(f.error);
|
|
380
|
+
byClass.set(c, [...(byClass.get(c) ?? []), f]);
|
|
381
|
+
}
|
|
382
|
+
for (const [c, fs] of byClass) {
|
|
383
|
+
const verb = FAILED_AS[c].what[fs.length === 1 ? 0 : 1];
|
|
384
|
+
out.push({
|
|
385
|
+
rule: 'catalog/failed',
|
|
386
|
+
severity: 'info',
|
|
387
|
+
...(fs.length === 1 ? { tool: fs[0].query } : {}),
|
|
388
|
+
message: `${plural(fs.length, 'router')} ${verb}: ${fs.map((f) => f.query).join(', ')}. ${fs.length === 1 ? 'Its' : 'Their'} operations aren't in the catalog, so diff can't see them.`,
|
|
389
|
+
detail: fs.map((f) => `${f.query}: ${oneLine(f.error)}`),
|
|
390
|
+
fix: FAILED_AS[c].fix,
|
|
391
|
+
// The cause is read from the server's words, which can mislead.
|
|
392
|
+
...(c === 'other' || c === 'unreadable' ? {} : { confidence: 'unsure' }),
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
if (catalog.skipped?.length) {
|
|
396
|
+
out.push({
|
|
397
|
+
rule: 'catalog/skipped',
|
|
398
|
+
severity: 'info',
|
|
399
|
+
message: `${plural(catalog.skipped.length, 'tool')} look${catalog.skipped.length === 1 ? 's' : ''} like command routers but weren't called, because listing ${catalog.skipped.length === 1 ? 'its' : 'their'} commands safely isn't possible: ${catalog.skipped.map((s) => s.tool).join(', ')}.`,
|
|
400
|
+
detail: catalog.skipped.map((s) => `${s.tool}: ${s.reason}`),
|
|
401
|
+
fix: 'If their operations matter, ask the server for a listing mode that needs no command (toolmenu never sends one), then name them in catalog.routers.',
|
|
402
|
+
confidence: 'unsure',
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
return out;
|
|
406
|
+
}
|
|
407
|
+
/** Which routers listed the most, and how many listed anything. */
|
|
408
|
+
function listedLine(routers) {
|
|
409
|
+
const some = routers.filter((r) => r.operations > 0);
|
|
410
|
+
if (!some.length)
|
|
411
|
+
return 'Listed: nothing';
|
|
412
|
+
const top = [...some].sort((a, b) => b.operations - a.operations || a.tool.localeCompare(b.tool)).slice(0, 8);
|
|
413
|
+
const rest = some.length - top.length;
|
|
414
|
+
return `Listed: ${top.map((r) => `${r.tool} ${r.operations}`).join(', ')}${rest ? `, and ${rest} more ${rest === 1 ? 'router' : 'routers'} (each router's count is in the menu file)` : ''}`;
|
|
415
|
+
}
|
|
416
|
+
function tooManyIn(failed) {
|
|
417
|
+
return failed.some((f) => RATE_LIMITED.test(f.error));
|
|
418
|
+
}
|
|
419
|
+
function plural(n, one, many = `${one}s`) {
|
|
420
|
+
return `${n.toLocaleString('en-US')} ${n === 1 ? one : many}`;
|
|
421
|
+
}
|
|
422
|
+
/** The first 120 characters of a text, quoted. */
|
|
423
|
+
function quote(text) {
|
|
424
|
+
const t = text.replace(/\s+/g, ' ').trim();
|
|
425
|
+
return t ? `“${t.length > 120 ? `${t.slice(0, 119)}…` : t}”` : '(an empty answer)';
|
|
426
|
+
}
|
|
427
|
+
/** " Did you mean x?" when a tool's name is one letter case or separator away. */
|
|
428
|
+
function nearby(name, tools) {
|
|
429
|
+
const key = (s) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
430
|
+
const hit = tools.find((t) => key(t.name) === key(name));
|
|
431
|
+
return hit ? ` Did you mean ${hit.name}?` : '';
|
|
432
|
+
}
|
|
108
433
|
/** An operation name as a search phrase: listJiraIssueWorklogs → "list jira issue worklogs". */
|
|
109
434
|
function phrase(name) {
|
|
110
435
|
return name
|
|
@@ -146,27 +471,19 @@ function limitArg(schema) {
|
|
|
146
471
|
* `inputs` (Atlassian), found anywhere in structuredContent or in text that
|
|
147
472
|
* parses as JSON.
|
|
148
473
|
*/
|
|
149
|
-
export function operationsIn(result) {
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
154
|
-
const text = part.text;
|
|
155
|
-
if (typeof text !== 'string')
|
|
156
|
-
continue;
|
|
157
|
-
const value = leadingJson(text);
|
|
158
|
-
if (value !== undefined)
|
|
159
|
-
roots.push(value);
|
|
160
|
-
}
|
|
474
|
+
export function operationsIn(result, options = {}) {
|
|
475
|
+
const nameKeys = options.nameKeys ?? ['name'];
|
|
476
|
+
const roots = jsonRoots(result, options.embedded);
|
|
477
|
+
// keepAll: two definitions under one name are both kept (the same one twice, once).
|
|
161
478
|
const out = new Map();
|
|
162
479
|
const walk = (v, depth) => {
|
|
163
480
|
if (depth > 6 || !v || typeof v !== 'object')
|
|
164
481
|
return;
|
|
165
482
|
if (Array.isArray(v))
|
|
166
483
|
return v.forEach((x) => walk(x, depth + 1));
|
|
167
|
-
const op = asOperation(v);
|
|
484
|
+
const op = asOperation(v, nameKeys);
|
|
168
485
|
if (op) {
|
|
169
|
-
out.set(op.name, op);
|
|
486
|
+
out.set(options.keepAll ? JSON.stringify(op) : op.name, op);
|
|
170
487
|
return;
|
|
171
488
|
}
|
|
172
489
|
for (const x of Object.values(v))
|
|
@@ -175,6 +492,26 @@ export function operationsIn(result) {
|
|
|
175
492
|
roots.forEach((r) => walk(r, 0));
|
|
176
493
|
return [...out.values()];
|
|
177
494
|
}
|
|
495
|
+
/** The JSON in a result: its structuredContent, and each text part's. */
|
|
496
|
+
function jsonRoots(result, embedded = false) {
|
|
497
|
+
const roots = [];
|
|
498
|
+
if (result.structuredContent)
|
|
499
|
+
roots.push(result.structuredContent);
|
|
500
|
+
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
501
|
+
const text = part.text;
|
|
502
|
+
if (typeof text !== 'string')
|
|
503
|
+
continue;
|
|
504
|
+
const value = embedded ? embeddedJson(text) : leadingJson(text);
|
|
505
|
+
if (value !== undefined)
|
|
506
|
+
roots.push(value);
|
|
507
|
+
}
|
|
508
|
+
return roots;
|
|
509
|
+
}
|
|
510
|
+
/** A router's answer that is a readable, empty command list: `[]`, or `{ "commands": [] }`. */
|
|
511
|
+
export function listsNothing(result) {
|
|
512
|
+
const empty = (v) => Array.isArray(v) && v.length === 0;
|
|
513
|
+
return jsonRoots(result, true).some((r) => empty(r) || (!!r && typeof r === 'object' && !Array.isArray(r) && Object.keys(r).length > 0 && Object.values(r).every(empty)));
|
|
514
|
+
}
|
|
178
515
|
/**
|
|
179
516
|
* The JSON value a text starts with, ignoring what follows: Atlassian's discover
|
|
180
517
|
* returns its results as JSON and then a prose list of related operations, which
|
|
@@ -199,6 +536,25 @@ export function leadingJson(text) {
|
|
|
199
536
|
}
|
|
200
537
|
}
|
|
201
538
|
}
|
|
539
|
+
/**
|
|
540
|
+
* The first JSON value in a text that opens with prose: Azure's routers answer
|
|
541
|
+
* "Here are the available commands… \n[{"command":…}]". The text itself when it
|
|
542
|
+
* starts with JSON; else each line that starts with [ or {, in order.
|
|
543
|
+
*/
|
|
544
|
+
export function embeddedJson(text) {
|
|
545
|
+
const leading = leadingJson(text);
|
|
546
|
+
if (leading !== undefined)
|
|
547
|
+
return leading;
|
|
548
|
+
let tries = 0;
|
|
549
|
+
for (const m of text.matchAll(/\n[ \t]*(?=[[{])/g)) {
|
|
550
|
+
if (++tries > 50)
|
|
551
|
+
break;
|
|
552
|
+
const value = leadingJson(text.slice(m.index + m[0].length));
|
|
553
|
+
if (value !== undefined && typeof value === 'object')
|
|
554
|
+
return value;
|
|
555
|
+
}
|
|
556
|
+
return undefined;
|
|
557
|
+
}
|
|
202
558
|
/**
|
|
203
559
|
* Operation names a result mentions without defining them: lines like
|
|
204
560
|
* "createJiraBoard — Create a new company-managed…" after Atlassian's JSON. Each
|
|
@@ -215,8 +571,9 @@ export function mentionedNames(result) {
|
|
|
215
571
|
}
|
|
216
572
|
return [...names];
|
|
217
573
|
}
|
|
218
|
-
function asOperation(o) {
|
|
219
|
-
|
|
574
|
+
function asOperation(o, nameKeys = ['name']) {
|
|
575
|
+
const name = nameKeys.map((k) => o[k]).find((v) => typeof v === 'string' && v.length > 0);
|
|
576
|
+
if (!name)
|
|
220
577
|
return undefined;
|
|
221
578
|
const schema = (o.inputSchema ?? o.input_schema ?? o.parameters);
|
|
222
579
|
let inputSchema;
|
|
@@ -226,7 +583,7 @@ function asOperation(o) {
|
|
|
226
583
|
inputSchema = fromInputs(o.inputs);
|
|
227
584
|
else
|
|
228
585
|
return undefined;
|
|
229
|
-
const def = { name
|
|
586
|
+
const def = { name, ...(typeof o.description === 'string' ? { description: o.description } : {}), inputSchema };
|
|
230
587
|
if (o.annotations && typeof o.annotations === 'object')
|
|
231
588
|
def.annotations = o.annotations;
|
|
232
589
|
return { ...def, tokens: toolTokens(def) };
|