toolmenu 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/catalog.js +4 -0
- package/dist/diff.js +5 -1
- package/dist/explain.d.ts +8 -0
- package/dist/explain.js +51 -7
- package/dist/failures.d.ts +1 -1
- package/dist/failures.js +18 -6
- package/dist/rules/gated.js +4 -1
- package/dist/rules/ids.js +2 -1
- package/dist/rules/index.js +2 -0
- package/dist/rules/search.d.ts +2 -0
- package/dist/rules/search.js +70 -0
- package/dist/search.d.ts +24 -0
- package/dist/search.js +86 -0
- package/dist/session.d.ts +5 -0
- package/dist/session.js +25 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
|
|
|
29
29
|
(`history` installs old versions with today's dependencies, so it records what they
|
|
30
30
|
serve now, which can differ from what they shipped with: FINDINGS F4.)
|
|
31
31
|
|
|
32
|
-
> **Status: 0.
|
|
32
|
+
> **Status: 0.15, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
|
|
33
33
|
> real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
34
34
|
|
|
35
35
|
## Start here
|
|
@@ -104,7 +104,7 @@ jobs:
|
|
|
104
104
|
- uses: actions/setup-node@v7
|
|
105
105
|
with: { node-version: 22 }
|
|
106
106
|
- run: npm ci && npm run build
|
|
107
|
-
- uses: niksa90/toolmenu@v0.
|
|
107
|
+
- uses: niksa90/toolmenu@v0.15.0
|
|
108
108
|
with:
|
|
109
109
|
command: node dist/server.js
|
|
110
110
|
baseline: menu.json # your committed snapshot
|
package/dist/catalog.js
CHANGED
|
@@ -305,6 +305,10 @@ const FAILED_AS = {
|
|
|
305
305
|
what: ['answered "not found"', 'answered "not found"'],
|
|
306
306
|
fix: 'Check the server is set up for this account or site, then rerun.',
|
|
307
307
|
},
|
|
308
|
+
client: {
|
|
309
|
+
what: ["asked the client for something toolmenu doesn't offer (roots, sampling or elicitation)", "asked the client for something toolmenu doesn't offer (roots, sampling or elicitation)"],
|
|
310
|
+
fix: "Nothing to fix in the server: toolmenu doesn't answer roots, sampling or elicitation requests, so it can't list these routers' commands.",
|
|
311
|
+
},
|
|
308
312
|
'invalid-arguments': {
|
|
309
313
|
what: ["rejected the listing call's arguments", "rejected the listing call's arguments"],
|
|
310
314
|
fix: "Read the router's own words above; if it lists its commands another way, name the right tool in catalog.tool or catalog.routers.",
|
package/dist/diff.js
CHANGED
|
@@ -1148,7 +1148,11 @@ function expandRefs(schema) {
|
|
|
1148
1148
|
return { ...walk(target, [...stack, o.$ref]), ...walk(siblings, stack) };
|
|
1149
1149
|
}
|
|
1150
1150
|
}
|
|
1151
|
-
|
|
1151
|
+
// A $ref left as a $ref (a recursive definition past MAX_UNROLL): `definitions` and `$defs` are one
|
|
1152
|
+
// place under two spellings, and the root pool is dropped below, so the reference reads the same.
|
|
1153
|
+
// Left as written, a generator upgrade that renames the pool made every recursive $ref look like a
|
|
1154
|
+
// different option ("no longer accepts #/definitions/x" beside the unchanged #/$defs/x).
|
|
1155
|
+
return Object.fromEntries(Object.entries(o).map(([k, v]) => [k, k === '$ref' && typeof v === 'string' ? v.replace(/^#\/definitions\//, '#/$defs/') : walk(v, stack)]));
|
|
1152
1156
|
};
|
|
1153
1157
|
const { $defs: _defs, definitions: _definitions, ...rest } = schema;
|
|
1154
1158
|
try {
|
package/dist/explain.d.ts
CHANGED
|
@@ -32,6 +32,8 @@ export declare class Trace {
|
|
|
32
32
|
/** stdout lines that aren't JSON-RPC: the first few, and how many in all. */
|
|
33
33
|
readonly stray: string[];
|
|
34
34
|
strayCount: number;
|
|
35
|
+
/** The first stray stdout lines, more of them than are shown: a CLI's usage runs long. */
|
|
36
|
+
readonly strayHead: string[];
|
|
35
37
|
private stdoutPartial;
|
|
36
38
|
/** The last stderr lines, and how many there were in all. */
|
|
37
39
|
readonly stderrTail: string[];
|
|
@@ -78,6 +80,12 @@ export declare function seconds(ms: number): string;
|
|
|
78
80
|
export declare function commandLine(command: string, args: string[]): string;
|
|
79
81
|
/** The stderr fact: the last lines, and a note of how many came before. */
|
|
80
82
|
export declare function stderrFact(trace: Trace): [string, string];
|
|
83
|
+
/**
|
|
84
|
+
* What the server's own output says about how it was started, as a next step: a
|
|
85
|
+
* CLI that printed its usage and wants a subcommand, or a server that opened an
|
|
86
|
+
* HTTP port instead of talking on stdio. Undefined when it says neither.
|
|
87
|
+
*/
|
|
88
|
+
export declare function startupHint(trace: Trace, command: string): string | undefined;
|
|
81
89
|
/** A request header that carries credentials: Authorization, a cookie, or a name with key, token, secret or auth in it (x-mcp-api-key). */
|
|
82
90
|
export declare function isCredentialHeader(name: string): boolean;
|
|
83
91
|
/**
|
package/dist/explain.js
CHANGED
|
@@ -21,6 +21,8 @@ export class Trace {
|
|
|
21
21
|
/** stdout lines that aren't JSON-RPC: the first few, and how many in all. */
|
|
22
22
|
stray = [];
|
|
23
23
|
strayCount = 0;
|
|
24
|
+
/** The first stray stdout lines, more of them than are shown: a CLI's usage runs long. */
|
|
25
|
+
strayHead = [];
|
|
24
26
|
stdoutPartial = '';
|
|
25
27
|
/** The last stderr lines, and how many there were in all. */
|
|
26
28
|
stderrTail = [];
|
|
@@ -48,6 +50,8 @@ export class Trace {
|
|
|
48
50
|
this.strayCount++;
|
|
49
51
|
if (this.stray.length < 2)
|
|
50
52
|
this.stray.push(clean(line, 160));
|
|
53
|
+
if (this.strayHead.length < 60)
|
|
54
|
+
this.strayHead.push(clean(line, 240));
|
|
51
55
|
}
|
|
52
56
|
}
|
|
53
57
|
stderr(chunk) {
|
|
@@ -144,6 +148,39 @@ function strayFact(trace) {
|
|
|
144
148
|
return ['stdout', `${quoted}${more}`];
|
|
145
149
|
}
|
|
146
150
|
const LOG_TO_STDERR = 'Log to stderr; stdout carries the protocol (in Node, console.error rather than console.log; in Python, logging or print(..., file=sys.stderr)).';
|
|
151
|
+
// A CLI's usage: argparse, click, yargs, commander, System.CommandLine, stricli.
|
|
152
|
+
const USAGE = /(?:^|\n)\s*(?:usage|(?:sub)?commands|options|examples|flags):|\bUSAGE\b|\s<command>|\[command\]|\bCOMMAND \[ARGS\]|arguments are required|subcommands? must be present|no command (?:specified|given)|required command was not provided|\b(?:run|see):? \S+ --help\b/i;
|
|
153
|
+
// A table of commands or options, two or more rows: " t3-watch start Run the service",
|
|
154
|
+
// " --version Show version number". Help output, even without a "Usage:" line.
|
|
155
|
+
const HELP_ROWS = /(?:^|\n)\s*(?:[\w.-]+ [\w-]+|-{1,2}[\w-]+(?:, -{1,2}[\w-]+)?) {2,}\S[^\n]*\n\s*(?:[\w.-]+ [\w-]+|-{1,2}[\w-]+(?:, -{1,2}[\w-]+)?) {2,}\S/;
|
|
156
|
+
// The subcommands that usually start the MCP server, most specific first. A word
|
|
157
|
+
// on its own ({mcp}, " mcp Start…", "mcp start [--transport…]"), not part of a
|
|
158
|
+
// name like agentation-mcp.
|
|
159
|
+
const SUBCOMMANDS = ['mcp start', 'mcp serve', 'server start', 'mcp', 'stdio', 'serve', 'server', 'start'];
|
|
160
|
+
// An HTTP server's address in its log: "MCP server listening on http://localhost:3001/mcp",
|
|
161
|
+
// "StreamableHTTP endpoint available at http://127.0.0.1:3333/mcp".
|
|
162
|
+
const HTTP_URL = /\b(?:listening|available|running|serving|started)\b[^\n]{0,60}?(https?:\/\/[^\s"'<>]+)/i;
|
|
163
|
+
const HTTP_PORT = /\blistening on (?:port )?:?(\d{2,5})\b/i;
|
|
164
|
+
/**
|
|
165
|
+
* What the server's own output says about how it was started, as a next step: a
|
|
166
|
+
* CLI that printed its usage and wants a subcommand, or a server that opened an
|
|
167
|
+
* HTTP port instead of talking on stdio. Undefined when it says neither.
|
|
168
|
+
*/
|
|
169
|
+
export function startupHint(trace, command) {
|
|
170
|
+
const text = [...trace.strayHead, ...trace.stderrExcerpt().lines].join('\n');
|
|
171
|
+
const url = HTTP_URL.exec(text)?.[1]?.replace(/[.,;)]+$/, '').replace('//0.0.0.0', '//localhost');
|
|
172
|
+
const port = url ? undefined : HTTP_PORT.exec(text)?.[1];
|
|
173
|
+
if (url || port) {
|
|
174
|
+
const flag = /--stdio\b/.test(text) ? '--stdio' : /--transport[ =]?(?:\[?stdio|<)/i.test(text) ? '--transport stdio' : undefined;
|
|
175
|
+
const where = url ?? `http://localhost:${port}/mcp`;
|
|
176
|
+
return `The server started an HTTP server (${where}) instead of talking on its stdin and stdout. Start it in stdio mode (${flag ? `its output mentions ${flag}` : 'often --stdio or --transport stdio'}), or run toolmenu against the address while the server runs: toolmenu snapshot ${where}`;
|
|
177
|
+
}
|
|
178
|
+
if (USAGE.test(text) || HELP_ROWS.test(text)) {
|
|
179
|
+
const sub = SUBCOMMANDS.find((c) => new RegExp(`(?:^|[\\s{,|\\[])${c.replace(' ', '\\s+')}(?=[\\s},|\\]]|$)`, 'm').test(text));
|
|
180
|
+
return `The command printed its usage and exited: it wants a subcommand or an argument it wasn't given. ${sub ? `Its usage lists “${sub}”: try ${command} ${sub}` : 'Its usage (above) or its docs say what starts the MCP server: add that after the command.'}`;
|
|
181
|
+
}
|
|
182
|
+
return undefined;
|
|
183
|
+
}
|
|
147
184
|
/** The error codes along an error's cause chain, innermost last. */
|
|
148
185
|
function causes(error) {
|
|
149
186
|
const out = [];
|
|
@@ -286,12 +323,16 @@ function explainStdio(error, trace, context, target) {
|
|
|
286
323
|
const stderrText = trace.stderrExcerpt().lines.join('\n');
|
|
287
324
|
const rejected = rejectedCredentials(stderrText);
|
|
288
325
|
let next;
|
|
326
|
+
let hint;
|
|
289
327
|
if (/cannot find (module|package)|ERR_MODULE_NOT_FOUND|ModuleNotFoundError|No module named/i.test(stderrText)) {
|
|
290
328
|
next = "The server is missing one of its own dependencies (see stderr): reinstall it, or try another version. It's a packaging problem, not your setup.";
|
|
291
329
|
}
|
|
292
330
|
else if (rejected) {
|
|
293
331
|
next = REJECTED_NEXT;
|
|
294
332
|
}
|
|
333
|
+
else if ((hint = startupHint(trace, command))) {
|
|
334
|
+
next = hint;
|
|
335
|
+
}
|
|
295
336
|
else if (trace.strayCount) {
|
|
296
337
|
next = `${LOG_TO_STDERR} Then fix what stderr says.`;
|
|
297
338
|
}
|
|
@@ -329,13 +370,16 @@ function explainStdio(error, trace, context, target) {
|
|
|
329
370
|
: rejected
|
|
330
371
|
? `${stage} timed out after ${seconds(waitedHere)}: the server is running but never answered; its stderr looks like rejected credentials (“${rejected}”).`
|
|
331
372
|
: `${stage} timed out after ${seconds(waitedHere)}: the server is running but never answered.`;
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
373
|
+
const hint = startupHint(trace, command);
|
|
374
|
+
const next = hint
|
|
375
|
+
? hint
|
|
376
|
+
: trace.strayCount
|
|
377
|
+
? LOG_TO_STDERR
|
|
378
|
+
: rejected
|
|
379
|
+
? REJECTED_NEXT
|
|
380
|
+
: stage === 'tools/list'
|
|
381
|
+
? 'The server answered initialize, then hung on tools/list: check its logs (stderr above). If building the list is just slow, raise --timeout.'
|
|
382
|
+
: 'Check the command starts an MCP server on stdio (some need an argument such as "stdio" or "--stdio"). If it is only slow to start (npx downloading, a first build), raise --timeout.';
|
|
339
383
|
return new ConnectError(headline, facts, next, stage, { cause: error });
|
|
340
384
|
}
|
|
341
385
|
// Anything else: the transport's words, with what the server said.
|
package/dist/failures.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'invalid-arguments' | 'other';
|
|
1
|
+
export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'client' | 'invalid-arguments' | 'other';
|
|
2
2
|
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
3
3
|
export declare const SETUP_FAILURES: Set<FailureClass>;
|
|
4
4
|
/**
|
package/dist/failures.js
CHANGED
|
@@ -5,16 +5,23 @@
|
|
|
5
5
|
* as a finding.
|
|
6
6
|
*/
|
|
7
7
|
/** Failures of the setup (credentials, this machine, the network), not of the server. */
|
|
8
|
-
export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found']);
|
|
8
|
+
export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found', 'client']);
|
|
9
|
+
// A tool that asks the client for something toolmenu doesn't offer: "Cannot request
|
|
10
|
+
// input 'roots' (roots/list): the request's client capabilities do not declare the
|
|
11
|
+
// required capability" (server-everything's get-roots-list), sampling, elicitation.
|
|
12
|
+
const CLIENT = /client capabilit(?:y|ies)\b[^.]{0,60}\b(?:do(?:es)? not|don't|doesn't|lack|missing|without)\b|\b(?:roots|sampling|elicitation)\b[^.]{0,40}\bnot (?:supported|declared|available)\b[^.]{0,20}\bclient\b|client does not support (?:roots|sampling|elicitation)/i;
|
|
9
13
|
// Something this machine lacks: "Could not find Google Chrome executable"
|
|
10
|
-
// (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright)
|
|
11
|
-
|
|
14
|
+
// (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright),
|
|
15
|
+
// or the wrong operating system: "Editmamei runs on Windows and macOS; this process
|
|
16
|
+
// is on "linux"" (editmamei), "Unsupported platform: linux".
|
|
17
|
+
const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b|\bruns on [^.;]{0,40}; this (?:process|machine|system) is on\b|\b(?:unsupported|not supported on (?:this|the current)) (?:platform|operating system|OS)\b/i;
|
|
12
18
|
// "401 Bad credentials" (GitHub), "Authorization Expired" (Sentry), "Invalid
|
|
13
19
|
// token" (Firecrawl), "authentication token is not valid" (Apify), "Please provide
|
|
14
20
|
// a valid access token" (Supabase), "NetlifyUnauthError: You're not logged into
|
|
15
|
-
// Netlify" (Netlify), "The API key you provided was rejected" (Pinecone)
|
|
16
|
-
//
|
|
17
|
-
|
|
21
|
+
// Netlify" (Netlify), "The API key you provided was rejected" (Pinecone), "Admin
|
|
22
|
+
// auth is missing", "Runtime credentials are missing" (securenow). Not a bare
|
|
23
|
+
// "token": plenty of errors mention one.
|
|
24
|
+
const AUTH = /\b(?:auth|authentication|credentials?|api[ _-]?key|access token)\b (?:is |are )?missing\b|\b(401|403)\b|unauthori[sz]ed|unauth(enticated|error)|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|not logged (?:in|into)\b(?! to the browser)|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|(api[ _-]?key|access token|credentials)\b[^.]{0,30}\b(was|were|is|are) (rejected|refused|revoked)|valid (access )?token/i;
|
|
18
25
|
const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
|
|
19
26
|
// "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
|
|
20
27
|
// "**Input Error**" (Sentry), JSON-RPC's invalid params.
|
|
@@ -151,6 +158,8 @@ export async function patiently(request, options = {}) {
|
|
|
151
158
|
* with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
|
|
152
159
|
*/
|
|
153
160
|
export function classifyFailure(text, options = {}) {
|
|
161
|
+
if (CLIENT.test(text))
|
|
162
|
+
return 'client';
|
|
154
163
|
if (ENVIRONMENT.test(text))
|
|
155
164
|
return 'environment';
|
|
156
165
|
if (AUTH.test(text))
|
|
@@ -168,6 +177,7 @@ export const FAILURE_LABELS = {
|
|
|
168
177
|
environment: 'something missing on this machine',
|
|
169
178
|
network: 'the network',
|
|
170
179
|
'not-found': 'a 404 on a call that named nothing (usually the account or site behind the credentials)',
|
|
180
|
+
client: "a client feature toolmenu doesn't offer (roots, sampling or elicitation)",
|
|
171
181
|
'invalid-arguments': 'the arguments',
|
|
172
182
|
other: 'other errors',
|
|
173
183
|
};
|
|
@@ -184,6 +194,8 @@ export function setupFailureAdvice(classes, transport) {
|
|
|
184
194
|
steps.push(`If the errors are about credentials, rerun with real ones (${credentials}); a test account is enough.`);
|
|
185
195
|
if (set.has('environment'))
|
|
186
196
|
steps.push('If something is missing on this machine (a browser, a binary, a module), install it where the server runs, or run the server in its own container image.');
|
|
197
|
+
if (set.has('client'))
|
|
198
|
+
steps.push("If a tool asks the client for roots, sampling or elicitation, toolmenu doesn't provide them: those calls only checked that the menu stays put, and the server isn't at fault.");
|
|
187
199
|
if (set.has('network'))
|
|
188
200
|
steps.push('If the server calls an upstream API, check this machine can reach it (DNS, proxy, firewall).');
|
|
189
201
|
return { fix: steps.join(' ') || 'Check the quoted errors: they came from before the tool ran.', confidence: 'unsure' };
|
package/dist/rules/gated.js
CHANGED
|
@@ -17,7 +17,10 @@ export const gated = {
|
|
|
17
17
|
// Only an unlock whose parameter lists what it unlocks (an enum of toolsets or
|
|
18
18
|
// domains), the kind session --auto walks: "Enables Cross-Region Restore on a
|
|
19
19
|
// vault" turns on a feature, not tools (Azure, 418 tools).
|
|
20
|
-
|
|
20
|
+
// And only one whose name or description says it unlocks tools: an enum of
|
|
21
|
+
// domains alone is as often data (Garmin's garmin_sync { domains: [sleep, hrv] }
|
|
22
|
+
// downloads them). session --auto still tries such a tool once.
|
|
23
|
+
const found = unlockers(ctx.menu.tools).filter((u) => u.param && u.values.length && u.claims);
|
|
21
24
|
if (!found.length)
|
|
22
25
|
return [];
|
|
23
26
|
// A value whose tools are already listed adds nothing: the same server started
|
package/dist/rules/ids.js
CHANGED
|
@@ -3,7 +3,8 @@ import { kept } from './rule.js';
|
|
|
3
3
|
// form_id, form-id, formId, formID. Not uid or paid: the suffix needs a boundary.
|
|
4
4
|
// Not "key" either: in real menus it's as often a keyboard key as an identifier.
|
|
5
5
|
const ID_SUFFIX = /^(.+?)[_-](id|ids|uuid|uuids)$/i;
|
|
6
|
-
|
|
6
|
+
// UUID before ID: viewUUID is a view's ID, not a viewUU's.
|
|
7
|
+
const CAMEL_ID = /^([a-z][a-zA-Z0-9]*?)(UUIDs?|Uuids?|IDs|ID|Ids|Id)$/;
|
|
7
8
|
const BARE_ID = /^(id|ids|uid|uuid)$/i;
|
|
8
9
|
const CREATE_VERBS = new Set(['create', 'add', 'new', 'insert', 'post', 'upsert', 'start', 'run', 'submit', 'duplicate', 'copy']);
|
|
9
10
|
// Words that join alternatives in a name: projectSlugOrId → project.
|
package/dist/rules/index.js
CHANGED
|
@@ -6,6 +6,7 @@ import { route, sharedWord, vagueId } from './naming.js';
|
|
|
6
6
|
import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
|
|
7
7
|
import { noDryRun, unannotated } from './write.js';
|
|
8
8
|
import { gated } from './gated.js';
|
|
9
|
+
import { findable } from './search.js';
|
|
9
10
|
export const MENU_RULES = [
|
|
10
11
|
nondeterministic,
|
|
11
12
|
duplicateName,
|
|
@@ -16,6 +17,7 @@ export const MENU_RULES = [
|
|
|
16
17
|
vagueId,
|
|
17
18
|
sharedWord,
|
|
18
19
|
route,
|
|
20
|
+
findable,
|
|
19
21
|
authoredIds,
|
|
20
22
|
buried,
|
|
21
23
|
cut,
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { buildIndex, placeOf, SEARCH_LIMIT } from '../search.js';
|
|
2
|
+
import { commonWords, VERBS, words } from '../words.js';
|
|
3
|
+
import { clip, count, kept, names, num } from './rule.js';
|
|
4
|
+
/**
|
|
5
|
+
* When a client defers tools and lets the model search for them (Claude's tool
|
|
6
|
+
* search, OpenAI's tool_search, Claude Code's deferred MCP tools), a tool the
|
|
7
|
+
* search doesn't return can't be called. The search returns 5 tools unless the
|
|
8
|
+
* model asks for more, so a tool that a search for its own name doesn't put in
|
|
9
|
+
* the top 5 is one the agent can miss.
|
|
10
|
+
*
|
|
11
|
+
* Only for menus a client would defer: the tool search docs suggest it from 10
|
|
12
|
+
* tools or 10K tokens of definitions.
|
|
13
|
+
*/
|
|
14
|
+
const MIN_TOOLS = 10;
|
|
15
|
+
const MIN_TOKENS = 10_000;
|
|
16
|
+
export const findable = {
|
|
17
|
+
id: 'search/findable',
|
|
18
|
+
severity: 'info',
|
|
19
|
+
summary: 'Each tool comes up when searched for by its own name, for clients that defer tools',
|
|
20
|
+
run(ctx) {
|
|
21
|
+
const tools = ctx.menu.tools;
|
|
22
|
+
if (tools.length < MIN_TOOLS || ((ctx.menu.totalTokens ?? 0) < MIN_TOKENS && tools.length < 30))
|
|
23
|
+
return [];
|
|
24
|
+
const index = buildIndex(tools);
|
|
25
|
+
const findings = [];
|
|
26
|
+
// The words of a name that say something: not a prefix most tools share (firecrawl_, API-).
|
|
27
|
+
const shared = commonWords(tools.map((t) => t.name));
|
|
28
|
+
const missed = [];
|
|
29
|
+
for (const t of kept(tools, ctx)) {
|
|
30
|
+
// A bare verb ("create" from ads_create) isn't how an agent searches: then the
|
|
31
|
+
// prefix stays, since it's the object the tool acts on.
|
|
32
|
+
const all = words(t.name);
|
|
33
|
+
const own = all.filter((w) => !shared.has(w));
|
|
34
|
+
const query = (own.some((w) => !VERBS.has(w)) ? own : all).join(' ');
|
|
35
|
+
if (!query || !all.some((w) => !VERBS.has(w)))
|
|
36
|
+
continue;
|
|
37
|
+
const { place, ahead } = placeOf(index, query, t.name);
|
|
38
|
+
if (place === undefined || place > SEARCH_LIMIT)
|
|
39
|
+
missed.push({ tool: t.name, query, place, ahead });
|
|
40
|
+
}
|
|
41
|
+
if (missed.length) {
|
|
42
|
+
const first = missed[0];
|
|
43
|
+
findings.push({
|
|
44
|
+
...(missed.length === 1 ? { tool: first.tool } : {}),
|
|
45
|
+
confidence: 'unsure',
|
|
46
|
+
message: `${count(missed.length, 'tool')} of ${num(tools.length)} ${missed.length === 1 ? "doesn't" : "don't"} come up in the top ${SEARCH_LIMIT} when searched for by ${missed.length === 1 ? 'its' : 'their'} own name. Clients that defer a menu this size let the model search it, and a search returns ${SEARCH_LIMIT} tools unless the model asks for more, so an agent looking for ${missed.length === 1 ? 'it' : 'them'} can be shown other tools instead. Ranked the way a BM25 tool search ranks them (name, description, argument names and descriptions); a client's own search can rank a little differently.`,
|
|
47
|
+
detail: missed.slice(0, 12).map((m) => `${m.tool}: “${m.query}” puts it ${m.place === undefined ? 'nowhere' : `at ${m.place}`}, behind ${names(m.ahead.slice(0, 4), 4)}${m.ahead.length > 4 ? ` and ${num(m.ahead.length - 4)} more` : ''}`).concat(missed.length > 12 ? [`and ${num(missed.length - 12)} more`] : []),
|
|
48
|
+
fix: 'Say the words of each tool\'s name in its own description, early on (a search ranks the description too, and "Returns the llms.txt" never says "guide"), give it a word the tools ahead of it lack (the object it acts on, not the verb), and cut repeats of other tools\' words from long descriptions: a description that mentions "domain" ten times outranks the tool named for domains.',
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
// routes.yml: a route's tools should be in what a search for it returns.
|
|
52
|
+
for (const [keyword, expectation] of Object.entries(ctx.routes ?? {})) {
|
|
53
|
+
const present = (expectation.must_match ?? []).filter((n) => tools.some((t) => t.name === n));
|
|
54
|
+
const lost = present
|
|
55
|
+
.map((n) => ({ n, ...placeOf(index, keyword, n) }))
|
|
56
|
+
.filter((r) => r.place === undefined || r.place > SEARCH_LIMIT);
|
|
57
|
+
for (const r of lost) {
|
|
58
|
+
findings.push({
|
|
59
|
+
tool: r.n,
|
|
60
|
+
severity: 'warn',
|
|
61
|
+
confidence: 'unsure',
|
|
62
|
+
message: `routes.yml says “${keyword}” should reach ${r.n}, but a tool search for “${keyword}” ${r.place === undefined ? "doesn't return it at all" : `puts it at ${r.place}`}, past the ${SEARCH_LIMIT} a search returns by default. A client that defers tools may never show it.`,
|
|
63
|
+
detail: [`ahead of it: ${names(r.ahead.slice(0, 5), 5)}`, `${r.n}: ${clip(tools.find((t) => t.name === r.n)?.description ?? '', 120)}`],
|
|
64
|
+
fix: `Put the words of “${keyword}” in ${r.n}'s name or the start of its description, or take them out of the tools ahead of it.`,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return findings;
|
|
69
|
+
},
|
|
70
|
+
};
|
package/dist/search.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { MenuTool } from './types.js';
|
|
2
|
+
/** How many tools a search returns when the model doesn't set a limit. */
|
|
3
|
+
export declare const SEARCH_LIMIT = 5;
|
|
4
|
+
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
5
|
+
export declare function searchText(tool: MenuTool): string[];
|
|
6
|
+
/** Lowercase, singular, without stop words: what a query and a document are compared on. */
|
|
7
|
+
export declare function terms(text: string): string[];
|
|
8
|
+
export interface SearchIndex {
|
|
9
|
+
/** Tools ranked for a query, best first, with their scores. Tools that match no term are left out. */
|
|
10
|
+
rank(query: string): {
|
|
11
|
+
tool: string;
|
|
12
|
+
score: number;
|
|
13
|
+
}[];
|
|
14
|
+
}
|
|
15
|
+
export declare function buildIndex(tools: MenuTool[]): SearchIndex;
|
|
16
|
+
/**
|
|
17
|
+
* Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
|
|
18
|
+
* (only tools that score strictly higher are ahead of it), so a miss is never a
|
|
19
|
+
* coin flip. Undefined when the query matches nothing in the tool.
|
|
20
|
+
*/
|
|
21
|
+
export declare function placeOf(index: SearchIndex, query: string, tool: string): {
|
|
22
|
+
place?: number;
|
|
23
|
+
ahead: string[];
|
|
24
|
+
};
|
package/dist/search.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { singular, words } from './words.js';
|
|
2
|
+
/** How many tools a search returns when the model doesn't set a limit. */
|
|
3
|
+
export const SEARCH_LIMIT = 5;
|
|
4
|
+
// Words that carry no meaning in a query or a description.
|
|
5
|
+
const STOP = new Set([
|
|
6
|
+
'a', 'an', 'the', 'of', 'for', 'to', 'in', 'on', 'at', 'by', 'from', 'with', 'and', 'or', 'is', 'are', 'be',
|
|
7
|
+
'this', 'that', 'it', 'its', 'as', 'use', 'when', 'if', 'you', 'your', 'can', 'will', 'not', 'any', 'all',
|
|
8
|
+
]);
|
|
9
|
+
const K1 = 1.2;
|
|
10
|
+
const B = 0.75;
|
|
11
|
+
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
12
|
+
export function searchText(tool) {
|
|
13
|
+
const parts = [tool.name, tool.description ?? ''];
|
|
14
|
+
const walk = (schema, depth) => {
|
|
15
|
+
if (!schema || typeof schema !== 'object' || depth > 6)
|
|
16
|
+
return;
|
|
17
|
+
const s = schema;
|
|
18
|
+
for (const [name, prop] of Object.entries(s.properties ?? {})) {
|
|
19
|
+
parts.push(name);
|
|
20
|
+
const d = prop?.description;
|
|
21
|
+
if (typeof d === 'string')
|
|
22
|
+
parts.push(d);
|
|
23
|
+
walk(prop, depth + 1);
|
|
24
|
+
}
|
|
25
|
+
if (s.items)
|
|
26
|
+
walk(s.items, depth + 1);
|
|
27
|
+
};
|
|
28
|
+
walk(tool.inputSchema, 0);
|
|
29
|
+
return terms(parts.join(' '));
|
|
30
|
+
}
|
|
31
|
+
/** Lowercase, singular, without stop words: what a query and a document are compared on. */
|
|
32
|
+
export function terms(text) {
|
|
33
|
+
return words(text)
|
|
34
|
+
.map(singular)
|
|
35
|
+
.filter((w) => w.length > 1 && !STOP.has(w));
|
|
36
|
+
}
|
|
37
|
+
export function buildIndex(tools) {
|
|
38
|
+
const docs = tools.map((t) => {
|
|
39
|
+
const tf = new Map();
|
|
40
|
+
const text = searchText(t);
|
|
41
|
+
for (const w of text)
|
|
42
|
+
tf.set(w, (tf.get(w) ?? 0) + 1);
|
|
43
|
+
return { name: t.name, tf, length: text.length };
|
|
44
|
+
});
|
|
45
|
+
const avg = docs.reduce((n, d) => n + d.length, 0) / Math.max(docs.length, 1);
|
|
46
|
+
const df = new Map();
|
|
47
|
+
for (const d of docs)
|
|
48
|
+
for (const w of d.tf.keys())
|
|
49
|
+
df.set(w, (df.get(w) ?? 0) + 1);
|
|
50
|
+
const idf = (w) => {
|
|
51
|
+
const n = df.get(w) ?? 0;
|
|
52
|
+
return Math.log(1 + (docs.length - n + 0.5) / (n + 0.5));
|
|
53
|
+
};
|
|
54
|
+
return {
|
|
55
|
+
rank(query) {
|
|
56
|
+
const q = [...new Set(terms(query))];
|
|
57
|
+
const out = [];
|
|
58
|
+
for (const d of docs) {
|
|
59
|
+
let score = 0;
|
|
60
|
+
for (const w of q) {
|
|
61
|
+
const f = d.tf.get(w);
|
|
62
|
+
if (!f)
|
|
63
|
+
continue;
|
|
64
|
+
score += idf(w) * ((f * (K1 + 1)) / (f + K1 * (1 - B + (B * d.length) / (avg || 1))));
|
|
65
|
+
}
|
|
66
|
+
if (score > 0)
|
|
67
|
+
out.push({ tool: d.name, score });
|
|
68
|
+
}
|
|
69
|
+
// Ties keep the menu's order, as a stable search would.
|
|
70
|
+
return out.sort((a, b) => b.score - a.score);
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
|
|
76
|
+
* (only tools that score strictly higher are ahead of it), so a miss is never a
|
|
77
|
+
* coin flip. Undefined when the query matches nothing in the tool.
|
|
78
|
+
*/
|
|
79
|
+
export function placeOf(index, query, tool) {
|
|
80
|
+
const ranked = index.rank(query);
|
|
81
|
+
const mine = ranked.find((r) => r.tool === tool);
|
|
82
|
+
if (!mine)
|
|
83
|
+
return { ahead: ranked.slice(0, SEARCH_LIMIT).map((r) => r.tool) };
|
|
84
|
+
const ahead = ranked.filter((r) => r.score > mine.score + 1e-9).map((r) => r.tool);
|
|
85
|
+
return { place: ahead.length + 1, ahead };
|
|
86
|
+
}
|
package/dist/session.d.ts
CHANGED
|
@@ -138,6 +138,11 @@ interface Unlocker {
|
|
|
138
138
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
139
139
|
*/
|
|
140
140
|
export declare function clip(text: string, max: number): string;
|
|
141
|
+
/**
|
|
142
|
+
* Absolute paths longer than 40 characters, kept to their last two parts:
|
|
143
|
+
* /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
|
|
144
|
+
*/
|
|
145
|
+
export declare function shortenPaths(text: string): string;
|
|
141
146
|
export declare function unlockers(tools: MenuTool[]): Unlocker[];
|
|
142
147
|
/**
|
|
143
148
|
* Unlocks whose values the schema doesn't give (no enum), each with a read-only
|
package/dist/session.js
CHANGED
|
@@ -490,7 +490,7 @@ function untested(steps, target) {
|
|
|
490
490
|
// Classified from the error's words: a guess, if a good one.
|
|
491
491
|
confidence: 'unsure',
|
|
492
492
|
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
|
|
493
|
-
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '')
|
|
493
|
+
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${clip((list[0].note ?? '').replace(/^the tool returned an error: /, ''), 100)}”)`),
|
|
494
494
|
fix,
|
|
495
495
|
},
|
|
496
496
|
];
|
|
@@ -509,6 +509,11 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
|
|
|
509
509
|
for (const u of unlockers(menu)) {
|
|
510
510
|
if (!u.param || u.values.length === 0)
|
|
511
511
|
continue;
|
|
512
|
+
// Never says it unlocks tools and the run didn't call it: nothing suggests tools
|
|
513
|
+
// are missing (Garmin's garmin_sync { domains } downloads data). One it called
|
|
514
|
+
// and stopped is covered by auto.stopped.
|
|
515
|
+
if (!u.claims && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
|
|
516
|
+
continue;
|
|
512
517
|
const reached = new Set();
|
|
513
518
|
let last;
|
|
514
519
|
const add = (arg) => {
|
|
@@ -1063,13 +1068,31 @@ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
|
1063
1068
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
1064
1069
|
*/
|
|
1065
1070
|
export function clip(text, max) {
|
|
1066
|
-
|
|
1071
|
+
let chars = Array.from(text.replace(/\uFFFD+$/u, ''));
|
|
1072
|
+
if (chars.length <= max)
|
|
1073
|
+
return chars.join('');
|
|
1074
|
+
// A long path loses its middle before the text loses its end: “spawn /tmp/…/x ENOENT”
|
|
1075
|
+
// cut at the end hides both the file and the reason.
|
|
1076
|
+
chars = Array.from(shortenPaths(chars.join('')));
|
|
1067
1077
|
if (chars.length <= max)
|
|
1068
1078
|
return chars.join('');
|
|
1069
1079
|
const cut = chars.slice(0, max - 1).join('');
|
|
1070
1080
|
const space = cut.lastIndexOf(' ');
|
|
1071
1081
|
return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
|
|
1072
1082
|
}
|
|
1083
|
+
/**
|
|
1084
|
+
* Absolute paths longer than 40 characters, kept to their last two parts:
|
|
1085
|
+
* /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
|
|
1086
|
+
*/
|
|
1087
|
+
export function shortenPaths(text) {
|
|
1088
|
+
// Not a URL's path: a path starts after nothing that could be a scheme, host or word.
|
|
1089
|
+
return text.replace(/(?<![\w:/\\.])(?:[A-Za-z]:)?(?:[\\/][^\s\\/"'“”()]+){3,}/g, (path) => {
|
|
1090
|
+
if (path.length <= 40)
|
|
1091
|
+
return path;
|
|
1092
|
+
const sep = path.includes('\\') && !path.includes('/') ? '\\' : '/';
|
|
1093
|
+
return `…${sep}${path.split(/[\\/]/).slice(-2).join(sep)}`;
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1073
1096
|
function isWriteLike(t) {
|
|
1074
1097
|
// Named as an unlock: changing the menu is its job, whatever its hints say
|
|
1075
1098
|
// (toolception marks enable_toolset destructive). It's still only suggested,
|
package/package.json
CHANGED