toolmenu 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
29
29
  (`history` installs old versions with today's dependencies, so it records what they
30
30
  serve now, which can differ from what they shipped with: FINDINGS F4.)
31
31
 
32
- > **Status: 0.14, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
32
+ > **Status: 0.15, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
33
33
  > real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
34
34
 
35
35
  ## Start here
@@ -104,7 +104,7 @@ jobs:
104
104
  - uses: actions/setup-node@v7
105
105
  with: { node-version: 22 }
106
106
  - run: npm ci && npm run build
107
- - uses: niksa90/toolmenu@v0.14.0
107
+ - uses: niksa90/toolmenu@v0.15.0
108
108
  with:
109
109
  command: node dist/server.js
110
110
  baseline: menu.json # your committed snapshot
package/dist/catalog.js CHANGED
@@ -305,6 +305,10 @@ const FAILED_AS = {
305
305
  what: ['answered "not found"', 'answered "not found"'],
306
306
  fix: 'Check the server is set up for this account or site, then rerun.',
307
307
  },
308
+ client: {
309
+ what: ["asked the client for something toolmenu doesn't offer (roots, sampling or elicitation)", "asked the client for something toolmenu doesn't offer (roots, sampling or elicitation)"],
310
+ fix: "Nothing to fix in the server: toolmenu doesn't answer roots, sampling or elicitation requests, so it can't list these routers' commands.",
311
+ },
308
312
  'invalid-arguments': {
309
313
  what: ["rejected the listing call's arguments", "rejected the listing call's arguments"],
310
314
  fix: "Read the router's own words above; if it lists its commands another way, name the right tool in catalog.tool or catalog.routers.",
package/dist/diff.js CHANGED
@@ -1148,7 +1148,11 @@ function expandRefs(schema) {
1148
1148
  return { ...walk(target, [...stack, o.$ref]), ...walk(siblings, stack) };
1149
1149
  }
1150
1150
  }
1151
- return Object.fromEntries(Object.entries(o).map(([k, v]) => [k, walk(v, stack)]));
1151
+ // A $ref left as a $ref (a recursive definition past MAX_UNROLL): `definitions` and `$defs` are one
1152
+ // place under two spellings, and the root pool is dropped below, so the reference reads the same.
1153
+ // Left as written, a generator upgrade that renames the pool made every recursive $ref look like a
1154
+ // different option ("no longer accepts #/definitions/x" beside the unchanged #/$defs/x).
1155
+ return Object.fromEntries(Object.entries(o).map(([k, v]) => [k, k === '$ref' && typeof v === 'string' ? v.replace(/^#\/definitions\//, '#/$defs/') : walk(v, stack)]));
1152
1156
  };
1153
1157
  const { $defs: _defs, definitions: _definitions, ...rest } = schema;
1154
1158
  try {
package/dist/explain.d.ts CHANGED
@@ -32,6 +32,8 @@ export declare class Trace {
32
32
  /** stdout lines that aren't JSON-RPC: the first few, and how many in all. */
33
33
  readonly stray: string[];
34
34
  strayCount: number;
35
+ /** The first stray stdout lines, more of them than are shown: a CLI's usage runs long. */
36
+ readonly strayHead: string[];
35
37
  private stdoutPartial;
36
38
  /** The last stderr lines, and how many there were in all. */
37
39
  readonly stderrTail: string[];
@@ -78,6 +80,12 @@ export declare function seconds(ms: number): string;
78
80
  export declare function commandLine(command: string, args: string[]): string;
79
81
  /** The stderr fact: the last lines, and a note of how many came before. */
80
82
  export declare function stderrFact(trace: Trace): [string, string];
83
+ /**
84
+ * What the server's own output says about how it was started, as a next step: a
85
+ * CLI that printed its usage and wants a subcommand, or a server that opened an
86
+ * HTTP port instead of talking on stdio. Undefined when it says neither.
87
+ */
88
+ export declare function startupHint(trace: Trace, command: string): string | undefined;
81
89
  /** A request header that carries credentials: Authorization, a cookie, or a name with key, token, secret or auth in it (x-mcp-api-key). */
82
90
  export declare function isCredentialHeader(name: string): boolean;
83
91
  /**
package/dist/explain.js CHANGED
@@ -21,6 +21,8 @@ export class Trace {
21
21
  /** stdout lines that aren't JSON-RPC: the first few, and how many in all. */
22
22
  stray = [];
23
23
  strayCount = 0;
24
+ /** The first stray stdout lines, more of them than are shown: a CLI's usage runs long. */
25
+ strayHead = [];
24
26
  stdoutPartial = '';
25
27
  /** The last stderr lines, and how many there were in all. */
26
28
  stderrTail = [];
@@ -48,6 +50,8 @@ export class Trace {
48
50
  this.strayCount++;
49
51
  if (this.stray.length < 2)
50
52
  this.stray.push(clean(line, 160));
53
+ if (this.strayHead.length < 60)
54
+ this.strayHead.push(clean(line, 240));
51
55
  }
52
56
  }
53
57
  stderr(chunk) {
@@ -144,6 +148,39 @@ function strayFact(trace) {
144
148
  return ['stdout', `${quoted}${more}`];
145
149
  }
146
150
  const LOG_TO_STDERR = 'Log to stderr; stdout carries the protocol (in Node, console.error rather than console.log; in Python, logging or print(..., file=sys.stderr)).';
151
+ // A CLI's usage: argparse, click, yargs, commander, System.CommandLine, stricli.
152
+ const USAGE = /(?:^|\n)\s*(?:usage|(?:sub)?commands|options|examples|flags):|\bUSAGE\b|\s<command>|\[command\]|\bCOMMAND \[ARGS\]|arguments are required|subcommands? must be present|no command (?:specified|given)|required command was not provided|\b(?:run|see):? \S+ --help\b/i;
153
+ // A table of commands or options, two or more rows: " t3-watch start Run the service",
154
+ // " --version Show version number". Help output, even without a "Usage:" line.
155
+ const HELP_ROWS = /(?:^|\n)\s*(?:[\w.-]+ [\w-]+|-{1,2}[\w-]+(?:, -{1,2}[\w-]+)?) {2,}\S[^\n]*\n\s*(?:[\w.-]+ [\w-]+|-{1,2}[\w-]+(?:, -{1,2}[\w-]+)?) {2,}\S/;
156
+ // The subcommands that usually start the MCP server, most specific first. A word
157
+ // on its own ({mcp}, " mcp Start…", "mcp start [--transport…]"), not part of a
158
+ // name like agentation-mcp.
159
+ const SUBCOMMANDS = ['mcp start', 'mcp serve', 'server start', 'mcp', 'stdio', 'serve', 'server', 'start'];
160
+ // An HTTP server's address in its log: "MCP server listening on http://localhost:3001/mcp",
161
+ // "StreamableHTTP endpoint available at http://127.0.0.1:3333/mcp".
162
+ const HTTP_URL = /\b(?:listening|available|running|serving|started)\b[^\n]{0,60}?(https?:\/\/[^\s"'<>]+)/i;
163
+ const HTTP_PORT = /\blistening on (?:port )?:?(\d{2,5})\b/i;
164
+ /**
165
+ * What the server's own output says about how it was started, as a next step: a
166
+ * CLI that printed its usage and wants a subcommand, or a server that opened an
167
+ * HTTP port instead of talking on stdio. Undefined when it says neither.
168
+ */
169
+ export function startupHint(trace, command) {
170
+ const text = [...trace.strayHead, ...trace.stderrExcerpt().lines].join('\n');
171
+ const url = HTTP_URL.exec(text)?.[1]?.replace(/[.,;)]+$/, '').replace('//0.0.0.0', '//localhost');
172
+ const port = url ? undefined : HTTP_PORT.exec(text)?.[1];
173
+ if (url || port) {
174
+ const flag = /--stdio\b/.test(text) ? '--stdio' : /--transport[ =]?(?:\[?stdio|<)/i.test(text) ? '--transport stdio' : undefined;
175
+ const where = url ?? `http://localhost:${port}/mcp`;
176
+ return `The server started an HTTP server (${where}) instead of talking on its stdin and stdout. Start it in stdio mode (${flag ? `its output mentions ${flag}` : 'often --stdio or --transport stdio'}), or run toolmenu against the address while the server runs: toolmenu snapshot ${where}`;
177
+ }
178
+ if (USAGE.test(text) || HELP_ROWS.test(text)) {
179
+ const sub = SUBCOMMANDS.find((c) => new RegExp(`(?:^|[\\s{,|\\[])${c.replace(' ', '\\s+')}(?=[\\s},|\\]]|$)`, 'm').test(text));
180
+ return `The command printed its usage and exited: it wants a subcommand or an argument it wasn't given. ${sub ? `Its usage lists “${sub}”: try ${command} ${sub}` : 'Its usage (above) or its docs say what starts the MCP server: add that after the command.'}`;
181
+ }
182
+ return undefined;
183
+ }
147
184
  /** The error codes along an error's cause chain, innermost last. */
148
185
  function causes(error) {
149
186
  const out = [];
@@ -286,12 +323,16 @@ function explainStdio(error, trace, context, target) {
286
323
  const stderrText = trace.stderrExcerpt().lines.join('\n');
287
324
  const rejected = rejectedCredentials(stderrText);
288
325
  let next;
326
+ let hint;
289
327
  if (/cannot find (module|package)|ERR_MODULE_NOT_FOUND|ModuleNotFoundError|No module named/i.test(stderrText)) {
290
328
  next = "The server is missing one of its own dependencies (see stderr): reinstall it, or try another version. It's a packaging problem, not your setup.";
291
329
  }
292
330
  else if (rejected) {
293
331
  next = REJECTED_NEXT;
294
332
  }
333
+ else if ((hint = startupHint(trace, command))) {
334
+ next = hint;
335
+ }
295
336
  else if (trace.strayCount) {
296
337
  next = `${LOG_TO_STDERR} Then fix what stderr says.`;
297
338
  }
@@ -329,13 +370,16 @@ function explainStdio(error, trace, context, target) {
329
370
  : rejected
330
371
  ? `${stage} timed out after ${seconds(waitedHere)}: the server is running but never answered; its stderr looks like rejected credentials (“${rejected}”).`
331
372
  : `${stage} timed out after ${seconds(waitedHere)}: the server is running but never answered.`;
332
- const next = trace.strayCount
333
- ? LOG_TO_STDERR
334
- : rejected
335
- ? REJECTED_NEXT
336
- : stage === 'tools/list'
337
- ? 'The server answered initialize, then hung on tools/list: check its logs (stderr above). If building the list is just slow, raise --timeout.'
338
- : 'Check the command starts an MCP server on stdio (some need an argument such as "stdio" or "--stdio"). If it is only slow to start (npx downloading, a first build), raise --timeout.';
373
+ const hint = startupHint(trace, command);
374
+ const next = hint
375
+ ? hint
376
+ : trace.strayCount
377
+ ? LOG_TO_STDERR
378
+ : rejected
379
+ ? REJECTED_NEXT
380
+ : stage === 'tools/list'
381
+ ? 'The server answered initialize, then hung on tools/list: check its logs (stderr above). If building the list is just slow, raise --timeout.'
382
+ : 'Check the command starts an MCP server on stdio (some need an argument such as "stdio" or "--stdio"). If it is only slow to start (npx downloading, a first build), raise --timeout.';
339
383
  return new ConnectError(headline, facts, next, stage, { cause: error });
340
384
  }
341
385
  // Anything else: the transport's words, with what the server said.
@@ -1,4 +1,4 @@
1
- export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'invalid-arguments' | 'other';
1
+ export type FailureClass = 'auth' | 'environment' | 'network' | 'not-found' | 'client' | 'invalid-arguments' | 'other';
2
2
  /** Failures of the setup (credentials, this machine, the network), not of the server. */
3
3
  export declare const SETUP_FAILURES: Set<FailureClass>;
4
4
  /**
package/dist/failures.js CHANGED
@@ -5,16 +5,23 @@
5
5
  * as a finding.
6
6
  */
7
7
  /** Failures of the setup (credentials, this machine, the network), not of the server. */
8
- export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found']);
8
+ export const SETUP_FAILURES = new Set(['auth', 'environment', 'network', 'not-found', 'client']);
9
+ // A tool that asks the client for something toolmenu doesn't offer: "Cannot request
10
+ // input 'roots' (roots/list): the request's client capabilities do not declare the
11
+ // required capability" (server-everything's get-roots-list), sampling, elicitation.
12
+ const CLIENT = /client capabilit(?:y|ies)\b[^.]{0,60}\b(?:do(?:es)? not|don't|doesn't|lack|missing|without)\b|\b(?:roots|sampling|elicitation)\b[^.]{0,40}\bnot (?:supported|declared|available)\b[^.]{0,20}\bclient\b|client does not support (?:roots|sampling|elicitation)/i;
9
13
  // Something this machine lacks: "Could not find Google Chrome executable"
10
- // (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright).
11
- const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b/i;
14
+ // (Chrome DevTools), "Chromium distribution 'chrome' is not found at" (Playwright),
15
+ // or the wrong operating system: "Editmamei runs on Windows and macOS; this process
16
+ // is on "linux"" (editmamei), "Unsupported platform: linux".
17
+ const ENVIRONMENT = /could not find [^.]{0,40}executable|is not found at \/|\bnot installed\b|command not found|cannot find (module|package)|\bENOENT\b|\bruns on [^.;]{0,40}; this (?:process|machine|system) is on\b|\b(?:unsupported|not supported on (?:this|the current)) (?:platform|operating system|OS)\b/i;
12
18
  // "401 Bad credentials" (GitHub), "Authorization Expired" (Sentry), "Invalid
13
19
  // token" (Firecrawl), "authentication token is not valid" (Apify), "Please provide
14
20
  // a valid access token" (Supabase), "NetlifyUnauthError: You're not logged into
15
- // Netlify" (Netlify), "The API key you provided was rejected" (Pinecone). Not a
16
- // bare "token": plenty of errors mention one.
17
- const AUTH = /\b(401|403)\b|unauthori[sz]ed|unauth(enticated|error)|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|not logged (?:in|into)\b(?! to the browser)|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|(api[ _-]?key|access token|credentials)\b[^.]{0,30}\b(was|were|is|are) (rejected|refused|revoked)|valid (access )?token/i;
21
+ // Netlify" (Netlify), "The API key you provided was rejected" (Pinecone), "Admin
22
+ // auth is missing", "Runtime credentials are missing" (securenow). Not a bare
23
+ // "token": plenty of errors mention one.
24
+ const AUTH = /\b(?:auth|authentication|credentials?|api[ _-]?key|access token)\b (?:is |are )?missing\b|\b(401|403)\b|unauthori[sz]ed|unauth(enticated|error)|forbidden|bad credentials|authori[sz]ation expired|re-?authori[sz]e|not authenticated|not logged (?:in|into)\b(?! to the browser)|authentication (token|failed|required|error)|(invalid|expired|revoked) (api[ _-]?key|access token|token|credentials)|(token|api[ _-]?key|credentials) (is |are )?(not valid|invalid|expired)|(api[ _-]?key|access token|credentials)\b[^.]{0,30}\b(was|were|is|are) (rejected|refused|revoked)|valid (access )?token/i;
18
25
  const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|EHOSTUNREACH)\b|socket hang up|getaddrinfo|network error/i;
19
26
  // "Either 'page_id' OR both 'title' and 'space_key' must be provided" (Atlassian),
20
27
  // "**Input Error**" (Sentry), JSON-RPC's invalid params.
@@ -151,6 +158,8 @@ export async function patiently(request, options = {}) {
151
158
  * with 404; a made-up Jira URL 404s), which is setup, not the server's fault.
152
159
  */
153
160
  export function classifyFailure(text, options = {}) {
161
+ if (CLIENT.test(text))
162
+ return 'client';
154
163
  if (ENVIRONMENT.test(text))
155
164
  return 'environment';
156
165
  if (AUTH.test(text))
@@ -168,6 +177,7 @@ export const FAILURE_LABELS = {
168
177
  environment: 'something missing on this machine',
169
178
  network: 'the network',
170
179
  'not-found': 'a 404 on a call that named nothing (usually the account or site behind the credentials)',
180
+ client: "a client feature toolmenu doesn't offer (roots, sampling or elicitation)",
171
181
  'invalid-arguments': 'the arguments',
172
182
  other: 'other errors',
173
183
  };
@@ -184,6 +194,8 @@ export function setupFailureAdvice(classes, transport) {
184
194
  steps.push(`If the errors are about credentials, rerun with real ones (${credentials}); a test account is enough.`);
185
195
  if (set.has('environment'))
186
196
  steps.push('If something is missing on this machine (a browser, a binary, a module), install it where the server runs, or run the server in its own container image.');
197
+ if (set.has('client'))
198
+ steps.push("If a tool asks the client for roots, sampling or elicitation, toolmenu doesn't provide them: those calls only checked that the menu stays put, and the server isn't at fault.");
187
199
  if (set.has('network'))
188
200
  steps.push('If the server calls an upstream API, check this machine can reach it (DNS, proxy, firewall).');
189
201
  return { fix: steps.join(' ') || 'Check the quoted errors: they came from before the tool ran.', confidence: 'unsure' };
@@ -17,7 +17,10 @@ export const gated = {
17
17
  // Only an unlock whose parameter lists what it unlocks (an enum of toolsets or
18
18
  // domains), the kind session --auto walks: "Enables Cross-Region Restore on a
19
19
  // vault" turns on a feature, not tools (Azure, 418 tools).
20
- const found = unlockers(ctx.menu.tools).filter((u) => u.param && u.values.length);
20
+ // And only one whose name or description says it unlocks tools: an enum of
21
+ // domains alone is as often data (Garmin's garmin_sync { domains: [sleep, hrv] }
22
+ // downloads them). session --auto still tries such a tool once.
23
+ const found = unlockers(ctx.menu.tools).filter((u) => u.param && u.values.length && u.claims);
21
24
  if (!found.length)
22
25
  return [];
23
26
  // A value whose tools are already listed adds nothing: the same server started
package/dist/rules/ids.js CHANGED
@@ -3,7 +3,8 @@ import { kept } from './rule.js';
3
3
  // form_id, form-id, formId, formID. Not uid or paid: the suffix needs a boundary.
4
4
  // Not "key" either: in real menus it's as often a keyboard key as an identifier.
5
5
  const ID_SUFFIX = /^(.+?)[_-](id|ids|uuid|uuids)$/i;
6
- const CAMEL_ID = /^([a-z][a-zA-Z0-9]*?)(Id|Ids|ID|IDs|Uuid)$/;
6
+ // UUID before ID: viewUUID is a view's ID, not a viewUU's.
7
+ const CAMEL_ID = /^([a-z][a-zA-Z0-9]*?)(UUIDs?|Uuids?|IDs|ID|Ids|Id)$/;
7
8
  const BARE_ID = /^(id|ids|uid|uuid)$/i;
8
9
  const CREATE_VERBS = new Set(['create', 'add', 'new', 'insert', 'post', 'upsert', 'start', 'run', 'submit', 'duplicate', 'copy']);
9
10
  // Words that join alternatives in a name: projectSlugOrId → project.
@@ -6,6 +6,7 @@ import { route, sharedWord, vagueId } from './naming.js';
6
6
  import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
7
7
  import { noDryRun, unannotated } from './write.js';
8
8
  import { gated } from './gated.js';
9
+ import { findable } from './search.js';
9
10
  export const MENU_RULES = [
10
11
  nondeterministic,
11
12
  duplicateName,
@@ -16,6 +17,7 @@ export const MENU_RULES = [
16
17
  vagueId,
17
18
  sharedWord,
18
19
  route,
20
+ findable,
19
21
  authoredIds,
20
22
  buried,
21
23
  cut,
@@ -0,0 +1,2 @@
1
+ import { type Rule } from './rule.js';
2
+ export declare const findable: Rule;
@@ -0,0 +1,70 @@
1
+ import { buildIndex, placeOf, SEARCH_LIMIT } from '../search.js';
2
+ import { commonWords, VERBS, words } from '../words.js';
3
+ import { clip, count, kept, names, num } from './rule.js';
4
+ /**
5
+ * When a client defers tools and lets the model search for them (Claude's tool
6
+ * search, OpenAI's tool_search, Claude Code's deferred MCP tools), a tool the
7
+ * search doesn't return can't be called. The search returns 5 tools unless the
8
+ * model asks for more, so a tool that a search for its own name doesn't put in
9
+ * the top 5 is one the agent can miss.
10
+ *
11
+ * Only for menus a client would defer: the tool search docs suggest it from 10
12
+ * tools or 10K tokens of definitions.
13
+ */
14
+ const MIN_TOOLS = 10;
15
+ const MIN_TOKENS = 10_000;
16
+ export const findable = {
17
+ id: 'search/findable',
18
+ severity: 'info',
19
+ summary: 'Each tool comes up when searched for by its own name, for clients that defer tools',
20
+ run(ctx) {
21
+ const tools = ctx.menu.tools;
22
+ if (tools.length < MIN_TOOLS || ((ctx.menu.totalTokens ?? 0) < MIN_TOKENS && tools.length < 30))
23
+ return [];
24
+ const index = buildIndex(tools);
25
+ const findings = [];
26
+ // The words of a name that say something: not a prefix most tools share (firecrawl_, API-).
27
+ const shared = commonWords(tools.map((t) => t.name));
28
+ const missed = [];
29
+ for (const t of kept(tools, ctx)) {
30
+ // A bare verb ("create" from ads_create) isn't how an agent searches: then the
31
+ // prefix stays, since it's the object the tool acts on.
32
+ const all = words(t.name);
33
+ const own = all.filter((w) => !shared.has(w));
34
+ const query = (own.some((w) => !VERBS.has(w)) ? own : all).join(' ');
35
+ if (!query || !all.some((w) => !VERBS.has(w)))
36
+ continue;
37
+ const { place, ahead } = placeOf(index, query, t.name);
38
+ if (place === undefined || place > SEARCH_LIMIT)
39
+ missed.push({ tool: t.name, query, place, ahead });
40
+ }
41
+ if (missed.length) {
42
+ const first = missed[0];
43
+ findings.push({
44
+ ...(missed.length === 1 ? { tool: first.tool } : {}),
45
+ confidence: 'unsure',
46
+ message: `${count(missed.length, 'tool')} of ${num(tools.length)} ${missed.length === 1 ? "doesn't" : "don't"} come up in the top ${SEARCH_LIMIT} when searched for by ${missed.length === 1 ? 'its' : 'their'} own name. Clients that defer a menu this size let the model search it, and a search returns ${SEARCH_LIMIT} tools unless the model asks for more, so an agent looking for ${missed.length === 1 ? 'it' : 'them'} can be shown other tools instead. Ranked the way a BM25 tool search ranks them (name, description, argument names and descriptions); a client's own search can rank a little differently.`,
47
+ detail: missed.slice(0, 12).map((m) => `${m.tool}: “${m.query}” puts it ${m.place === undefined ? 'nowhere' : `at ${m.place}`}, behind ${names(m.ahead.slice(0, 4), 4)}${m.ahead.length > 4 ? ` and ${num(m.ahead.length - 4)} more` : ''}`).concat(missed.length > 12 ? [`and ${num(missed.length - 12)} more`] : []),
48
+ fix: 'Say the words of each tool\'s name in its own description, early on (a search ranks the description too, and "Returns the llms.txt" never says "guide"), give it a word the tools ahead of it lack (the object it acts on, not the verb), and cut repeats of other tools\' words from long descriptions: a description that mentions "domain" ten times outranks the tool named for domains.',
49
+ });
50
+ }
51
+ // routes.yml: a route's tools should be in what a search for it returns.
52
+ for (const [keyword, expectation] of Object.entries(ctx.routes ?? {})) {
53
+ const present = (expectation.must_match ?? []).filter((n) => tools.some((t) => t.name === n));
54
+ const lost = present
55
+ .map((n) => ({ n, ...placeOf(index, keyword, n) }))
56
+ .filter((r) => r.place === undefined || r.place > SEARCH_LIMIT);
57
+ for (const r of lost) {
58
+ findings.push({
59
+ tool: r.n,
60
+ severity: 'warn',
61
+ confidence: 'unsure',
62
+ message: `routes.yml says “${keyword}” should reach ${r.n}, but a tool search for “${keyword}” ${r.place === undefined ? "doesn't return it at all" : `puts it at ${r.place}`}, past the ${SEARCH_LIMIT} a search returns by default. A client that defers tools may never show it.`,
63
+ detail: [`ahead of it: ${names(r.ahead.slice(0, 5), 5)}`, `${r.n}: ${clip(tools.find((t) => t.name === r.n)?.description ?? '', 120)}`],
64
+ fix: `Put the words of “${keyword}” in ${r.n}'s name or the start of its description, or take them out of the tools ahead of it.`,
65
+ });
66
+ }
67
+ }
68
+ return findings;
69
+ },
70
+ };
@@ -0,0 +1,24 @@
1
+ import type { MenuTool } from './types.js';
2
+ /** How many tools a search returns when the model doesn't set a limit. */
3
+ export declare const SEARCH_LIMIT = 5;
4
+ /** The words a search sees in a tool: its name, description, and every argument's name and description. */
5
+ export declare function searchText(tool: MenuTool): string[];
6
+ /** Lowercase, singular, without stop words: what a query and a document are compared on. */
7
+ export declare function terms(text: string): string[];
8
+ export interface SearchIndex {
9
+ /** Tools ranked for a query, best first, with their scores. Tools that match no term are left out. */
10
+ rank(query: string): {
11
+ tool: string;
12
+ score: number;
13
+ }[];
14
+ }
15
+ export declare function buildIndex(tools: MenuTool[]): SearchIndex;
16
+ /**
17
+ * Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
18
+ * (only tools that score strictly higher are ahead of it), so a miss is never a
19
+ * coin flip. Undefined when the query matches nothing in the tool.
20
+ */
21
+ export declare function placeOf(index: SearchIndex, query: string, tool: string): {
22
+ place?: number;
23
+ ahead: string[];
24
+ };
package/dist/search.js ADDED
@@ -0,0 +1,86 @@
1
+ import { singular, words } from './words.js';
2
+ /** How many tools a search returns when the model doesn't set a limit. */
3
+ export const SEARCH_LIMIT = 5;
4
+ // Words that carry no meaning in a query or a description.
5
+ const STOP = new Set([
6
+ 'a', 'an', 'the', 'of', 'for', 'to', 'in', 'on', 'at', 'by', 'from', 'with', 'and', 'or', 'is', 'are', 'be',
7
+ 'this', 'that', 'it', 'its', 'as', 'use', 'when', 'if', 'you', 'your', 'can', 'will', 'not', 'any', 'all',
8
+ ]);
9
+ const K1 = 1.2;
10
+ const B = 0.75;
11
+ /** The words a search sees in a tool: its name, description, and every argument's name and description. */
12
+ export function searchText(tool) {
13
+ const parts = [tool.name, tool.description ?? ''];
14
+ const walk = (schema, depth) => {
15
+ if (!schema || typeof schema !== 'object' || depth > 6)
16
+ return;
17
+ const s = schema;
18
+ for (const [name, prop] of Object.entries(s.properties ?? {})) {
19
+ parts.push(name);
20
+ const d = prop?.description;
21
+ if (typeof d === 'string')
22
+ parts.push(d);
23
+ walk(prop, depth + 1);
24
+ }
25
+ if (s.items)
26
+ walk(s.items, depth + 1);
27
+ };
28
+ walk(tool.inputSchema, 0);
29
+ return terms(parts.join(' '));
30
+ }
31
+ /** Lowercase, singular, without stop words: what a query and a document are compared on. */
32
+ export function terms(text) {
33
+ return words(text)
34
+ .map(singular)
35
+ .filter((w) => w.length > 1 && !STOP.has(w));
36
+ }
37
+ export function buildIndex(tools) {
38
+ const docs = tools.map((t) => {
39
+ const tf = new Map();
40
+ const text = searchText(t);
41
+ for (const w of text)
42
+ tf.set(w, (tf.get(w) ?? 0) + 1);
43
+ return { name: t.name, tf, length: text.length };
44
+ });
45
+ const avg = docs.reduce((n, d) => n + d.length, 0) / Math.max(docs.length, 1);
46
+ const df = new Map();
47
+ for (const d of docs)
48
+ for (const w of d.tf.keys())
49
+ df.set(w, (df.get(w) ?? 0) + 1);
50
+ const idf = (w) => {
51
+ const n = df.get(w) ?? 0;
52
+ return Math.log(1 + (docs.length - n + 0.5) / (n + 0.5));
53
+ };
54
+ return {
55
+ rank(query) {
56
+ const q = [...new Set(terms(query))];
57
+ const out = [];
58
+ for (const d of docs) {
59
+ let score = 0;
60
+ for (const w of q) {
61
+ const f = d.tf.get(w);
62
+ if (!f)
63
+ continue;
64
+ score += idf(w) * ((f * (K1 + 1)) / (f + K1 * (1 - B + (B * d.length) / (avg || 1))));
65
+ }
66
+ if (score > 0)
67
+ out.push({ tool: d.name, score });
68
+ }
69
+ // Ties keep the menu's order, as a stable search would.
70
+ return out.sort((a, b) => b.score - a.score);
71
+ },
72
+ };
73
+ }
74
+ /**
75
+ * Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
76
+ * (only tools that score strictly higher are ahead of it), so a miss is never a
77
+ * coin flip. Undefined when the query matches nothing in the tool.
78
+ */
79
+ export function placeOf(index, query, tool) {
80
+ const ranked = index.rank(query);
81
+ const mine = ranked.find((r) => r.tool === tool);
82
+ if (!mine)
83
+ return { ahead: ranked.slice(0, SEARCH_LIMIT).map((r) => r.tool) };
84
+ const ahead = ranked.filter((r) => r.score > mine.score + 1e-9).map((r) => r.tool);
85
+ return { place: ahead.length + 1, ahead };
86
+ }
package/dist/session.d.ts CHANGED
@@ -138,6 +138,11 @@ interface Unlocker {
138
138
  * character. Drops U+FFFD left over from a server that cut its own message by bytes.
139
139
  */
140
140
  export declare function clip(text: string, max: number): string;
141
+ /**
142
+ * Absolute paths longer than 40 characters, kept to their last two parts:
143
+ * /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
144
+ */
145
+ export declare function shortenPaths(text: string): string;
141
146
  export declare function unlockers(tools: MenuTool[]): Unlocker[];
142
147
  /**
143
148
  * Unlocks whose values the schema doesn't give (no enum), each with a read-only
package/dist/session.js CHANGED
@@ -490,7 +490,7 @@ function untested(steps, target) {
490
490
  // Classified from the error's words: a guess, if a good one.
491
491
  confidence: 'unsure',
492
492
  message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
493
- detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
493
+ detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${clip((list[0].note ?? '').replace(/^the tool returned an error: /, ''), 100)}”)`),
494
494
  fix,
495
495
  },
496
496
  ];
@@ -509,6 +509,11 @@ function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], sto
509
509
  for (const u of unlockers(menu)) {
510
510
  if (!u.param || u.values.length === 0)
511
511
  continue;
512
+ // Never says it unlocks tools and the run didn't call it: nothing suggests tools
513
+ // are missing (Garmin's garmin_sync { domains } downloads data). One it called
514
+ // and stopped is covered by auto.stopped.
515
+ if (!u.claims && !steps.some((r) => scenario.steps[r.index - 1]?.kind === 'call' && scenario.steps[r.index - 1].tool === u.tool.name))
516
+ continue;
512
517
  const reached = new Set();
513
518
  let last;
514
519
  const add = (arg) => {
@@ -1063,13 +1068,31 @@ const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
1063
1068
  * character. Drops U+FFFD left over from a server that cut its own message by bytes.
1064
1069
  */
1065
1070
  export function clip(text, max) {
1066
- const chars = Array.from(text.replace(/\uFFFD+$/u, ''));
1071
+ let chars = Array.from(text.replace(/\uFFFD+$/u, ''));
1072
+ if (chars.length <= max)
1073
+ return chars.join('');
1074
+ // A long path loses its middle before the text loses its end: “spawn /tmp/…/x ENOENT”
1075
+ // cut at the end hides both the file and the reason.
1076
+ chars = Array.from(shortenPaths(chars.join('')));
1067
1077
  if (chars.length <= max)
1068
1078
  return chars.join('');
1069
1079
  const cut = chars.slice(0, max - 1).join('');
1070
1080
  const space = cut.lastIndexOf(' ');
1071
1081
  return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
1072
1082
  }
1083
+ /**
1084
+ * Absolute paths longer than 40 characters, kept to their last two parts:
1085
+ * /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
1086
+ */
1087
+ export function shortenPaths(text) {
1088
+ // Not a URL's path: a path starts after nothing that could be a scheme, host or word.
1089
+ return text.replace(/(?<![\w:/\\.])(?:[A-Za-z]:)?(?:[\\/][^\s\\/"'“”()]+){3,}/g, (path) => {
1090
+ if (path.length <= 40)
1091
+ return path;
1092
+ const sep = path.includes('\\') && !path.includes('/') ? '\\' : '/';
1093
+ return `…${sep}${path.split(/[\\/]/).slice(-2).join(sep)}`;
1094
+ });
1095
+ }
1073
1096
  function isWriteLike(t) {
1074
1097
  // Named as an unlock: changing the menu is its job, whatever its hints say
1075
1098
  // (toolception marks enable_toolset destructive). It's still only suggested,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.14.0",
3
+ "version": "0.15.0",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
5
  "author": "Nikša Perović (https://niksa.me)",
6
6
  "main": "dist/index.js",