toolmenu 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/cli.js +8 -2
- package/dist/clients.d.ts +63 -0
- package/dist/clients.js +102 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.js +4 -0
- package/dist/help.js +2 -1
- package/dist/options.d.ts +3 -0
- package/dist/options.js +1 -0
- package/dist/report.d.ts +2 -1
- package/dist/report.js +13 -1
- package/dist/rules/clients.d.ts +7 -0
- package/dist/rules/clients.js +31 -0
- package/dist/rules/index.js +2 -0
- package/dist/rules/rule.d.ts +2 -0
- package/dist/rules/search.js +13 -3
- package/dist/search.d.ts +4 -2
- package/dist/search.js +6 -4
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +7 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -394,6 +394,7 @@ server, or from the spec itself. If a rule can't point to one, it doesn't ship.
|
|
|
394
394
|
| `write/no-dry-run` | info | Destructive tools with no dry-run parameter or preview tool, as one summary |
|
|
395
395
|
| `spec/discover` | info | The server only speaks the 2025 protocol, so 2026-07-28 rules are skipped |
|
|
396
396
|
| `spec/deprecated`, `spec/cache-hints` | info | 2026-07-28: deprecated features still advertised; `ttlMs: 0`, or `cacheScope: "public"` on a server that took credentials |
|
|
397
|
+
| `naming/client-name` | warn | With `--client`: tool names the client's API refuses: `^[a-zA-Z0-9_-]{1,128}$` for the Claude API, `^[a-zA-Z0-9_-]+$` up to 128 for OpenAI's functions. MCP allows names such as `files.read` that neither takes as they are |
|
|
397
398
|
| `menu/auth-variance` | info | With `--as` (or `identities` in the config): the tool set differs by identity, what each adds, lacks or defines differently against the first, and its tokens. Allowed by the spec; a baseline of one identity's menu doesn't cover the others. An identity that can't be listed is said here too |
|
|
398
399
|
| `spec/cache-scope-public` | warn | 2026-07-28, with `--as`: the tool set differs by identity, and `tools/list` says `cacheScope: "public"`, which lets shared caches serve one caller's list to another |
|
|
399
400
|
|
|
@@ -443,6 +444,23 @@ Clients that send descriptions in full don't need a setting beyond that. If your
|
|
|
443
444
|
client cuts shorter, as mine did at 280, set its number. `fullDescriptions` lists the
|
|
444
445
|
tools it sends uncut, so the description rules skip them.
|
|
445
446
|
|
|
447
|
+
### Checking for one client (`--client`)
|
|
448
|
+
|
|
449
|
+
What an agent sees depends on its client. `snapshot --client <name>` (or `"client"` in the
|
|
450
|
+
config) checks the menu against what that client's own docs say, and every value
|
|
451
|
+
toolmenu uses names its source and the date it was read (`src/clients.ts`). What a
|
|
452
|
+
client doesn't document is left out, not guessed.
|
|
453
|
+
|
|
454
|
+
| `--client` | Up front | Search | Description cut | Names |
|
|
455
|
+
|---|---|---|---|---|
|
|
456
|
+
| `claude-code` | tool names; full definitions for tools marked `_meta["anthropic/alwaysLoad"]` | reads a description up to 4,096 characters | 16,384 through search, 4,096 up front | Claude API's |
|
|
457
|
+
| `claude-api-search` | nothing from tools you defer (keep the 3–5 used most loaded) | name, description, argument names and descriptions; 5 results by default | none documented | `^[a-zA-Z0-9_-]{1,128}$` |
|
|
458
|
+
| `openai` | not in OpenAI's spec, so no figure | not in the spec, so `search/findable` doesn't run | none documented | `^[a-zA-Z0-9_-]+$`, up to 128 |
|
|
459
|
+
|
|
460
|
+
The snapshot adds a line such as `as Claude Code: up front ~524 tokens · on demand ~344
|
|
461
|
+
per tool a search loads (largest: update_contacts, ~2,093)`, and `--json` a `client`
|
|
462
|
+
object. With a client that defers tools, `search/findable` runs at any menu size.
|
|
463
|
+
|
|
446
464
|
## What toolmenu can't catch
|
|
447
465
|
|
|
448
466
|
toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
@@ -475,6 +493,8 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
475
493
|
(2026-07-28 only; an error if the server lacks it)
|
|
476
494
|
--processes <n> server processes or connections to compare (default: 2);
|
|
477
495
|
1 opens no second one, session's scope check included
|
|
496
|
+
--client <name> snapshot: check the menu for one client, from its own docs:
|
|
497
|
+
claude-code, claude-api-search or openai
|
|
478
498
|
--as <name>:<cred> snapshot: list the menu as this identity too, repeatable:
|
|
479
499
|
"reader:Authorization: Bearer …" (HTTP) or "admin:ROLE=admin"
|
|
480
500
|
(stdio). The first is the menu saved; the others are compared
|
package/dist/cli.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { parseArgs } from 'node:util';
|
|
5
|
+
import { CLIENT_NAMES, CLIENTS } from './clients.js';
|
|
5
6
|
import { loadConfig } from './config.js';
|
|
6
7
|
import { ELICITATIONS, PROTOCOLS } from './connect.js';
|
|
7
8
|
import { diffMenus } from './diff.js';
|
|
@@ -82,6 +83,10 @@ export async function main(argv) {
|
|
|
82
83
|
const elicitation = (values.elicitation ?? 'decline');
|
|
83
84
|
if (!ELICITATIONS.includes(elicitation))
|
|
84
85
|
throw new UsageError('--elicitation must be decline or cancel');
|
|
86
|
+
if (values.client !== undefined && sub !== 'snapshot')
|
|
87
|
+
throw new UsageError('--client goes with snapshot.');
|
|
88
|
+
if (values.client !== undefined && !(values.client in CLIENTS))
|
|
89
|
+
throw new UsageError(`--client must be one of ${CLIENT_NAMES.join(', ')}`);
|
|
85
90
|
if (values.as !== undefined && sub !== 'snapshot')
|
|
86
91
|
throw new UsageError('--as goes with snapshot: it lists the menu once per identity.');
|
|
87
92
|
if (values.elicitation !== undefined && sub !== 'session')
|
|
@@ -300,11 +305,12 @@ export async function main(argv) {
|
|
|
300
305
|
const identities = values.as?.length ? parseIdentities(values.as, target.kind) : configIdentities(config.identities, target.kind);
|
|
301
306
|
const routesPath = values.routes ?? config.routes;
|
|
302
307
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
303
|
-
const
|
|
308
|
+
const client = values.client ?? config.client;
|
|
309
|
+
const { menu, findings, cost } = await snapshot(target, { routes, timeoutMs, processes, identities, client, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
304
310
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
305
311
|
if (outPath)
|
|
306
312
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
|
307
|
-
const output = formatSnapshot(menu, findings, format, outPath);
|
|
313
|
+
const output = formatSnapshot(menu, findings, format, outPath, cost);
|
|
308
314
|
if (output)
|
|
309
315
|
process.stdout.write(output + '\n');
|
|
310
316
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { Menu, MenuTool } from './types.js';
|
|
2
|
+
/** A value, where it was read, and when. */
|
|
3
|
+
export interface Sourced<T> {
|
|
4
|
+
value: T;
|
|
5
|
+
source: string;
|
|
6
|
+
read: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ClientProfile {
|
|
9
|
+
/** How the client is named in messages. */
|
|
10
|
+
label: string;
|
|
11
|
+
/**
|
|
12
|
+
* Whether MCP tools are deferred behind a tool search: `all` (every tool unless
|
|
13
|
+
* marked to load up front), `chosen` (the integrator marks each tool), or `none`.
|
|
14
|
+
*/
|
|
15
|
+
defers: Sourced<'all' | 'chosen' | 'none'>;
|
|
16
|
+
/** What the model has of a deferred tool before it searches: its name, or nothing. */
|
|
17
|
+
beforeSearch?: Sourced<'name' | 'nothing'>;
|
|
18
|
+
/** Where the client cuts a description: for a tool loaded up front, and one loaded through search. */
|
|
19
|
+
descriptionCut?: Sourced<{
|
|
20
|
+
upfront: number;
|
|
21
|
+
searched: number;
|
|
22
|
+
}>;
|
|
23
|
+
/** How much of a description the search reads. */
|
|
24
|
+
searchReads?: Sourced<number>;
|
|
25
|
+
/** What the search matches on. */
|
|
26
|
+
searchFields?: Sourced<string[]>;
|
|
27
|
+
/** How many tools a search returns unless the model asks for more. */
|
|
28
|
+
searchLimit?: Sourced<number>;
|
|
29
|
+
/** Tool names the API accepts. */
|
|
30
|
+
namePattern?: Sourced<{
|
|
31
|
+
pattern: string;
|
|
32
|
+
max: number;
|
|
33
|
+
api: string;
|
|
34
|
+
}>;
|
|
35
|
+
/** Advice on which tools to keep loaded up front. */
|
|
36
|
+
keepLoaded?: Sourced<string>;
|
|
37
|
+
}
|
|
38
|
+
export declare const CLIENTS: Record<string, ClientProfile>;
|
|
39
|
+
export declare const CLIENT_NAMES: string[];
|
|
40
|
+
/** The profile for a client name, or undefined for none given. Throws for an unknown name. */
|
|
41
|
+
export declare function clientProfile(name: string | undefined): ClientProfile | undefined;
|
|
42
|
+
/** Whether a tool name fits the client's API, and why not. */
|
|
43
|
+
export declare function nameProblem(name: string, profile: ClientProfile): string | undefined;
|
|
44
|
+
/** Whether Claude Code loads this tool up front: marked so by its server (`_meta["anthropic/alwaysLoad"]: true`). */
|
|
45
|
+
export declare function loadsUpfront(tool: MenuTool): boolean;
|
|
46
|
+
export interface ClientCost {
|
|
47
|
+
client: string;
|
|
48
|
+
label: string;
|
|
49
|
+
/** Tokens before any search: the deferred tools' names (Claude Code) and the definitions of tools loaded up front. Undefined when the client's docs don't say what the model sees. */
|
|
50
|
+
upfront?: number;
|
|
51
|
+
/** Tools loaded up front, in full. */
|
|
52
|
+
loaded: string[];
|
|
53
|
+
/** Average tokens a deferred tool adds when a search loads it, and the largest. */
|
|
54
|
+
perTool: number;
|
|
55
|
+
largest?: {
|
|
56
|
+
tool: string;
|
|
57
|
+
tokens: number;
|
|
58
|
+
};
|
|
59
|
+
/** One line saying what the numbers stand on. */
|
|
60
|
+
basis: string;
|
|
61
|
+
}
|
|
62
|
+
/** What the menu costs under a client that defers tools: up front, and per tool a search loads. */
|
|
63
|
+
export declare function clientCost(name: string, menu: Menu): ClientCost | undefined;
|
package/dist/clients.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* What an agent sees depends on its client. Each profile holds only what that
|
|
3
|
+
* client's own docs or API spec say, and every value carries where it was read and
|
|
4
|
+
* when. A value nobody has sourced is not here: toolmenu leaves that part out
|
|
5
|
+
* rather than guess (docs/plans/0.15-2026-readiness.md, point 6).
|
|
6
|
+
*/
|
|
7
|
+
import { countTokens } from './menu.js';
|
|
8
|
+
const CLAUDE_CODE_MCP = 'https://code.claude.com/docs/en/mcp';
|
|
9
|
+
const CLAUDE_TOOL_SEARCH = 'https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool';
|
|
10
|
+
const CLAUDE_DEFINE_TOOLS = 'https://platform.claude.com/docs/en/agents-and-tools/tool-use/define-tools';
|
|
11
|
+
const OPENAI_SPEC = 'https://github.com/openai/openai-openapi (openapi.yaml: FunctionToolParam, NamespaceToolParam, MCPTool)';
|
|
12
|
+
const READ = '2026-11-02';
|
|
13
|
+
const claudeName = { value: { pattern: '^[a-zA-Z0-9_-]{1,128}$', max: 128, api: 'the Claude API' }, source: CLAUDE_DEFINE_TOOLS, read: READ };
|
|
14
|
+
export const CLIENTS = {
|
|
15
|
+
'claude-code': {
|
|
16
|
+
label: 'Claude Code',
|
|
17
|
+
// "Tool search is enabled by default: MCP tools are deferred and discovered on demand."
|
|
18
|
+
defers: { value: 'all', source: CLAUDE_CODE_MCP, read: READ },
|
|
19
|
+
// "Only tool names and server instructions load at session start."
|
|
20
|
+
beforeSearch: { value: 'name', source: CLAUDE_CODE_MCP, read: READ },
|
|
21
|
+
// "Server instructions, and the description of a tool that loads upfront: 4,096 characters";
|
|
22
|
+
// "The description of a tool that Claude loads through tool search: 16,384 characters".
|
|
23
|
+
descriptionCut: { value: { upfront: 4096, searched: 16384 }, source: CLAUDE_CODE_MCP, read: READ },
|
|
24
|
+
// "When tool search matches Claude's query against a tool's description, it reads only the part within the shorter limit."
|
|
25
|
+
searchReads: { value: 4096, source: CLAUDE_CODE_MCP, read: READ },
|
|
26
|
+
namePattern: claudeName,
|
|
27
|
+
},
|
|
28
|
+
'claude-api-search': {
|
|
29
|
+
label: "Claude's tool search (Claude API)",
|
|
30
|
+
// "set defer_loading: true on the tools that shouldn't load up front"
|
|
31
|
+
defers: { value: 'chosen', source: CLAUDE_TOOL_SEARCH, read: READ },
|
|
32
|
+
// "Initially, Claude's context contains only the tool search tool and any non-deferred tools."
|
|
33
|
+
beforeSearch: { value: 'nothing', source: CLAUDE_TOOL_SEARCH, read: READ },
|
|
34
|
+
// "Both tool search variants (regex and bm25) search tool names, descriptions, argument names, and argument descriptions."
|
|
35
|
+
searchFields: { value: ['name', 'description', 'argument names', 'argument descriptions'], source: CLAUDE_TOOL_SEARCH, read: READ },
|
|
36
|
+
// "each search returns up to 5 matching tools by default"
|
|
37
|
+
searchLimit: { value: 5, source: CLAUDE_TOOL_SEARCH, read: READ },
|
|
38
|
+
keepLoaded: { value: 'Keep your 3–5 most frequently used tools non-deferred so Claude can call them without searching first.', source: CLAUDE_TOOL_SEARCH, read: READ },
|
|
39
|
+
namePattern: claudeName,
|
|
40
|
+
},
|
|
41
|
+
openai: {
|
|
42
|
+
label: "OpenAI's tool_search (Responses API)",
|
|
43
|
+
// FunctionToolParam.defer_loading, MCPTool.defer_loading: per function, or per MCP server.
|
|
44
|
+
defers: { value: 'chosen', source: OPENAI_SPEC, read: READ },
|
|
45
|
+
// FunctionToolParam.name: maxLength 128, pattern ^[a-zA-Z0-9_-]+$.
|
|
46
|
+
namePattern: { value: { pattern: '^[a-zA-Z0-9_-]+$', max: 128, api: "OpenAI's Responses API" }, source: OPENAI_SPEC, read: READ },
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
export const CLIENT_NAMES = Object.keys(CLIENTS);
|
|
50
|
+
/** The profile for a client name, or undefined for none given. Throws for an unknown name. */
|
|
51
|
+
export function clientProfile(name) {
|
|
52
|
+
if (name === undefined)
|
|
53
|
+
return undefined;
|
|
54
|
+
const profile = CLIENTS[name];
|
|
55
|
+
if (!profile)
|
|
56
|
+
throw new Error(`unknown client "${name}": one of ${CLIENT_NAMES.join(', ')}`);
|
|
57
|
+
return profile;
|
|
58
|
+
}
|
|
59
|
+
/** Whether a tool name fits the client's API, and why not. */
|
|
60
|
+
export function nameProblem(name, profile) {
|
|
61
|
+
const rule = profile.namePattern?.value;
|
|
62
|
+
if (!rule)
|
|
63
|
+
return undefined;
|
|
64
|
+
if (name.length > rule.max)
|
|
65
|
+
return `${name.length} characters, over ${rule.max}`;
|
|
66
|
+
if (!new RegExp(rule.pattern).test(name)) {
|
|
67
|
+
const bad = [...new Set([...name].filter((c) => !/[a-zA-Z0-9_-]/.test(c)))];
|
|
68
|
+
return `has ${bad.map((c) => (c === ' ' ? 'a space' : `"${c}"`)).join(', ')}, outside letters, digits, _ and -`;
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
/** Whether Claude Code loads this tool up front: marked so by its server (`_meta["anthropic/alwaysLoad"]: true`). */
|
|
73
|
+
export function loadsUpfront(tool) {
|
|
74
|
+
return tool._meta?.['anthropic/alwaysLoad'] === true;
|
|
75
|
+
}
|
|
76
|
+
/** What the menu costs under a client that defers tools: up front, and per tool a search loads. */
|
|
77
|
+
export function clientCost(name, menu) {
|
|
78
|
+
const profile = CLIENTS[name];
|
|
79
|
+
if (!profile || profile.defers.value === 'none')
|
|
80
|
+
return undefined;
|
|
81
|
+
const loaded = profile.defers.value === 'all' ? menu.tools.filter(loadsUpfront) : [];
|
|
82
|
+
const deferred = menu.tools.filter((t) => !loaded.includes(t));
|
|
83
|
+
const perTool = deferred.length ? Math.round(deferred.reduce((n, t) => n + t.tokens, 0) / deferred.length) : 0;
|
|
84
|
+
const largest = [...deferred].sort((a, b) => b.tokens - a.tokens)[0];
|
|
85
|
+
const loadedTokens = loaded.reduce((n, t) => n + t.tokens, 0);
|
|
86
|
+
const seen = profile.beforeSearch?.value;
|
|
87
|
+
const upfront = seen === 'name' ? countTokens(deferred.map((t) => t.name).join('\n')) + loadedTokens : seen === 'nothing' ? loadedTokens : undefined;
|
|
88
|
+
const basis = seen === 'name'
|
|
89
|
+
? `${profile.label} defers MCP tools and shows their names until a search loads one; the server's instructions load up front too and aren't counted here (${profile.beforeSearch.source}, read ${profile.beforeSearch.read})`
|
|
90
|
+
: seen === 'nothing'
|
|
91
|
+
? `with every tool deferred, the model sees none of them until a search loads one, and the docs advise keeping the 3–5 tools used most loaded (${profile.beforeSearch.source}, read ${profile.beforeSearch.read})`
|
|
92
|
+
: `${profile.label} defers per function or per MCP server; what the model sees before a search isn't in ${profile.defers.source}, so no up-front figure`;
|
|
93
|
+
return {
|
|
94
|
+
client: name,
|
|
95
|
+
label: profile.label,
|
|
96
|
+
...(upfront !== undefined ? { upfront } : {}),
|
|
97
|
+
loaded: loaded.map((t) => t.name),
|
|
98
|
+
perTool,
|
|
99
|
+
...(largest ? { largest: { tool: largest.name, tokens: largest.tokens } } : {}),
|
|
100
|
+
basis,
|
|
101
|
+
};
|
|
102
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -26,6 +26,8 @@ export interface Config {
|
|
|
26
26
|
headers?: Record<string, string>;
|
|
27
27
|
env?: Record<string, string>;
|
|
28
28
|
}>;
|
|
29
|
+
/** snapshot: the client to check the menu for (claude-code, claude-api-search, openai). --client wins. */
|
|
30
|
+
client?: string;
|
|
29
31
|
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
30
32
|
processes?: number;
|
|
31
33
|
/**
|
package/dist/config.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { CLIENTS } from './clients.js';
|
|
3
4
|
import { CLIENT_LIMITS } from './rules/description.js';
|
|
4
5
|
const DEFAULT_PATH = 'toolmenu.config.json';
|
|
5
6
|
/** Optional: no config file is fine. */
|
|
@@ -29,6 +30,9 @@ export async function loadConfig(path) {
|
|
|
29
30
|
if (config.processes !== undefined && !(Number.isInteger(config.processes) && config.processes >= 1)) {
|
|
30
31
|
throw new Error(`${file}: processes must be a whole number, 1 or more`);
|
|
31
32
|
}
|
|
33
|
+
if (config.client !== undefined && !(typeof config.client === 'string' && config.client in CLIENTS)) {
|
|
34
|
+
throw new Error(`${file}: client must be one of ${Object.keys(CLIENTS).join(', ')}`);
|
|
35
|
+
}
|
|
32
36
|
if (config.identities !== undefined) {
|
|
33
37
|
const strings = (v) => v === undefined || (typeof v === 'object' && v !== null && !Array.isArray(v) && Object.values(v).every((x) => typeof x === 'string'));
|
|
34
38
|
const ids = config.identities;
|
package/dist/help.js
CHANGED
|
@@ -19,6 +19,7 @@ const OPTIONS = {
|
|
|
19
19
|
env: ['--env <K=V>', ['environment variable for a stdio server, repeatable. The server only', 'gets a minimal environment (PATH, HOME, ...) plus these']],
|
|
20
20
|
protocol: ['--protocol <auto|legacy|modern>', ['which handshake to make (default: auto)', 'legacy: the 2025 handshake only; modern: 2026-07-28 only, an error if the server lacks it']],
|
|
21
21
|
as: ['--as <name>[:<credential>]', ['list the menu as this identity too, repeatable: a header ("reader:Authorization:', 'Bearer ...") or env var ("admin:ROLE=admin") on top of --header/--env. The first', 'is the main one (its menu is saved); the others are compared with it']],
|
|
22
|
+
client: ['--client <name>', ['check the menu for one client, from its own docs: claude-code (tools deferred,', 'its description cut and search), claude-api-search (Claude API tool search)', 'or openai (Responses API tool_search): cost up front and per tool, names it refuses']],
|
|
22
23
|
'no-auth': ['--no-auth', ["don't use a stored OAuth login (toolmenu auth login) for this server"]],
|
|
23
24
|
timeout: ['--timeout <ms>', ['per-request timeout (default: 30000)']],
|
|
24
25
|
processes: ['--processes <n>', ['server processes (stdio) or connections (HTTP) to compare, the main', 'one included (default: 2). 1 opens no second one: for servers with one', 'session per client']],
|
|
@@ -78,7 +79,7 @@ const HELP = {
|
|
|
78
79
|
'reports what in the menu confuses agents or breaks prompt caches.',
|
|
79
80
|
SERVER_NOTE,
|
|
80
81
|
],
|
|
81
|
-
own: ['out', 'no-write', 'routes', 'catalog', 'as'],
|
|
82
|
+
own: ['out', 'no-write', 'routes', 'catalog', 'as', 'client'],
|
|
82
83
|
shared: ['config', 'format', 'json', 'fail-on', 'header', 'env', 'no-auth', 'protocol', 'timeout', 'processes'],
|
|
83
84
|
examples: [
|
|
84
85
|
'toolmenu snapshot -- npx -y @modelcontextprotocol/server-memory',
|
package/dist/options.d.ts
CHANGED
package/dist/options.js
CHANGED
|
@@ -13,6 +13,7 @@ export const CLI_OPTIONS = {
|
|
|
13
13
|
header: { type: 'string', multiple: true },
|
|
14
14
|
env: { type: 'string', multiple: true },
|
|
15
15
|
as: { type: 'string', multiple: true },
|
|
16
|
+
client: { type: 'string' },
|
|
16
17
|
timeout: { type: 'string' },
|
|
17
18
|
protocol: { type: 'string' },
|
|
18
19
|
elicitation: { type: 'string' },
|
package/dist/report.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import type { ClientCost } from './clients.js';
|
|
1
2
|
import type { DiffResult } from './diff.js';
|
|
2
3
|
import type { HistoryResult } from './history.js';
|
|
3
4
|
import { type Scenario, type SessionResult } from './session.js';
|
|
4
5
|
import { type Finding, type Menu, type Severity } from './types.js';
|
|
5
6
|
export type Format = 'text' | 'json' | 'github' | 'markdown';
|
|
6
7
|
export declare function counts(findings: Finding[]): Record<Severity, number>;
|
|
7
|
-
export declare function formatSnapshot(menu: Menu, findings: Finding[], format: Format, outPath?: string): string;
|
|
8
|
+
export declare function formatSnapshot(menu: Menu, findings: Finding[], format: Format, outPath?: string, cost?: ClientCost): string;
|
|
8
9
|
export declare function formatDiff(d: DiffResult, beforeTools: number, afterTools: number, format: Format): string;
|
|
9
10
|
export declare function formatHistory(h: HistoryResult, format: Format, outDir: string, csvPath?: string): string;
|
|
10
11
|
export declare function formatSession(s: SessionResult, format: Format): string;
|
package/dist/report.js
CHANGED
|
@@ -12,7 +12,15 @@ export function counts(findings) {
|
|
|
12
12
|
function plural(n, word) {
|
|
13
13
|
return `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
14
14
|
}
|
|
15
|
-
|
|
15
|
+
/** --client: "as Claude Code: up front ~N tokens (…) · on demand ~M per tool loaded (largest: x, ~K)". */
|
|
16
|
+
function costLine(cost) {
|
|
17
|
+
const parts = [
|
|
18
|
+
cost.upfront !== undefined ? `up front ~${cost.upfront.toLocaleString('en-US')} tokens${cost.loaded.length ? ` (names, and ${plural(cost.loaded.length, 'tool')} loaded in full)` : ''}` : '',
|
|
19
|
+
`on demand ~${cost.perTool.toLocaleString('en-US')} per tool a search loads${cost.largest ? ` (largest: ${cost.largest.tool}, ~${cost.largest.tokens.toLocaleString('en-US')})` : ''}`,
|
|
20
|
+
].filter(Boolean);
|
|
21
|
+
return `as ${cost.label}: ${parts.join(' · ')}`;
|
|
22
|
+
}
|
|
23
|
+
export function formatSnapshot(menu, findings, format, outPath, cost) {
|
|
16
24
|
if (format === 'json') {
|
|
17
25
|
return JSON.stringify({
|
|
18
26
|
command: 'snapshot',
|
|
@@ -21,6 +29,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
21
29
|
totalTokens: menu.totalTokens,
|
|
22
30
|
listMeta: menu.listMeta,
|
|
23
31
|
breakdown: breakdown(menu),
|
|
32
|
+
...(cost ? { client: cost } : {}),
|
|
24
33
|
written: outPath ?? null,
|
|
25
34
|
counts: counts(findings),
|
|
26
35
|
findings,
|
|
@@ -34,6 +43,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
34
43
|
`### toolmenu snapshot: \`${s.name ?? 'server'}\` ${s.version ?? ''}`.trimEnd(),
|
|
35
44
|
'',
|
|
36
45
|
`Protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
46
|
+
...(cost ? ['', `${costLine(cost)}. _${cost.basis}._`] : []),
|
|
37
47
|
'',
|
|
38
48
|
...mdFindings(findings),
|
|
39
49
|
...mdBreakdown(menu),
|
|
@@ -44,6 +54,8 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
44
54
|
`toolmenu snapshot ${s.name ?? 'unknown server'} ${s.version ?? ''}`.trimEnd(),
|
|
45
55
|
` protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
46
56
|
];
|
|
57
|
+
if (cost)
|
|
58
|
+
lines.push(` ${costLine(cost)}`, ` ${cost.basis}`);
|
|
47
59
|
if (outPath)
|
|
48
60
|
lines.push(` wrote ${outPath}`);
|
|
49
61
|
lines.push('');
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type Rule } from './rule.js';
|
|
2
|
+
/**
|
|
3
|
+
* --client: tool names the client's API doesn't take. MCP allows names its APIs
|
|
4
|
+
* refuse ("files.read", "get user"): Claude's tool definitions must match
|
|
5
|
+
* ^[a-zA-Z0-9_-]{1,128}$, OpenAI's function names ^[a-zA-Z0-9_-]+$ up to 128.
|
|
6
|
+
*/
|
|
7
|
+
export declare const clientName: Rule;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { CLIENTS, nameProblem } from '../clients.js';
|
|
2
|
+
import { count, kept } from './rule.js';
|
|
3
|
+
/**
|
|
4
|
+
* --client: tool names the client's API doesn't take. MCP allows names its APIs
|
|
5
|
+
* refuse ("files.read", "get user"): Claude's tool definitions must match
|
|
6
|
+
* ^[a-zA-Z0-9_-]{1,128}$, OpenAI's function names ^[a-zA-Z0-9_-]+$ up to 128.
|
|
7
|
+
*/
|
|
8
|
+
export const clientName = {
|
|
9
|
+
id: 'naming/client-name',
|
|
10
|
+
severity: 'warn',
|
|
11
|
+
summary: "Tool names the client's API doesn't accept as they are",
|
|
12
|
+
run(ctx) {
|
|
13
|
+
const profile = ctx.client ? CLIENTS[ctx.client] : undefined;
|
|
14
|
+
if (!profile?.namePattern)
|
|
15
|
+
return [];
|
|
16
|
+
const bad = kept(ctx.menu.tools, ctx)
|
|
17
|
+
.map((t) => ({ tool: t.name, why: nameProblem(t.name, profile) }))
|
|
18
|
+
.filter((b) => !!b.why);
|
|
19
|
+
if (!bad.length)
|
|
20
|
+
return [];
|
|
21
|
+
const rule = profile.namePattern;
|
|
22
|
+
return [
|
|
23
|
+
{
|
|
24
|
+
...(bad.length === 1 ? { tool: bad[0].tool } : {}),
|
|
25
|
+
message: `${count(bad.length, 'tool name')} ${bad.length === 1 ? "doesn't" : "don't"} fit the tool name format of ${rule.value.api}${ctx.client === 'claude-code' ? `, which ${profile.label} sends tools to` : ''} (${rule.value.pattern}, at most ${rule.value.max} characters). MCP allows them, but a request that passes them to the API as they are is refused; a client that rewrites them shows the model a name that isn't the server's.`,
|
|
26
|
+
detail: [...bad.slice(0, 12).map((b) => `${b.tool}: ${b.why}`), ...(bad.length > 12 ? [`and ${bad.length - 12} more`] : []), `Source: ${rule.source} (read ${rule.read}).`],
|
|
27
|
+
fix: 'Use letters, digits, _ and - in tool names (files_read, not files.read), at most 128 characters.',
|
|
28
|
+
},
|
|
29
|
+
];
|
|
30
|
+
},
|
|
31
|
+
};
|
package/dist/rules/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import { SEVERITY_RANK } from '../types.js';
|
|
|
2
2
|
import { buried, cut, lateInstruction } from './description.js';
|
|
3
3
|
import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
|
+
import { clientName } from './clients.js';
|
|
5
6
|
import { handleLifetime } from './handles.js';
|
|
6
7
|
import { authVariance, cacheScopePublic } from './identities.js';
|
|
7
8
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
@@ -21,6 +22,7 @@ export const MENU_RULES = [
|
|
|
21
22
|
vagueId,
|
|
22
23
|
sharedWord,
|
|
23
24
|
route,
|
|
25
|
+
clientName,
|
|
24
26
|
findable,
|
|
25
27
|
authoredIds,
|
|
26
28
|
handleLifetime,
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface RuleContext {
|
|
|
28
28
|
server?: string;
|
|
29
29
|
/** snapshot --as: each identity's menu, the main one first (menu/auth-variance, spec/cache-scope-public). */
|
|
30
30
|
identities?: import('./identities.js').IdentityMenu[];
|
|
31
|
+
/** --client: the client profile the menu is checked for (src/clients.ts). */
|
|
32
|
+
client?: string;
|
|
31
33
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
|
32
34
|
descriptionLimit?: number | string;
|
|
33
35
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
package/dist/rules/search.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CLIENTS } from '../clients.js';
|
|
1
2
|
import { buildIndex, placeIn, placeOf, SEARCH_LIMIT } from '../search.js';
|
|
2
3
|
import { commonWords, VERBS, words } from '../words.js';
|
|
3
4
|
import { clip, count, kept, names, num } from './rule.js';
|
|
@@ -19,9 +20,18 @@ export const findable = {
|
|
|
19
20
|
summary: 'Each tool comes up when searched for by its own name, for clients that defer tools',
|
|
20
21
|
run(ctx) {
|
|
21
22
|
const tools = ctx.menu.tools;
|
|
22
|
-
|
|
23
|
+
const profile = ctx.client ? CLIENTS[ctx.client] : undefined;
|
|
24
|
+
if (profile) {
|
|
25
|
+
// A client that defers tools does it at any size; one whose search results
|
|
26
|
+
// aren't documented (OpenAI's spec doesn't say how many) isn't ranked against a guess.
|
|
27
|
+
if (profile.defers.value === 'none' || tools.length < 2)
|
|
28
|
+
return [];
|
|
29
|
+
if (!profile.searchLimit && ctx.client !== 'claude-code')
|
|
30
|
+
return [];
|
|
31
|
+
}
|
|
32
|
+
else if (tools.length < MIN_TOOLS || ((ctx.menu.totalTokens ?? 0) < MIN_TOKENS && tools.length < 30))
|
|
23
33
|
return [];
|
|
24
|
-
const index = buildIndex(tools);
|
|
34
|
+
const index = buildIndex(tools, { descriptionChars: profile?.searchReads?.value });
|
|
25
35
|
const findings = [];
|
|
26
36
|
// The words of a name that say something: not a prefix most tools share (firecrawl_, API-).
|
|
27
37
|
const shared = commonWords(tools.map((t) => t.name));
|
|
@@ -43,7 +53,7 @@ export const findable = {
|
|
|
43
53
|
findings.push({
|
|
44
54
|
...(missed.length === 1 ? { tool: first.tool } : {}),
|
|
45
55
|
confidence: 'unsure',
|
|
46
|
-
message: `${count(missed.length, 'tool')} of ${num(tools.length)} ${missed.length === 1 ? "doesn't" : "don't"} come up in the top ${SEARCH_LIMIT} when searched for by ${missed.length === 1 ? 'its' : 'their'} own name. Clients that defer a menu this size let the model search it, and a search returns ${SEARCH_LIMIT} tools unless the model asks for more, so an agent looking for ${missed.length === 1 ? 'it' : 'them'} can be shown other tools instead. Ranked the way a BM25 tool search ranks them (name, description, argument names and descriptions); a client's own search can rank a little differently.`,
|
|
56
|
+
message: `${count(missed.length, 'tool')} of ${num(tools.length)} ${missed.length === 1 ? "doesn't" : "don't"} come up in the top ${SEARCH_LIMIT} when searched for by ${missed.length === 1 ? 'its' : 'their'} own name. Clients that defer a menu this size let the model search it, and a search returns ${SEARCH_LIMIT} tools unless the model asks for more, so an agent looking for ${missed.length === 1 ? 'it' : 'them'} can be shown other tools instead. Ranked the way a BM25 tool search ranks them (name, description, argument names and descriptions${profile?.searchReads ? `, the description up to character ${num(profile.searchReads.value)}, all ${profile.label}'s search reads` : ''}); a client's own search can rank a little differently.`,
|
|
47
57
|
detail: missed.slice(0, 12).map((m) => `${m.tool}: “${m.query}” puts it ${m.place === undefined ? 'nowhere' : `at ${m.place}`}, behind ${names(m.ahead.slice(0, 4), 4)}${m.ahead.length > 4 ? ` and ${num(m.ahead.length - 4)} more` : ''}`).concat(missed.length > 12 ? [`and ${num(missed.length - 12)} more`] : []),
|
|
48
58
|
fix: 'Say the words of each tool\'s name in its own description, early on (a search ranks the description too, and "Returns the llms.txt" never says "guide"), give it a word the tools ahead of it lack (the object it acts on, not the verb), and cut repeats of other tools\' words from long descriptions: a description that mentions "domain" ten times outranks the tool named for domains.',
|
|
49
59
|
});
|
package/dist/search.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { MenuTool } from './types.js';
|
|
|
2
2
|
/** How many tools a search returns when the model doesn't set a limit. */
|
|
3
3
|
export declare const SEARCH_LIMIT = 5;
|
|
4
4
|
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
5
|
-
export declare function searchText(tool: MenuTool): string[];
|
|
5
|
+
export declare function searchText(tool: MenuTool, descriptionChars?: number): string[];
|
|
6
6
|
/** Lowercase, singular, without stop words: what a query and a document are compared on. */
|
|
7
7
|
export declare function terms(text: string): string[];
|
|
8
8
|
export interface SearchIndex {
|
|
@@ -12,7 +12,9 @@ export interface SearchIndex {
|
|
|
12
12
|
score: number;
|
|
13
13
|
}[];
|
|
14
14
|
}
|
|
15
|
-
export declare function buildIndex(tools: MenuTool[]
|
|
15
|
+
export declare function buildIndex(tools: MenuTool[], options?: {
|
|
16
|
+
descriptionChars?: number;
|
|
17
|
+
}): SearchIndex;
|
|
16
18
|
/**
|
|
17
19
|
* Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
|
|
18
20
|
* (only tools that score strictly higher are ahead of it), so a miss is never a
|
package/dist/search.js
CHANGED
|
@@ -9,8 +9,10 @@ const STOP = new Set([
|
|
|
9
9
|
const K1 = 1.2;
|
|
10
10
|
const B = 0.75;
|
|
11
11
|
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
12
|
-
export function searchText(tool) {
|
|
13
|
-
|
|
12
|
+
export function searchText(tool, descriptionChars) {
|
|
13
|
+
// A client whose search reads only the start of a description (Claude Code: the first 4,096 characters).
|
|
14
|
+
const description = tool.description ?? '';
|
|
15
|
+
const parts = [tool.name, descriptionChars !== undefined ? description.slice(0, descriptionChars) : description];
|
|
14
16
|
const walk = (schema, depth) => {
|
|
15
17
|
if (!schema || typeof schema !== 'object' || depth > 6)
|
|
16
18
|
return;
|
|
@@ -34,10 +36,10 @@ export function terms(text) {
|
|
|
34
36
|
.map(singular)
|
|
35
37
|
.filter((w) => w.length > 1 && !STOP.has(w));
|
|
36
38
|
}
|
|
37
|
-
export function buildIndex(tools) {
|
|
39
|
+
export function buildIndex(tools, options = {}) {
|
|
38
40
|
const docs = tools.map((t) => {
|
|
39
41
|
const tf = new Map();
|
|
40
|
-
const text = searchText(t);
|
|
42
|
+
const text = searchText(t, options.descriptionChars);
|
|
41
43
|
for (const w of text)
|
|
42
44
|
tf.set(w, (tf.get(w) ?? 0) + 1);
|
|
43
45
|
return { name: t.name, tf, length: text.length };
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type Target } from './connect.js';
|
|
2
2
|
import { type Identity } from './probe.js';
|
|
3
3
|
import type { IdentityMenu } from './rules/identities.js';
|
|
4
|
+
import { type ClientCost } from './clients.js';
|
|
4
5
|
import { type RuleSettings } from './rules/index.js';
|
|
5
6
|
import { type CatalogOptions } from './catalog.js';
|
|
6
7
|
import type { Routes } from './routes.js';
|
|
@@ -22,12 +23,16 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
22
23
|
* menu is the one saved and checked; the others are compared with it.
|
|
23
24
|
*/
|
|
24
25
|
identities?: Identity[];
|
|
26
|
+
/** --client: check the menu for this client (src/clients.ts): its description cut, its search, its name format. */
|
|
27
|
+
client?: string;
|
|
25
28
|
}
|
|
26
29
|
export interface SnapshotResult {
|
|
27
30
|
menu: Menu;
|
|
28
31
|
findings: Finding[];
|
|
29
32
|
/** snapshot --as: each identity's tools and tokens, or why it couldn't be listed. */
|
|
30
33
|
identities?: IdentityMenu[];
|
|
34
|
+
/** --client: what the menu costs up front and per tool a search loads. */
|
|
35
|
+
cost?: ClientCost;
|
|
31
36
|
}
|
|
32
37
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
33
38
|
export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
package/dist/snapshot.js
CHANGED
|
@@ -3,6 +3,7 @@ import { patiently, tooMany } from './failures.js';
|
|
|
3
3
|
import { ConnectError } from './explain.js';
|
|
4
4
|
import { buildMenu } from './menu.js';
|
|
5
5
|
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
|
+
import { clientCost } from './clients.js';
|
|
6
7
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
7
8
|
import { catalogFindings, readCatalog } from './catalog.js';
|
|
8
9
|
import { SEVERITY_RANK } from './types.js';
|
|
@@ -68,7 +69,10 @@ export async function snapshot(given, options = {}) {
|
|
|
68
69
|
capabilities: connection.capabilities,
|
|
69
70
|
usedAuth: connection.usedAuth,
|
|
70
71
|
routes: options.routes,
|
|
71
|
-
descriptionLimit
|
|
72
|
+
// A client profile's cut, unless descriptionLimit says otherwise. A client whose docs
|
|
73
|
+
// give no cut (Claude's API, OpenAI) isn't checked against Claude Code's.
|
|
74
|
+
descriptionLimit: options.descriptionLimit ?? (options.client === undefined ? undefined : options.client === 'claude-code' ? 'claude-code' : Number.POSITIVE_INFINITY),
|
|
75
|
+
client: options.client,
|
|
72
76
|
fullDescriptions: options.fullDescriptions,
|
|
73
77
|
transport: target.kind,
|
|
74
78
|
probes,
|
|
@@ -89,7 +93,8 @@ export async function snapshot(given, options = {}) {
|
|
|
89
93
|
}
|
|
90
94
|
findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
91
95
|
}
|
|
92
|
-
|
|
96
|
+
const cost = options.client ? clientCost(options.client, menu) : undefined;
|
|
97
|
+
return { menu, findings, ...(identities ? { identities } : {}), ...(cost ? { cost } : {}) };
|
|
93
98
|
}
|
|
94
99
|
const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
|
|
95
100
|
/**
|
package/package.json
CHANGED