toolmenu 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/auth.js +6 -4
- package/dist/auto.d.ts +11 -1
- package/dist/auto.js +8 -3
- package/dist/breakdown.js +3 -1
- package/dist/cli.js +100 -25
- package/dist/close.d.ts +8 -0
- package/dist/close.js +18 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/connect.d.ts +10 -0
- package/dist/connect.js +20 -7
- package/dist/diff.js +48 -6
- package/dist/explain.d.ts +10 -0
- package/dist/explain.js +29 -2
- package/dist/failures.d.ts +6 -0
- package/dist/failures.js +42 -2
- package/dist/help.d.ts +2 -0
- package/dist/help.js +13 -6
- package/dist/init.d.ts +1 -0
- package/dist/init.js +1 -1
- package/dist/options.d.ts +6 -0
- package/dist/options.js +2 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +76 -2
- package/dist/rules/description.js +4 -3
- package/dist/rules/determinism.d.ts +1 -0
- package/dist/rules/determinism.js +30 -0
- package/dist/rules/gated.d.ts +7 -0
- package/dist/rules/gated.js +77 -0
- package/dist/rules/index.js +6 -2
- package/dist/rules/naming.js +26 -3
- package/dist/rules/rule.d.ts +7 -0
- package/dist/rules/spec.d.ts +6 -0
- package/dist/rules/spec.js +23 -0
- package/dist/rules/write.js +25 -7
- package/dist/session.d.ts +16 -1
- package/dist/session.js +54 -15
- package/dist/snapshot.d.ts +6 -0
- package/dist/snapshot.js +16 -0
- package/dist/types.d.ts +7 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
|
|
|
29
29
|
(`history` installs old versions with today's dependencies, so it records what they
|
|
30
30
|
serve now, which can differ from what they shipped with: FINDINGS F4.)
|
|
31
31
|
|
|
32
|
-
> **Status: 0.
|
|
32
|
+
> **Status: 0.14, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
|
|
33
33
|
> real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
34
34
|
|
|
35
35
|
## Start here
|
|
@@ -104,7 +104,7 @@ jobs:
|
|
|
104
104
|
- uses: actions/setup-node@v7
|
|
105
105
|
with: { node-version: 22 }
|
|
106
106
|
- run: npm ci && npm run build
|
|
107
|
-
- uses: niksa90/toolmenu@v0.
|
|
107
|
+
- uses: niksa90/toolmenu@v0.14.0
|
|
108
108
|
with:
|
|
109
109
|
command: node dist/server.js
|
|
110
110
|
baseline: menu.json # your committed snapshot
|
|
@@ -361,6 +361,9 @@ into description and schema), large enums, and waste you can remove:
|
|
|
361
361
|
- **Unused `$defs`**: definitions nothing in the tool refers to. Notion's server 2.5.2
|
|
362
362
|
ships the same 9 in all 24 tools, 72% of its menu.
|
|
363
363
|
|
|
364
|
+
Every tool's estimate is in the menu file, for the ones the summary leaves out:
|
|
365
|
+
`jq -r '.tools[] | "\(.tokens)\t\(.name)"' menu.json | sort -rn`.
|
|
366
|
+
|
|
364
367
|
## `snapshot` rules
|
|
365
368
|
|
|
366
369
|
Every rule comes from a failure I actually hit building 115 tools for one MCP
|
|
@@ -458,6 +461,8 @@ toolmenu sees what the server sends. Plenty of agent failures happen elsewhere:
|
|
|
458
461
|
--env <K=V> env var for a stdio server, repeatable (it only gets a
|
|
459
462
|
minimal environment otherwise)
|
|
460
463
|
--timeout <ms> per-request timeout (default: 30000)
|
|
464
|
+
--protocol <p> auto (default), legacy (2025 handshake only) or modern
|
|
465
|
+
(2026-07-28 only; an error if the server lacks it)
|
|
461
466
|
--processes <n> server processes or connections to compare (default: 2);
|
|
462
467
|
1 opens no second one, session's scope check included
|
|
463
468
|
```
|
|
@@ -482,9 +487,9 @@ or bad usage. `toolmenu <command> --help` lists that command's own options.
|
|
|
482
487
|
|
|
483
488
|
## About
|
|
484
489
|
|
|
485
|
-
Built by [
|
|
490
|
+
Built by [Nikša Perović](https://niksa.me) while running a 115-tool MCP server and watching
|
|
486
491
|
agents trip over its menu. Every rule started as one of those failures, and the
|
|
487
|
-
research behind them is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
492
|
+
research behind them is in [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md). The story of building it, and where it was wrong, is [on niksa.me](https://niksa.me/toolmenu).
|
|
488
493
|
|
|
489
494
|
Found a false positive, or a failure toolmenu should catch?
|
|
490
495
|
[Open an issue](https://github.com/niksa90/toolmenu/issues) with the menu (a
|
package/dist/auth.js
CHANGED
|
@@ -7,6 +7,7 @@ import { homedir } from 'node:os';
|
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
import { Client, StreamableHTTPClientTransport, UnauthorizedError } from '@modelcontextprotocol/client';
|
|
9
9
|
import { VERSION } from './version.js';
|
|
10
|
+
import { endSession } from './close.js';
|
|
10
11
|
/**
|
|
11
12
|
* The redirect port. Fixed, not random: dynamic registration records the exact
|
|
12
13
|
* redirect URI, so a new port would invalidate the stored client on every login.
|
|
@@ -218,7 +219,7 @@ export async function login(serverUrl, options = {}) {
|
|
|
218
219
|
try {
|
|
219
220
|
await client.connect(transport);
|
|
220
221
|
// Already logged in with a valid (or refreshed) token.
|
|
221
|
-
await client
|
|
222
|
+
await endSession(client, transport);
|
|
222
223
|
}
|
|
223
224
|
catch (error) {
|
|
224
225
|
if (!(error instanceof UnauthorizedError))
|
|
@@ -233,7 +234,7 @@ export async function login(serverUrl, options = {}) {
|
|
|
233
234
|
throw new Error(`The login callback for ${serverUrl} carried the wrong state (not the one this login sent), so it was ignored (an old browser tab, or a second login at once).\n→ Next: Run toolmenu auth login ${serverUrl} again and use the tab it opens.`);
|
|
234
235
|
}
|
|
235
236
|
await transport.finishAuth(params);
|
|
236
|
-
await client
|
|
237
|
+
await endSession(client, transport);
|
|
237
238
|
}
|
|
238
239
|
return await check(serverUrl);
|
|
239
240
|
}
|
|
@@ -256,13 +257,14 @@ export async function login(serverUrl, options = {}) {
|
|
|
256
257
|
/** Connect with the stored login and list tools: the login works. */
|
|
257
258
|
async function check(serverUrl) {
|
|
258
259
|
const client = new Client({ name: 'toolmenu', version: VERSION });
|
|
259
|
-
|
|
260
|
+
const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { authProvider: new StoredOAuthProvider(serverUrl) });
|
|
261
|
+
await client.connect(transport);
|
|
260
262
|
try {
|
|
261
263
|
const { tools } = await client.listTools();
|
|
262
264
|
return { name: client.getServerVersion()?.name, tools: tools.length };
|
|
263
265
|
}
|
|
264
266
|
finally {
|
|
265
|
-
await client
|
|
267
|
+
await endSession(client, transport);
|
|
266
268
|
}
|
|
267
269
|
}
|
|
268
270
|
export async function logout(serverUrl) {
|
package/dist/auto.d.ts
CHANGED
|
@@ -13,6 +13,11 @@ export interface AutoOptions {
|
|
|
13
13
|
openWorld?: boolean;
|
|
14
14
|
/** At most this many calls (default 20). */
|
|
15
15
|
maxCalls?: number;
|
|
16
|
+
/**
|
|
17
|
+
* Only the calls: no list first, no repeat of the first call at the end. For
|
|
18
|
+
* tools that appear mid-session, planned into a session already running.
|
|
19
|
+
*/
|
|
20
|
+
callsOnly?: boolean;
|
|
16
21
|
/**
|
|
17
22
|
* Values for required parameters, by name (`repo_path`) or for one tool
|
|
18
23
|
* (`get_issue.issue_key`). Used before anything is synthesized.
|
|
@@ -26,7 +31,7 @@ export interface AutoOptions {
|
|
|
26
31
|
*/
|
|
27
32
|
assumeReadOnly?: string[];
|
|
28
33
|
}
|
|
29
|
-
export type SkipReason = 'not read-only' | 'unmarked' | 'open world' | 'needs values' | 'over the call budget';
|
|
34
|
+
export type SkipReason = 'not read-only' | 'unmarked' | 'open world' | 'needs values' | 'over the call budget' | 'over the per-unlock budget';
|
|
30
35
|
export interface AutoPlan {
|
|
31
36
|
scenario: Scenario;
|
|
32
37
|
called: string[];
|
|
@@ -45,10 +50,15 @@ export interface AutoPlan {
|
|
|
45
50
|
input: string;
|
|
46
51
|
why: string;
|
|
47
52
|
}[];
|
|
53
|
+
/** Calls counted against --max-calls (unlocks aren't). */
|
|
54
|
+
spent: number;
|
|
55
|
+
/** Unlock calls the plan makes, one per value (not the repeat): each gets its own budget for the tools it brings. */
|
|
56
|
+
unlockValues: number;
|
|
48
57
|
}
|
|
49
58
|
/** What a session report shows about an --auto plan. */
|
|
50
59
|
export type AutoSummary = Pick<AutoPlan, 'called' | 'skipped'> & Partial<Pick<AutoPlan, 'assumed' | 'withValues' | 'ignored'>> & {
|
|
51
60
|
maxCalls?: number;
|
|
61
|
+
maxCallsPerUnlock?: number;
|
|
52
62
|
/** Guessed unlocks whose first value changed nothing, and how many calls to them were left out. */
|
|
53
63
|
stopped?: {
|
|
54
64
|
tool: string;
|
package/dist/auto.js
CHANGED
|
@@ -85,7 +85,7 @@ export function autoScenario(menu, options = {}) {
|
|
|
85
85
|
const chosen = candidates.slice(0, maxCalls);
|
|
86
86
|
for (const c of candidates.slice(maxCalls))
|
|
87
87
|
skipped.push({ tool: c.tool.name, reason: 'over the call budget' });
|
|
88
|
-
const steps = [{ kind: 'list' }];
|
|
88
|
+
const steps = options.callsOnly ? [] : [{ kind: 'list' }];
|
|
89
89
|
for (const c of chosen)
|
|
90
90
|
steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
|
|
91
91
|
for (const u of unlocks) {
|
|
@@ -99,7 +99,7 @@ export function autoScenario(menu, options = {}) {
|
|
|
99
99
|
// The same unlock again should change nothing, as in the --init starter.
|
|
100
100
|
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [values[0]] : values[0] }, ...tentative });
|
|
101
101
|
}
|
|
102
|
-
if (chosen.length)
|
|
102
|
+
if (chosen.length && !options.callsOnly)
|
|
103
103
|
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
104
104
|
const called = [...chosen.map((c) => c.tool.name), ...unlocks.map((u) => u.tool.name)];
|
|
105
105
|
return {
|
|
@@ -109,6 +109,8 @@ export function autoScenario(menu, options = {}) {
|
|
|
109
109
|
withValues: chosen.filter((c) => c.valued).map((c) => c.tool.name),
|
|
110
110
|
skipped,
|
|
111
111
|
ignored,
|
|
112
|
+
spent: chosen.length,
|
|
113
|
+
unlockValues: unlocks.reduce((n, u) => n + Math.min(u.values.length, MAX_UNLOCKS), 0),
|
|
112
114
|
};
|
|
113
115
|
}
|
|
114
116
|
/**
|
|
@@ -281,7 +283,7 @@ export function autoSummary(auto) {
|
|
|
281
283
|
if (needs.length) {
|
|
282
284
|
const params = neededValues(auto.skipped);
|
|
283
285
|
const shown = params.slice(0, 4).map((p) => `--value ${p.param}=… (${p.tools.length})`);
|
|
284
|
-
rows.push(`${needs.length} need values the schema doesn't give → ${shown.join(', ')}${params.length > 4 ? `, and ${params.length - 4} more (--save-scenario lists them)` : ''}`);
|
|
286
|
+
rows.push(`${needs.length} ${needs.length === 1 ? 'needs' : 'need'} values the schema doesn't give → ${shown.join(', ')}${params.length > 4 ? `, and ${params.length - 4} more (--save-scenario lists them)` : ''}`);
|
|
285
287
|
}
|
|
286
288
|
const unmarked = skip('unmarked');
|
|
287
289
|
if (unmarked.length)
|
|
@@ -296,6 +298,9 @@ export function autoSummary(auto) {
|
|
|
296
298
|
// Unlocks run outside the budget, so called.length would overshoot.
|
|
297
299
|
if (over.length)
|
|
298
300
|
rows.push(`${over.length} over the call budget → --max-calls ${(auto.maxCalls ?? 20) + over.length}`);
|
|
301
|
+
const overUnlock = skip('over the per-unlock budget');
|
|
302
|
+
if (overUnlock.length)
|
|
303
|
+
rows.push(`${overUnlock.length} over the per-unlock budget → --max-calls-per-unlock ${(auto.maxCallsPerUnlock ?? 5) + overUnlock.length}`);
|
|
299
304
|
rows.forEach((r, i) => lines.push(`${i === 0 ? 'not called: ' : ' '}${r}`));
|
|
300
305
|
for (const x of auto.stopped ?? [])
|
|
301
306
|
lines.push(`stopped: ${x.tool}: its first value changed nothing and it doesn't say it unlocks tools, so the other ${x.calls} call${x.calls === 1 ? ' was' : 's were'} left out (a scenario can still make them)`);
|
package/dist/breakdown.js
CHANGED
|
@@ -160,13 +160,15 @@ function items(b, tools) {
|
|
|
160
160
|
fix: "Remove the unreferenced $defs entries from each tool's inputSchema.",
|
|
161
161
|
}]
|
|
162
162
|
: [];
|
|
163
|
+
// Two different blocks under the same name ("form") are told apart by where the first one is.
|
|
164
|
+
const label = (r) => (b.repeated.filter((x) => x.tools > 1 && x.param === r.param).length > 1 && r.where[0] ? `${r.param} at ${r.where[0]}` : r.param);
|
|
163
165
|
const repeated = b.repeated.map((r) => {
|
|
164
166
|
if (r.tools === 1) {
|
|
165
167
|
const places = list(r.where.map((w) => w.slice(r.withinTool.length + 1)), r.count);
|
|
166
168
|
return { kind: 'repeated', text: `repeated: one ${tok(r.tokens)}-token block ×${r.count} in ${r.withinTool} (${places})${r.saving > 0 ? `: a $defs entry could save ${tok(r.saving)}` : ''}`, ...(r.saving > 0 ? { fix: `Move the block into ${r.withinTool}'s $defs once and $ref it.` } : {}) };
|
|
167
169
|
}
|
|
168
170
|
const inside = r.within > 1 ? `, ×${r.within} inside ${r.withinTool}${r.saving > 0 ? `: a $defs entry there could save ${tok(r.saving)}` : ''}` : '';
|
|
169
|
-
return { kind: 'repeated', text: `repeated: ${r
|
|
171
|
+
return { kind: 'repeated', text: `repeated: ${label(r)} (${tok(r.tokens)} tokens) in ${r.tools} tools, ${tok(r.total)} in all${inside}` };
|
|
170
172
|
});
|
|
171
173
|
// A small menu gets no breakdown, but waste inside one tool is worth a line anyway.
|
|
172
174
|
if (tools < 5) {
|
package/dist/cli.js
CHANGED
|
@@ -3,6 +3,7 @@ import { writeFile } from 'node:fs/promises';
|
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { parseArgs } from 'node:util';
|
|
5
5
|
import { loadConfig } from './config.js';
|
|
6
|
+
import { PROTOCOLS } from './connect.js';
|
|
6
7
|
import { diffMenus } from './diff.js';
|
|
7
8
|
import { loadMenu } from './menu.js';
|
|
8
9
|
import { history, historyCsv } from './history.js';
|
|
@@ -17,22 +18,29 @@ import { snapshot } from './snapshot.js';
|
|
|
17
18
|
import { connectPatiently, MAIN_SEED, probeMenu, refusedForTooMany, seeded } from './probe.js';
|
|
18
19
|
import { autoScenario, loadValuesFile, parseAssumeReadOnly, parseValueFlag, scenarioYaml } from './auto.js';
|
|
19
20
|
import { authDir, listLogins, login, logout } from './auth.js';
|
|
20
|
-
import { existingMessage, initReport, planInit, writeInit } from './init.js';
|
|
21
|
+
import { existingMessage, initReport, planInit, shellWord, writeInit } from './init.js';
|
|
21
22
|
import { SEVERITY_RANK } from './types.js';
|
|
22
23
|
import { VERSION } from './version.js';
|
|
23
24
|
import { CLI_OPTIONS } from './options.js';
|
|
24
|
-
import { commandHelp, isCommand, overview, unknownCommand } from './help.js';
|
|
25
|
+
import { commandHelp, commandOptions, isCommand, overview, unknownCommand } from './help.js';
|
|
25
26
|
class UsageError extends Error {
|
|
26
27
|
}
|
|
27
28
|
export async function main(argv) {
|
|
28
29
|
const dash = argv.indexOf('--');
|
|
29
30
|
const before = dash === -1 ? argv : argv.slice(0, dash);
|
|
30
31
|
const command = dash === -1 ? undefined : argv.slice(dash + 1);
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
allowPositionals: true,
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
let parsed;
|
|
33
|
+
try {
|
|
34
|
+
parsed = parseArgs({ args: before, allowPositionals: true, options: CLI_OPTIONS });
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
// Which command was meant, so the hint can name its help and its options.
|
|
38
|
+
const meant = commandIn(before);
|
|
39
|
+
if (meant && error && typeof error === 'object')
|
|
40
|
+
error.command = meant;
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
const { values, positionals } = parsed;
|
|
36
44
|
if (values.version) {
|
|
37
45
|
process.stdout.write(`${VERSION}\n`);
|
|
38
46
|
return 0;
|
|
@@ -62,7 +70,15 @@ export async function main(argv) {
|
|
|
62
70
|
throw new UsageError(`--timeout must be a positive number of milliseconds`);
|
|
63
71
|
if (sub === 'auth')
|
|
64
72
|
return authCommand(rest, values);
|
|
73
|
+
// The invocation up to the server, for examples in messages: a session example
|
|
74
|
+
// without --auto or --scenario wouldn't run.
|
|
75
|
+
const scenarioFlag = values.scenario ? ` --scenario ${shellWord(values.scenario)}` : '';
|
|
76
|
+
const mode = sub !== 'session' ? '' : values.init ? ` --init${scenarioFlag}` : values.auto ? ' --auto' : scenarioFlag;
|
|
77
|
+
const prefix = `${sub}${mode}`;
|
|
65
78
|
const config = await loadConfig(values.config);
|
|
79
|
+
const protocol = (values.protocol ?? config.protocol ?? 'auto');
|
|
80
|
+
if (!PROTOCOLS.includes(protocol))
|
|
81
|
+
throw new UsageError(`--protocol must be auto, legacy or modern`);
|
|
66
82
|
const processes = values.processes !== undefined ? Number(values.processes) : config.processes ?? 2;
|
|
67
83
|
if (!Number.isInteger(processes) || processes < 1)
|
|
68
84
|
throw new UsageError('--processes must be a whole number, 1 or more');
|
|
@@ -120,7 +136,7 @@ export async function main(argv) {
|
|
|
120
136
|
const path = values.scenario ?? 'scenario.yml';
|
|
121
137
|
if (existsSync(path))
|
|
122
138
|
throw new UsageError(`${path} already exists. Pick another path with --scenario.`);
|
|
123
|
-
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
139
|
+
const initTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
124
140
|
const conn = await connectPatiently(seeded(initTarget, MAIN_SEED), { timeoutMs });
|
|
125
141
|
try {
|
|
126
142
|
const list = await patiently(() => listTools(conn, { timeoutMs }), { error: tooMany });
|
|
@@ -151,7 +167,7 @@ export async function main(argv) {
|
|
|
151
167
|
return 0;
|
|
152
168
|
}
|
|
153
169
|
if (sub === 'init') {
|
|
154
|
-
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
170
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
155
171
|
const plan = planInit({ cwd: process.cwd(), target, out: values.out, session: values['with-session'] });
|
|
156
172
|
if (plan.existing.length)
|
|
157
173
|
throw new UsageError(existingMessage(plan));
|
|
@@ -163,9 +179,12 @@ export async function main(argv) {
|
|
|
163
179
|
if (sub === 'session' && values.auto) {
|
|
164
180
|
if (values.scenario)
|
|
165
181
|
throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
|
|
166
|
-
const
|
|
167
|
-
if (!Number.isInteger(
|
|
182
|
+
const givenMax = values['max-calls'] ? Number(values['max-calls']) : undefined;
|
|
183
|
+
if (givenMax !== undefined && !(Number.isInteger(givenMax) && givenMax >= 1))
|
|
168
184
|
throw new UsageError('--max-calls must be a whole number, 1 or more');
|
|
185
|
+
const perUnlock = values['max-calls-per-unlock'] ? Number(values['max-calls-per-unlock']) : 5;
|
|
186
|
+
if (!Number.isInteger(perUnlock) || perUnlock < 1)
|
|
187
|
+
throw new UsageError('--max-calls-per-unlock must be a whole number, 1 or more');
|
|
169
188
|
const savePath = values['save-scenario'];
|
|
170
189
|
if (savePath && existsSync(savePath))
|
|
171
190
|
throw new UsageError(`${savePath} already exists. Pick another path for --save-scenario.`);
|
|
@@ -184,21 +203,58 @@ export async function main(argv) {
|
|
|
184
203
|
catch (error) {
|
|
185
204
|
throw new UsageError(error instanceof Error ? error.message : String(error));
|
|
186
205
|
}
|
|
187
|
-
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
206
|
+
const autoTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
188
207
|
const waited = { ms: 0 };
|
|
189
208
|
const menu = await probeMenu(seeded(autoTarget, MAIN_SEED), timeoutMs, undefined, waited).catch((error) => {
|
|
190
209
|
throw refusedForTooMany(error, waited.ms);
|
|
191
210
|
});
|
|
192
|
-
const
|
|
211
|
+
const options = { openWorld: values['open-world'], values: given, assumeReadOnly };
|
|
212
|
+
let plan = autoScenario(menu, { ...options, maxCalls: givenMax ?? 20 });
|
|
213
|
+
// Each unlock value gets its own budget for the tools it brings, so the first
|
|
214
|
+
// toolsets can't use up the calls the last ones need. --max-calls, when given,
|
|
215
|
+
// caps every call, and the starting menu leaves up to half of it for the unlocks.
|
|
216
|
+
const reserve = givenMax === undefined ? 0 : Math.min(perUnlock * plan.unlockValues, Math.floor(givenMax / 2));
|
|
217
|
+
if (reserve)
|
|
218
|
+
plan = autoScenario(menu, { ...options, maxCalls: givenMax - reserve });
|
|
219
|
+
const maxCalls = givenMax ?? 20 + perUnlock * plan.unlockValues;
|
|
193
220
|
if (savePath)
|
|
194
221
|
await writeFile(savePath, scenarioYaml(plan, menu.server.name));
|
|
195
222
|
if (values.plan) {
|
|
196
223
|
process.stdout.write(formatPlan(plan.scenario, 'auto') + '\n');
|
|
197
224
|
return 0;
|
|
198
225
|
}
|
|
199
|
-
const auto = { called: plan.called, skipped: plan.skipped, assumed: plan.assumed, withValues: plan.withValues, ignored: plan.ignored, maxCalls };
|
|
200
|
-
|
|
201
|
-
|
|
226
|
+
const auto = { called: plan.called, skipped: plan.skipped, assumed: plan.assumed, withValues: plan.withValues, ignored: plan.ignored, maxCalls, maxCallsPerUnlock: perUnlock };
|
|
227
|
+
// Tools that appear mid-session (behind an unlock) are planned when they appear,
|
|
228
|
+
// under the same rules and what's left of --max-calls. Each tool once.
|
|
229
|
+
const planned = new Set([...plan.called, ...plan.skipped.map((s) => s.tool)]);
|
|
230
|
+
let spent = plan.spent;
|
|
231
|
+
const replan = (added) => {
|
|
232
|
+
const fresh = added.filter((t) => !planned.has(t.name));
|
|
233
|
+
for (const t of fresh)
|
|
234
|
+
planned.add(t.name);
|
|
235
|
+
if (!fresh.length)
|
|
236
|
+
return [];
|
|
237
|
+
const left = maxCalls - spent;
|
|
238
|
+
const more = autoScenario({ ...menu, tools: fresh }, { ...options, maxCalls: Math.min(perUnlock, left), callsOnly: true });
|
|
239
|
+
// Left out by this unlock's own budget, not by the total: say which, for the right flag.
|
|
240
|
+
if (perUnlock < left)
|
|
241
|
+
for (const s of more.skipped)
|
|
242
|
+
if (s.reason === 'over the call budget')
|
|
243
|
+
s.reason = 'over the per-unlock budget';
|
|
244
|
+
spent += more.spent;
|
|
245
|
+
auto.called.push(...more.called);
|
|
246
|
+
auto.skipped.push(...more.skipped);
|
|
247
|
+
auto.assumed.push(...more.assumed);
|
|
248
|
+
auto.withValues.push(...more.withValues);
|
|
249
|
+
if (more.assumed.length)
|
|
250
|
+
plan.scenario.assumeReadOnly = [...(plan.scenario.assumeReadOnly ?? []), ...more.assumed];
|
|
251
|
+
return more.scenario.steps;
|
|
252
|
+
};
|
|
253
|
+
const result = await session(autoTarget, plan.scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: 'auto', unionOut: !!values['union-out'], auto, replan });
|
|
254
|
+
// A --value or --assume-read-only for a tool behind an unlock isn't unused: check
|
|
255
|
+
// them against every tool the session saw, not just the starting menu.
|
|
256
|
+
auto.ignored = autoScenario(result.union, { values: given, assumeReadOnly }).ignored;
|
|
257
|
+
result.auto = { ...result.auto, ...auto };
|
|
202
258
|
if (values['union-out'])
|
|
203
259
|
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
204
260
|
const output = formatSession(result, format);
|
|
@@ -222,7 +278,7 @@ export async function main(argv) {
|
|
|
222
278
|
process.stdout.write(formatPlan(scenario, values.scenario) + '\n');
|
|
223
279
|
return 0;
|
|
224
280
|
}
|
|
225
|
-
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
281
|
+
const sessionTarget = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
226
282
|
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario, unionOut: !!values['union-out'] });
|
|
227
283
|
if (values['union-out'])
|
|
228
284
|
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
@@ -231,7 +287,7 @@ export async function main(argv) {
|
|
|
231
287
|
process.stdout.write(output + '\n');
|
|
232
288
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
233
289
|
}
|
|
234
|
-
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
290
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
235
291
|
const routesPath = values.routes ?? config.routes;
|
|
236
292
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
237
293
|
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
@@ -243,22 +299,22 @@ export async function main(argv) {
|
|
|
243
299
|
process.stdout.write(output + '\n');
|
|
244
300
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
245
301
|
}
|
|
246
|
-
function parseTarget(positionals, command, headers, env, noAuth
|
|
302
|
+
function parseTarget(positionals, command, headers, env, noAuth, prefix, protocol) {
|
|
247
303
|
if (command) {
|
|
248
304
|
if (command.length === 0)
|
|
249
305
|
throw new UsageError('Nothing after "--": give the command that starts the server.');
|
|
250
306
|
if (positionals.length)
|
|
251
307
|
throw new UsageError(`Unexpected "${positionals[0]}" before "--".`);
|
|
252
|
-
return { kind: 'stdio', command: command[0], args: command.slice(1), env: parsePairs(env, '=', '--env') };
|
|
308
|
+
return { kind: 'stdio', command: command[0], args: command.slice(1), env: parsePairs(env, '=', '--env'), ...(protocol !== 'auto' ? { protocol } : {}) };
|
|
253
309
|
}
|
|
254
310
|
const [url, ...extra] = positionals;
|
|
255
311
|
if (!url)
|
|
256
|
-
throw new UsageError(
|
|
312
|
+
throw new UsageError(`Give a server: a URL, or the command that starts it after "--".\n → Next: toolmenu ${prefix} -- node dist/server.js (stdio) or toolmenu ${prefix} https://example.com/mcp (HTTP)`);
|
|
257
313
|
if (extra.length)
|
|
258
314
|
throw new UsageError(`Unexpected "${extra[0]}". For a stdio server, put the command after "--".`);
|
|
259
315
|
if (!/^https?:\/\//.test(url))
|
|
260
316
|
throw new UsageError(`"${url}" isn't an http(s) URL. For a stdio server, put the command after "--".`);
|
|
261
|
-
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}) };
|
|
317
|
+
return { kind: 'http', url, headers: parsePairs(headers, ':', '--header'), ...(noAuth ? { noAuth } : {}), ...(protocol !== 'auto' ? { protocol } : {}) };
|
|
262
318
|
}
|
|
263
319
|
async function authCommand(args, values) {
|
|
264
320
|
const [action, url, ...extra] = args;
|
|
@@ -324,17 +380,36 @@ main(process.argv.slice(2)).then(exitWhenFlushed, (error) => {
|
|
|
324
380
|
function failureMessage(error) {
|
|
325
381
|
const message = error instanceof Error ? error.message : String(error);
|
|
326
382
|
const code = error?.code;
|
|
383
|
+
const command = error?.command;
|
|
384
|
+
const next = command
|
|
385
|
+
? `toolmenu ${command} --help lists its options. (A server's own flags go after "--".)`
|
|
386
|
+
: 'toolmenu --help lists the commands; toolmenu <command> --help lists its options.';
|
|
327
387
|
if (code === 'ERR_PARSE_ARGS_UNKNOWN_OPTION') {
|
|
328
388
|
const option = /Unknown option '([^']+)'/.exec(message)?.[1] ?? '';
|
|
329
|
-
const known = Object.keys(CLI_OPTIONS).map((name) => `--${name}`);
|
|
389
|
+
const known = (command ? [...commandOptions(command), 'help', 'version'] : Object.keys(CLI_OPTIONS)).map((name) => `--${name}`);
|
|
330
390
|
const guess = known.map((k) => [k, editDistance(option, k)]).sort((a, b) => a[1] - b[1])[0];
|
|
331
391
|
const hint = guess && guess[1] <= 2 ? ` Did you mean ${guess[0]}?` : '';
|
|
332
|
-
return `Unknown option ${option}.${hint}\n → Next:
|
|
392
|
+
return `Unknown option ${option}.${hint}\n → Next: ${next}`;
|
|
333
393
|
}
|
|
334
394
|
if (typeof code === 'string' && code.startsWith('ERR_PARSE_ARGS_'))
|
|
335
|
-
return `${message}\n → Next:
|
|
395
|
+
return `${message}\n → Next: ${next}`;
|
|
336
396
|
return message;
|
|
337
397
|
}
|
|
398
|
+
/** The command in arguments parseArgs refused: the first command name that isn't an option's value (--out diff). */
|
|
399
|
+
function commandIn(args) {
|
|
400
|
+
for (let i = 0; i < args.length; i++) {
|
|
401
|
+
const a = args[i];
|
|
402
|
+
if (a.startsWith('--')) {
|
|
403
|
+
const option = CLI_OPTIONS[a.slice(2)];
|
|
404
|
+
if (option?.type === 'string' && !a.includes('='))
|
|
405
|
+
i++;
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
if (isCommand(a))
|
|
409
|
+
return a;
|
|
410
|
+
}
|
|
411
|
+
return undefined;
|
|
412
|
+
}
|
|
338
413
|
function editDistance(a, b) {
|
|
339
414
|
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
340
415
|
for (let i = 1; i <= a.length; i++) {
|
package/dist/close.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Client, Transport } from '@modelcontextprotocol/client';
|
|
2
|
+
/**
|
|
3
|
+
* Close a client and end its HTTP session. Closing the client only drops the
|
|
4
|
+
* connection: the server keeps the session (Mcp-Session-Id) until a DELETE ends
|
|
5
|
+
* it, and servers that keep idle sessions for hours under a cap fill up after a
|
|
6
|
+
* few dozen runs. A server that doesn't allow the DELETE (405) is fine.
|
|
7
|
+
*/
|
|
8
|
+
export declare function endSession(client: Client, transport: Transport): Promise<void>;
|
package/dist/close.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
|
+
/** A server that never answers the DELETE must not hold the run up. */
|
|
3
|
+
const END_SESSION_MS = 3000;
|
|
4
|
+
/**
|
|
5
|
+
* Close a client and end its HTTP session. Closing the client only drops the
|
|
6
|
+
* connection: the server keeps the session (Mcp-Session-Id) until a DELETE ends
|
|
7
|
+
* it, and servers that keep idle sessions for hours under a cap fill up after a
|
|
8
|
+
* few dozen runs. A server that doesn't allow the DELETE (405) is fine.
|
|
9
|
+
*/
|
|
10
|
+
export async function endSession(client, transport) {
|
|
11
|
+
if (transport instanceof StreamableHTTPClientTransport && transport.sessionId) {
|
|
12
|
+
let timer;
|
|
13
|
+
const gaveUp = new Promise((done) => (timer = setTimeout(done, END_SESSION_MS)));
|
|
14
|
+
await Promise.race([transport.terminateSession().catch(() => { }), gaveUp]);
|
|
15
|
+
clearTimeout(timer);
|
|
16
|
+
}
|
|
17
|
+
await client.close().catch(() => { });
|
|
18
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -4,6 +4,8 @@ export interface Config {
|
|
|
4
4
|
ignore?: string[];
|
|
5
5
|
routes?: string;
|
|
6
6
|
baseline?: string;
|
|
7
|
+
/** Which handshake to make: auto (default), legacy (2025 only) or modern (2026-07-28 only). --protocol wins. */
|
|
8
|
+
protocol?: 'auto' | 'legacy' | 'modern';
|
|
7
9
|
tokenBudget?: number;
|
|
8
10
|
/** diff checks the semver bump against serverInfo.version (off: it's often not the release version). */
|
|
9
11
|
serverVersionIsRelease?: boolean;
|
package/dist/config.js
CHANGED
|
@@ -19,6 +19,9 @@ export async function loadConfig(path) {
|
|
|
19
19
|
throw new Error(`${file}: rules["${id}"] must be error, warn, info or off`);
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
|
+
if (config.protocol !== undefined && !['auto', 'legacy', 'modern'].includes(config.protocol)) {
|
|
23
|
+
throw new Error(`${file}: protocol must be auto, legacy or modern`);
|
|
24
|
+
}
|
|
22
25
|
const limit = config.descriptionLimit;
|
|
23
26
|
if (limit !== undefined && !(typeof limit === 'number' && limit > 0) && !(typeof limit === 'string' && limit in CLIENT_LIMITS)) {
|
|
24
27
|
throw new Error(`${file}: descriptionLimit must be a positive number or one of ${Object.keys(CLIENT_LIMITS).join(', ')}`);
|
package/dist/connect.d.ts
CHANGED
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
import { Client } from '@modelcontextprotocol/client';
|
|
2
2
|
import type { Transport } from '@modelcontextprotocol/client';
|
|
3
3
|
import type { Era } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Which handshake to make. auto: try the 2026-07-28 `server/discover` first and fall back
|
|
6
|
+
* to the 2025 one. legacy: the 2025 handshake only. modern: 2026-07-28 only, an error if
|
|
7
|
+
* the server can't speak it.
|
|
8
|
+
*/
|
|
9
|
+
export type Protocol = 'auto' | 'legacy' | 'modern';
|
|
10
|
+
export declare const PROTOCOLS: readonly Protocol[];
|
|
11
|
+
export declare const MODERN_VERSION = "2026-07-28";
|
|
4
12
|
export type Target = {
|
|
5
13
|
kind: 'stdio';
|
|
6
14
|
command: string;
|
|
7
15
|
args: string[];
|
|
8
16
|
env?: Record<string, string>;
|
|
9
17
|
cwd?: string;
|
|
18
|
+
protocol?: Protocol;
|
|
10
19
|
} | {
|
|
11
20
|
kind: 'http';
|
|
12
21
|
url: string;
|
|
13
22
|
headers?: Record<string, string>;
|
|
14
23
|
noAuth?: boolean;
|
|
24
|
+
protocol?: Protocol;
|
|
15
25
|
};
|
|
16
26
|
export interface WireResponse {
|
|
17
27
|
method: string;
|
package/dist/connect.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client';
|
|
2
2
|
import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/client/stdio';
|
|
3
3
|
import { hasLogin, StoredOAuthProvider } from './auth.js';
|
|
4
|
-
import { explain, Trace } from './explain.js';
|
|
4
|
+
import { explain, isCredentialHeader, Trace } from './explain.js';
|
|
5
|
+
import { endSession } from './close.js';
|
|
5
6
|
import { eraOf } from './menu.js';
|
|
6
7
|
import { VERSION } from './version.js';
|
|
8
|
+
export const PROTOCOLS = ['auto', 'legacy', 'modern'];
|
|
9
|
+
export const MODERN_VERSION = '2026-07-28';
|
|
7
10
|
/** Every response the server sent, matched to the request method, as it came off the wire. */
|
|
8
11
|
export class WireLog {
|
|
9
12
|
responses = [];
|
|
@@ -47,7 +50,7 @@ export class WireLog {
|
|
|
47
50
|
}
|
|
48
51
|
}
|
|
49
52
|
export async function connect(target, options = {}) {
|
|
50
|
-
const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: 'auto' } });
|
|
53
|
+
const client = new Client({ name: 'toolmenu', version: VERSION }, { versionNegotiation: { mode: target.protocol === 'legacy' ? 'legacy' : target.protocol === 'modern' ? { pin: MODERN_VERSION } : 'auto' } });
|
|
51
54
|
let transport;
|
|
52
55
|
let stderrText = '';
|
|
53
56
|
let oauth = false;
|
|
@@ -81,9 +84,12 @@ export async function connect(target, options = {}) {
|
|
|
81
84
|
...(oauth ? { authProvider: new StoredOAuthProvider(target.url) } : {}),
|
|
82
85
|
});
|
|
83
86
|
}
|
|
84
|
-
const context = { target, timeoutMs: options.timeoutMs, oauth };
|
|
87
|
+
const context = { target, timeoutMs: options.timeoutMs, oauth, ...(target.protocol && target.protocol !== 'auto' ? { protocol: target.protocol } : {}) };
|
|
85
88
|
try {
|
|
86
|
-
|
|
89
|
+
// A URL that answered as a 2025 server earlier in this run isn't probed again:
|
|
90
|
+
// each server/discover gets a 400 there, one line in its log per connection.
|
|
91
|
+
const prior = target.kind === 'http' && (!target.protocol || target.protocol === 'auto') && LEGACY_URLS.has(target.url) ? { prior: { kind: 'legacy' } } : {};
|
|
92
|
+
await client.connect(transport, { timeout: options.timeoutMs, ...prior });
|
|
87
93
|
}
|
|
88
94
|
catch (error) {
|
|
89
95
|
// What failed is fixed now: the exit that follows toolmenu's own SIGTERM isn't the server's doing.
|
|
@@ -102,12 +108,14 @@ export async function connect(target, options = {}) {
|
|
|
102
108
|
await exited(trace, 1000);
|
|
103
109
|
await stderrEnded(stdioStderr, trace.exit ? STDERR_AFTER_EXIT_MS : 1000);
|
|
104
110
|
}
|
|
105
|
-
await client
|
|
111
|
+
await endSession(client, transport);
|
|
106
112
|
throw explain(error, trace, context);
|
|
107
113
|
}
|
|
108
114
|
const wire = new WireLog();
|
|
109
115
|
wire.tap(transport);
|
|
110
116
|
const protocolVersion = client.getNegotiatedProtocolVersion();
|
|
117
|
+
if (target.kind === 'http' && eraOf(protocolVersion) === 'legacy')
|
|
118
|
+
LEGACY_URLS.add(target.url);
|
|
111
119
|
const server = client.getServerVersion() ?? {};
|
|
112
120
|
return {
|
|
113
121
|
client,
|
|
@@ -117,9 +125,9 @@ export async function connect(target, options = {}) {
|
|
|
117
125
|
server: { name: server.name, version: server.version },
|
|
118
126
|
capabilities: (client.getServerCapabilities() ?? {}),
|
|
119
127
|
usedAuth: target.kind === 'http' &&
|
|
120
|
-
(oauth || Object.keys(target.headers ?? {}).some(
|
|
128
|
+
(oauth || Object.keys(target.headers ?? {}).some(isCredentialHeader)),
|
|
121
129
|
stderr: () => stderrText,
|
|
122
|
-
close: () => client
|
|
130
|
+
close: () => endSession(client, transport),
|
|
123
131
|
explain: (error, stage) => explain(error, trace, { ...context, stage }),
|
|
124
132
|
stderrSettled: async () => {
|
|
125
133
|
if (target.kind !== 'stdio')
|
|
@@ -132,6 +140,11 @@ export async function connect(target, options = {}) {
|
|
|
132
140
|
},
|
|
133
141
|
};
|
|
134
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* HTTP servers that answered with the 2025 handshake in this process. Kept for the
|
|
145
|
+
* run only: a server upgraded between runs is probed afresh next time.
|
|
146
|
+
*/
|
|
147
|
+
const LEGACY_URLS = new Set();
|
|
135
148
|
/** How long to wait for an exited server's stderr to end: its pipe is closing, so this is only a ceiling for a stuck one. */
|
|
136
149
|
const STDERR_AFTER_EXIT_MS = 10_000;
|
|
137
150
|
/** Resolves when the traced process has exited, or after `maxMs`. */
|