toolmenu 0.21.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -37
- package/dist/apps.d.ts +56 -0
- package/dist/apps.js +99 -0
- package/dist/auto.js +2 -2
- package/dist/card.d.ts +18 -0
- package/dist/card.js +39 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +70 -11
- package/dist/colour.d.ts +8 -0
- package/dist/colour.js +70 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -0
- package/dist/connect.d.ts +15 -0
- package/dist/connect.js +54 -7
- package/dist/diff.js +113 -6
- package/dist/explain.d.ts +16 -0
- package/dist/explain.js +14 -0
- package/dist/failures.d.ts +18 -5
- package/dist/failures.js +27 -12
- package/dist/help.d.ts +1 -1
- package/dist/help.js +18 -3
- package/dist/mcp-env.d.ts +5 -0
- package/dist/mcp-env.js +5 -0
- package/dist/mcp.d.ts +65 -0
- package/dist/mcp.js +293 -0
- package/dist/mrtr.d.ts +9 -1
- package/dist/mrtr.js +17 -0
- package/dist/options.d.ts +3 -0
- package/dist/options.js +1 -0
- package/dist/probe.d.ts +11 -1
- package/dist/probe.js +18 -3
- package/dist/report.js +12 -5
- package/dist/rule-docs.json +94 -4
- package/dist/rules/apps.d.ts +11 -0
- package/dist/rules/apps.js +156 -0
- package/dist/rules/capabilities.d.ts +21 -0
- package/dist/rules/capabilities.js +103 -0
- package/dist/rules/card.d.ts +7 -0
- package/dist/rules/card.js +65 -0
- package/dist/rules/clients.js +1 -0
- package/dist/rules/description.js +3 -0
- package/dist/rules/headers.d.ts +0 -2
- package/dist/rules/headers.js +7 -12
- package/dist/rules/ids.js +35 -1
- package/dist/rules/index.js +18 -2
- package/dist/rules/naming.js +3 -0
- package/dist/rules/network.d.ts +21 -0
- package/dist/rules/network.js +63 -0
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/search.js +1 -0
- package/dist/rules/spec.d.ts +9 -1
- package/dist/rules/spec.js +117 -3
- package/dist/rules/write.d.ts +0 -6
- package/dist/rules/write.js +11 -5
- package/dist/session.js +60 -11
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +89 -5
- package/dist/types.d.ts +5 -0
- package/dist/words.d.ts +8 -0
- package/dist/words.js +30 -2
- package/package.json +8 -7
- package/schemas/server-card.json +260 -0
package/dist/session.js
CHANGED
|
@@ -4,9 +4,9 @@ import { cacheBreak, canonical, compareMenus } from './compare.js';
|
|
|
4
4
|
import { listTools } from './connect.js';
|
|
5
5
|
import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
|
|
6
6
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
7
|
-
import { connectPatiently, isContainerWrapper, MAIN_SEED,
|
|
7
|
+
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeSameEra, probeVariance, seeded } from './probe.js';
|
|
8
8
|
import { varianceFinding } from './rules/determinism.js';
|
|
9
|
-
import { classifyFailure,
|
|
9
|
+
import { classifyFailure, errorWords, structuredError, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
10
10
|
import { SEVERITY_RANK } from './types.js';
|
|
11
11
|
import { leadingJson } from './catalog.js';
|
|
12
12
|
import { allDifferences, describeToolDifference } from './difference.js';
|
|
@@ -82,6 +82,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
82
82
|
// --create-handles: the handles creating tools returned, by kind, and what went wrong on the way.
|
|
83
83
|
const created = new Map();
|
|
84
84
|
const roundtrips = [];
|
|
85
|
+
// HTTP errors passed on as plain answers, not isError (session/error-not-flagged).
|
|
86
|
+
const unflagged = [];
|
|
85
87
|
// --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
|
|
86
88
|
const handleErrors = [];
|
|
87
89
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
@@ -89,7 +91,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
89
91
|
// How every elicitation is answered: the same each run (--elicitation). Session
|
|
90
92
|
// connections declare elicitation; snapshot's don't, so its menu is the one 0.14 saw.
|
|
91
93
|
const answer = target.elicitation ?? 'decline';
|
|
92
|
-
|
|
94
|
+
// And MCP Apps, as snapshot's baseline is listed: the same tools, the same variants.
|
|
95
|
+
target = { ...target, elicitation: answer, apps: true };
|
|
93
96
|
const mainTarget = seeded(target, MAIN_SEED);
|
|
94
97
|
const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
|
|
95
98
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
@@ -150,7 +153,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
150
153
|
};
|
|
151
154
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
152
155
|
let connectionCheck;
|
|
153
|
-
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
|
|
156
|
+
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits, era)) {
|
|
154
157
|
if (probe.error) {
|
|
155
158
|
raw.push({
|
|
156
159
|
rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
|
|
@@ -372,14 +375,17 @@ export async function session(target, scenario, options = {}) {
|
|
|
372
375
|
}
|
|
373
376
|
}
|
|
374
377
|
else if (!call.input.rounds && !step.creates) {
|
|
375
|
-
// An
|
|
378
|
+
// An HTTP error passed on as an answer (not isError): a client that reads
|
|
379
|
+
// isError takes it for a success. An auth one also means the call reached the
|
|
376
380
|
// server but not the API behind it, so it tested no more than a refused one.
|
|
377
381
|
// A tool that takes a URL fetches it: what its answer says (a status, "Forbidden") is the page's.
|
|
378
382
|
const fetches = Object.keys(tool?.inputSchema?.properties ?? {}).some((p) => /^(?:url|uri|href|endpoint|link)s?$/i.test(p) || /_url$|Url$/.test(p));
|
|
379
|
-
const
|
|
380
|
-
if (
|
|
381
|
-
|
|
382
|
-
record.note = `the tool answered, not as an error, with the API's ${clip(
|
|
383
|
+
const passedOn = fetches ? undefined : structuredError(result);
|
|
384
|
+
if (passedOn) {
|
|
385
|
+
unflagged.push({ step: index, tool: step.tool, said: passedOn.said });
|
|
386
|
+
record.note = `the tool answered, not as an error, with the API's ${clip(passedOn.said, 120)}`;
|
|
387
|
+
if (passedOn.auth)
|
|
388
|
+
record.failure = 'auth';
|
|
383
389
|
}
|
|
384
390
|
}
|
|
385
391
|
}
|
|
@@ -476,7 +482,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
476
482
|
}
|
|
477
483
|
const last = scenario.steps.length;
|
|
478
484
|
raw.push({
|
|
479
|
-
rule: 'session/
|
|
485
|
+
rule: 'session/server-unresponsive',
|
|
480
486
|
severity: 'error',
|
|
481
487
|
step: index,
|
|
482
488
|
...(step.kind === 'call' ? { tool: step.tool } : {}),
|
|
@@ -541,7 +547,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
541
547
|
const probeWaited = { ms: 0 };
|
|
542
548
|
try {
|
|
543
549
|
probedAfter = index;
|
|
544
|
-
const probe = await
|
|
550
|
+
const probe = await probeSameEra(mainTarget, era, timeoutMs, waits, probeWaited);
|
|
545
551
|
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
546
552
|
raw.push(...scopeFindings(record.scope, modern, index));
|
|
547
553
|
}
|
|
@@ -569,6 +575,9 @@ export async function session(target, scenario, options = {}) {
|
|
|
569
575
|
raw.push(...inputFindings(inputs, answer));
|
|
570
576
|
raw.push(...handleErrorFindings(handleErrors));
|
|
571
577
|
raw.push(...roundtripFindings(roundtrips));
|
|
578
|
+
// `ignore` is applied by tool name; this finding sums several tools up, so it's applied here.
|
|
579
|
+
const ignored = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
580
|
+
raw.push(...unflaggedFindings(unflagged.filter((u) => !ignored.some((re) => re.test(u.tool)))));
|
|
572
581
|
if (explained.length)
|
|
573
582
|
raw.push(knownVariance(explained));
|
|
574
583
|
if (onWord.length)
|
|
@@ -1517,12 +1526,37 @@ function inputNote(input, answer) {
|
|
|
1517
1526
|
return `asked for ${kindList(input.undeclared ?? [])}, which the request didn't declare`;
|
|
1518
1527
|
case 'decline-failed':
|
|
1519
1528
|
return `${asked}; the retry with the ${answer} failed: ${clip(input.error?.message ?? '', 160)}`;
|
|
1529
|
+
case 'task':
|
|
1530
|
+
return `answered with a task to poll (resultType "task"${input.task?.taskId ? `, taskId ${clip(input.task.taskId, 40)}` : ''}${input.task?.status ? `, ${input.task.status}` : ''}), which the request didn't declare it could take`;
|
|
1520
1531
|
case 'bad-state':
|
|
1521
1532
|
return `${input.kinds.length ? `${asked}, with` : 'answered input_required with'} a requestState that isn't a string, so it can't be sent back as it came`;
|
|
1522
1533
|
default:
|
|
1523
1534
|
return asked;
|
|
1524
1535
|
}
|
|
1525
1536
|
}
|
|
1537
|
+
/**
|
|
1538
|
+
* Tools that passed an API's HTTP error on as a plain answer, not isError: one finding
|
|
1539
|
+
* for the run. The spec reports tool execution errors, API failures among them, in
|
|
1540
|
+
* the result with isError: true; a client that reads it (retry logic, its UI, a check
|
|
1541
|
+
* like this one) takes the call for a success.
|
|
1542
|
+
*/
|
|
1543
|
+
function unflaggedFindings(list) {
|
|
1544
|
+
if (!list.length)
|
|
1545
|
+
return [];
|
|
1546
|
+
const tools = [...new Set(list.map((u) => u.tool))];
|
|
1547
|
+
return [
|
|
1548
|
+
{
|
|
1549
|
+
rule: 'session/error-not-flagged',
|
|
1550
|
+
severity: 'warn',
|
|
1551
|
+
// Read from the answer's shape: a tool may return an error object as its data.
|
|
1552
|
+
confidence: 'unsure',
|
|
1553
|
+
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
1554
|
+
message: `${tools.length === 1 ? tools[0] : `${tools.length} tools`} answered with an HTTP error from the API behind ${tools.length === 1 ? 'it' : 'them'}, passed on as a plain result, not marked isError. The spec reports tool errors, API failures among them, with isError: true; a client that reads it (retries, its UI, a check like this one) takes the call for a success.`,
|
|
1555
|
+
detail: list.slice(0, 8).map((u) => `step ${u.step}, ${u.tool}: ${clip(u.said, 120)}`).concat(list.length > 8 ? [`… and ${list.length - 8} more`] : []),
|
|
1556
|
+
fix: 'Return the API\'s error with isError: true (in the official SDKs: throw from the handler, or return { isError: true, content }), keeping its message for the model.',
|
|
1557
|
+
},
|
|
1558
|
+
];
|
|
1559
|
+
}
|
|
1526
1560
|
/**
|
|
1527
1561
|
* --create-handles: a handle a creating tool returned that its own tools then refuse, or
|
|
1528
1562
|
* a creating tool whose result carries no handle to find.
|
|
@@ -1635,6 +1669,21 @@ function inputFindings(inputs, answer) {
|
|
|
1635
1669
|
});
|
|
1636
1670
|
}
|
|
1637
1671
|
}
|
|
1672
|
+
// ext-tasks: "A server MUST NOT return CreateTaskResult to a client that did not
|
|
1673
|
+
// include the extension capability on its request". session declares no Tasks support.
|
|
1674
|
+
const tasks = inputs.filter((i) => i.input.outcome === 'task');
|
|
1675
|
+
if (tasks.length) {
|
|
1676
|
+
const one = tasks.length === 1;
|
|
1677
|
+
out.push({
|
|
1678
|
+
rule: 'session/undeclared-task',
|
|
1679
|
+
severity: 'error',
|
|
1680
|
+
step: tasks[0].step,
|
|
1681
|
+
...(one ? { tool: tasks[0].tool } : {}),
|
|
1682
|
+
message: `${one ? `${tasks[0].tool} answered` : `${tasks.length} calls answered`} with a task to poll (resultType "task", the Tasks extension), though the request didn't declare io.modelcontextprotocol/tasks. Servers must not send a task to such a client: it has no tasks/get to poll with, so the agent gets no result at all.`,
|
|
1683
|
+
detail: tasks.map((i) => `${i.tool} (step ${i.step}): ${inputNote(i.input, answer)}`),
|
|
1684
|
+
fix: `Check the request's io.modelcontextprotocol/clientCapabilities for the io.modelcontextprotocol/tasks extension before creating a task. Without it, answer the call in one go, or return -32021 (Missing Required Client Capability) naming the extension.`,
|
|
1685
|
+
});
|
|
1686
|
+
}
|
|
1638
1687
|
const unmarked = inputs.filter((i) => i.input.missingResultType);
|
|
1639
1688
|
if (unmarked.length) {
|
|
1640
1689
|
out.push({
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
28
28
|
lists?: string[];
|
|
29
29
|
/** --client: check the menu for this client (src/clients.ts): its description cut, its search, its name format. */
|
|
30
30
|
client?: string;
|
|
31
|
+
/** --offline-check (stdio): list the menu once more with the network cut off (menu/network-dependent). */
|
|
32
|
+
offlineCheck?: boolean;
|
|
31
33
|
}
|
|
32
34
|
export interface SnapshotResult {
|
|
33
35
|
menu: Menu;
|
|
@@ -51,6 +53,9 @@ export declare function withInvalidCredential(target: Extract<Target, {
|
|
|
51
53
|
}>): Extract<Target, {
|
|
52
54
|
kind: 'http';
|
|
53
55
|
}>;
|
|
56
|
+
export declare function offlineEnvironment(): Record<string, string>;
|
|
57
|
+
/** capabilities.extensions as the server sent it: identifier → settings, {} for none. */
|
|
58
|
+
export declare function declaredExtensions(capabilities: Record<string, unknown>): Record<string, unknown>;
|
|
54
59
|
/**
|
|
55
60
|
* How this run reached the server, as the end of a command line, for next steps:
|
|
56
61
|
* the options that got it past the door (names only; values are the user's) and
|
package/dist/snapshot.js
CHANGED
|
@@ -2,8 +2,11 @@ import { listTools } from './connect.js';
|
|
|
2
2
|
import { patiently, tooMany } from './failures.js';
|
|
3
3
|
import { ConnectError } from './explain.js';
|
|
4
4
|
import { buildMenu } from './menu.js';
|
|
5
|
-
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, PROBE_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
5
|
+
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, PROBE_SEED, probeMenu, probeSameEra, probeVariance, seeded } from './probe.js';
|
|
6
6
|
import { kindsToRead, readList } from './lists.js';
|
|
7
|
+
import { agentMenu, readAppResources, resourceUriOf } from './apps.js';
|
|
8
|
+
import { readServerCard } from './card.js';
|
|
9
|
+
import { listingsDiffer } from './rules/capabilities.js';
|
|
7
10
|
import { isCredentialHeader } from './explain.js';
|
|
8
11
|
import { httpStatus } from './failures.js';
|
|
9
12
|
import { clientCost } from './clients.js';
|
|
@@ -14,7 +17,11 @@ import { shellWord } from './init.js';
|
|
|
14
17
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
15
18
|
export async function snapshot(given, options = {}) {
|
|
16
19
|
const [mainIdentity, ...otherIdentities] = options.identities ?? [];
|
|
17
|
-
|
|
20
|
+
// Listed as a client that can be asked things (elicitation, answered "decline"; no tool
|
|
21
|
+
// is called, so none is asked): a server may list some tools only to such clients, and
|
|
22
|
+
// the baseline should hold every tool some client sees. One more listing without it
|
|
23
|
+
// says what a client without elicitation gets (rules/capabilities.ts).
|
|
24
|
+
const target = withElicitation(asIdentity(given, mainIdentity));
|
|
18
25
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
19
26
|
// out and they're sent again (CI behind a shared rate limit).
|
|
20
27
|
const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
@@ -23,6 +30,7 @@ export async function snapshot(given, options = {}) {
|
|
|
23
30
|
let first;
|
|
24
31
|
const reads = [];
|
|
25
32
|
let undeclared = [];
|
|
33
|
+
let appResources;
|
|
26
34
|
try {
|
|
27
35
|
first = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
28
36
|
const second = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
@@ -31,6 +39,7 @@ export async function snapshot(given, options = {}) {
|
|
|
31
39
|
version: connection.server.version,
|
|
32
40
|
protocolVersion: connection.protocolVersion,
|
|
33
41
|
era: connection.era,
|
|
42
|
+
extensions: declaredExtensions(connection.capabilities),
|
|
34
43
|
};
|
|
35
44
|
menu = buildMenu(first.tools, server, first.listMeta);
|
|
36
45
|
secondList = buildMenu(second.tools, server, second.listMeta).tools;
|
|
@@ -44,6 +53,11 @@ export async function snapshot(given, options = {}) {
|
|
|
44
53
|
if (!first.error)
|
|
45
54
|
menu[kind] = first.items;
|
|
46
55
|
}
|
|
56
|
+
// MCP Apps: each ui:// resource a tool names, read as a host would before rendering
|
|
57
|
+
// it (apps/resource). A read, not a tool call.
|
|
58
|
+
if (options.rules?.['apps/resource'] !== 'off' && menu.tools.some((t) => resourceUriOf(t)?.startsWith('ui://'))) {
|
|
59
|
+
appResources = await readAppResources(connection, menu.tools, options.timeoutMs);
|
|
60
|
+
}
|
|
47
61
|
if (options.catalog) {
|
|
48
62
|
menu.catalog = await readCatalog(connection, menu.tools, { ...(typeof options.catalog === 'object' ? options.catalog : {}), timeoutMs: options.timeoutMs });
|
|
49
63
|
}
|
|
@@ -60,7 +74,9 @@ export async function snapshot(given, options = {}) {
|
|
|
60
74
|
}
|
|
61
75
|
// After the main process has exited: a server that holds a file or a port
|
|
62
76
|
// shouldn't have to share it with its own probe.
|
|
63
|
-
|
|
77
|
+
// Eras later processes negotiated, before any retry on the main one's (spec/discover).
|
|
78
|
+
const eras = new Set();
|
|
79
|
+
const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000, undefined, connection.era, eras);
|
|
64
80
|
// The other lists from a second process (stdio) or connection (HTTP), as tools get.
|
|
65
81
|
if ((options.processes ?? 2) > 1 && reads.some((r) => !r.first.error)) {
|
|
66
82
|
try {
|
|
@@ -81,13 +97,39 @@ export async function snapshot(given, options = {}) {
|
|
|
81
97
|
// A credential that can't be valid, in place of the one this run used (HTTP): the
|
|
82
98
|
// server should refuse it, not serve a menu (rules/auth.ts).
|
|
83
99
|
const invalid = target.kind === 'http' && connection.usedAuth && (options.processes ?? 2) > 1 ? await invalidCredentialProbe(target, options.timeoutMs ?? 30_000) : undefined;
|
|
100
|
+
// The Server Card at its reserved location, compared with the live server (rules/card.ts).
|
|
101
|
+
const card = target.kind === 'http' && options.rules?.['spec/server-card'] !== 'off' ? await readServerCard(target.url, Math.min(options.timeoutMs ?? 30_000, 10_000)) : undefined;
|
|
102
|
+
// The menu a client without elicitation gets (rules/capabilities.ts).
|
|
103
|
+
// Another process or connection: not with --processes 1 (one session per client), nor when the rule is off.
|
|
104
|
+
let plain;
|
|
105
|
+
if ((options.processes ?? 2) > 1 && options.rules?.['menu/capability-dependent'] !== 'off') {
|
|
106
|
+
try {
|
|
107
|
+
plain = { tools: (await probeSameEra(seeded({ ...target, elicitation: undefined, apps: undefined }, MAIN_SEED), connection.era, options.timeoutMs ?? 30_000, undefined, undefined, eras)).tools };
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
plain = { error: firstLine(error) };
|
|
111
|
+
}
|
|
112
|
+
// The menus differ: once more with elicitation alone, to tell which capability a tool
|
|
113
|
+
// depends on (MCP Apps or elicitation). Only then: it's one more process or connection.
|
|
114
|
+
if (plain.tools && listingsDiffer(menu.tools, plain.tools)) {
|
|
115
|
+
try {
|
|
116
|
+
plain.elicitationOnly = (await probeSameEra(seeded({ ...target, apps: undefined }, MAIN_SEED), connection.era, options.timeoutMs ?? 30_000, undefined, undefined, eras)).tools;
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// Not attributed: the finding names both capabilities.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
// --offline-check: one more process with the network cut off (rules/network.ts).
|
|
124
|
+
// Not run when the rule is off: it's a whole server start.
|
|
125
|
+
const offline = options.offlineCheck && target.kind === 'stdio' && options.rules?.['menu/network-dependent'] !== 'off' ? (isContainerWrapper(target) ? { wrapper: true } : await offlineProbe(target, options.timeoutMs ?? 30_000)) : undefined;
|
|
84
126
|
// Each other identity's menu, from its own process (stdio) or connection (HTTP).
|
|
85
127
|
const identities = mainIdentity
|
|
86
128
|
? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
|
|
87
129
|
: undefined;
|
|
88
130
|
for (const identity of otherIdentities) {
|
|
89
131
|
try {
|
|
90
|
-
const m = await probeMenu(seeded(asIdentity(given, identity), MAIN_SEED), options.timeoutMs ?? 30_000);
|
|
132
|
+
const m = await probeMenu(seeded(withElicitation(asIdentity(given, identity)), MAIN_SEED), options.timeoutMs ?? 30_000);
|
|
91
133
|
identities.push({ name: identity.name, tools: m.tools, tokens: m.totalTokens, cacheScope: m.listMeta?.cacheScope });
|
|
92
134
|
}
|
|
93
135
|
catch (error) {
|
|
@@ -116,6 +158,12 @@ export async function snapshot(given, options = {}) {
|
|
|
116
158
|
server: serverArgs(target),
|
|
117
159
|
identities,
|
|
118
160
|
...(invalid ? { invalidCredential: invalid } : {}),
|
|
161
|
+
...(offline ? { offline } : {}),
|
|
162
|
+
...(plain ? { plain } : {}),
|
|
163
|
+
...(appResources ? { appResources } : {}),
|
|
164
|
+
...(card ? { serverCard: card } : {}),
|
|
165
|
+
...(target.kind === 'http' ? { serverUrl: new URL(target.url).origin + new URL(target.url).pathname } : {}),
|
|
166
|
+
...(connection.era === 'legacy' && eras.has('modern') ? { laterModern: true } : {}),
|
|
119
167
|
...(reads.length || undeclared.length ? { lists: { read: reads, undeclared } } : {}),
|
|
120
168
|
}, options);
|
|
121
169
|
// What --catalog called and what it got, under the same settings as the rules
|
|
@@ -130,7 +178,8 @@ export async function snapshot(given, options = {}) {
|
|
|
130
178
|
}
|
|
131
179
|
findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
132
180
|
}
|
|
133
|
-
|
|
181
|
+
// What the agent pays: app-only tools aren't in its list.
|
|
182
|
+
const cost = options.client ? clientCost(options.client, agentMenu(menu).shown) : undefined;
|
|
134
183
|
return { menu, findings, ...(identities ? { identities } : {}), ...(cost ? { cost } : {}), ...(undeclared.length ? { notOffered: undeclared } : {}) };
|
|
135
184
|
}
|
|
136
185
|
/**
|
|
@@ -158,6 +207,29 @@ export function withInvalidCredential(target) {
|
|
|
158
207
|
headers.Authorization = 'Bearer toolmenu-invalid-credential';
|
|
159
208
|
return { ...target, headers, noAuth: true };
|
|
160
209
|
}
|
|
210
|
+
/**
|
|
211
|
+
* The proxy variables pointed at a port nothing listens on (the discard port): every
|
|
212
|
+
* client that honours them (Python requests and httpx, Go, curl, Node's fetch with
|
|
213
|
+
* NODE_USE_ENV_PROXY on 22.21+) fails to connect at once. Loopback stays exempt: a
|
|
214
|
+
* server's own local helper (TabRunner's bridge on 127.0.0.1) isn't the network.
|
|
215
|
+
* These win over --env: a proxy given there would undo the cut.
|
|
216
|
+
*/
|
|
217
|
+
const LOOPBACK = 'localhost,127.0.0.1,::1';
|
|
218
|
+
export function offlineEnvironment() {
|
|
219
|
+
const dead = 'http://127.0.0.1:9';
|
|
220
|
+
return { HTTPS_PROXY: dead, https_proxy: dead, HTTP_PROXY: dead, http_proxy: dead, ALL_PROXY: dead, all_proxy: dead, NO_PROXY: LOOPBACK, no_proxy: LOOPBACK, NODE_USE_ENV_PROXY: '1' };
|
|
221
|
+
}
|
|
222
|
+
async function offlineProbe(target, timeoutMs) {
|
|
223
|
+
const launcher = /(?:^|[\\/])(?:npx|uvx|bunx|pnpx|pipx)(?:\.cmd|\.exe)?$/.test(target.command) || (/(?:^|[\\/])(?:pnpm|yarn|npm|bun)(?:\.cmd|\.exe)?$/.test(target.command) && /^(?:dlx|exec|x)$/.test(target.args[0] ?? ''));
|
|
224
|
+
try {
|
|
225
|
+
const m = await probeMenu(seeded({ ...target, env: { ...target.env, ...offlineEnvironment() } }, MAIN_SEED), timeoutMs);
|
|
226
|
+
return { tools: m.tools };
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
const e = error instanceof ConnectError ? error.headline : firstLine(error);
|
|
230
|
+
return { error: e.replace(/\.$/, ''), ...(launcher ? { launcher } : {}) };
|
|
231
|
+
}
|
|
232
|
+
}
|
|
161
233
|
async function invalidCredentialProbe(target, timeoutMs) {
|
|
162
234
|
try {
|
|
163
235
|
const m = await probeMenu(withInvalidCredential(target), timeoutMs);
|
|
@@ -168,6 +240,18 @@ async function invalidCredentialProbe(target, timeoutMs) {
|
|
|
168
240
|
return status !== undefined ? { status } : { error: firstLine(error) };
|
|
169
241
|
}
|
|
170
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* The target, listed as a client that supports elicitation (answered "decline" if ever
|
|
245
|
+
* asked) and renders MCP Apps: the client that sees every tool some client sees.
|
|
246
|
+
*/
|
|
247
|
+
function withElicitation(t) {
|
|
248
|
+
return { ...t, elicitation: t.elicitation ?? 'decline', apps: true };
|
|
249
|
+
}
|
|
250
|
+
/** capabilities.extensions as the server sent it: identifier → settings, {} for none. */
|
|
251
|
+
export function declaredExtensions(capabilities) {
|
|
252
|
+
const e = capabilities.extensions;
|
|
253
|
+
return typeof e === 'object' && e !== null && !Array.isArray(e) ? { ...e } : {};
|
|
254
|
+
}
|
|
171
255
|
const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
|
|
172
256
|
/**
|
|
173
257
|
* How this run reached the server, as the end of a command line, for next steps:
|
package/dist/types.d.ts
CHANGED
|
@@ -53,6 +53,11 @@ export interface Menu {
|
|
|
53
53
|
version?: string;
|
|
54
54
|
protocolVersion?: string;
|
|
55
55
|
era?: Era;
|
|
56
|
+
/**
|
|
57
|
+
* The extensions the server declares (capabilities.extensions: identifier → settings),
|
|
58
|
+
* {} for none. Absent in menu files from before 0.23, so diff can't tell those apart.
|
|
59
|
+
*/
|
|
60
|
+
extensions?: Record<string, unknown>;
|
|
56
61
|
};
|
|
57
62
|
capturedAt: string;
|
|
58
63
|
/** In server order. Order is the point. */
|
package/dist/words.d.ts
CHANGED
|
@@ -13,6 +13,14 @@ export declare const WRITE_VERBS: Set<string>;
|
|
|
13
13
|
export declare function article(word: string): 'a' | 'an';
|
|
14
14
|
/** Words most of a server's tool names share (`firecrawl_`, `API-`, `browser_`): no information. */
|
|
15
15
|
export declare function commonWords(names: string[]): Set<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Words a name's leading verb is read after: those most of the menu's names share
|
|
18
|
+
* (commonWords), and a namespace prefix: a word, not a verb, that opens 3 or more
|
|
19
|
+
* names and is followed by a verb in 3 or more of them. kubectl_create leads with
|
|
20
|
+
* "create" although kubectl opens 10 of 23 names, under the 60% that makes it common.
|
|
21
|
+
* percent_change keeps "percent": a prefix of one or two names is not a namespace.
|
|
22
|
+
*/
|
|
23
|
+
export declare function prefixWords(names: string[]): Set<string>;
|
|
16
24
|
/** The tool's verb: the first word that is one, skipping prefixes like `API-`. */
|
|
17
25
|
export declare function verbOf(name: string): string | undefined;
|
|
18
26
|
/**
|
package/dist/words.js
CHANGED
|
@@ -64,6 +64,27 @@ export function commonWords(names) {
|
|
|
64
64
|
counts.set(w, (counts.get(w) ?? 0) + 1);
|
|
65
65
|
return new Set([...counts].filter(([, n]) => names.length >= 4 && n >= names.length * 0.6).map(([w]) => w));
|
|
66
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Words a name's leading verb is read after: those most of the menu's names share
|
|
69
|
+
* (commonWords), and a namespace prefix: a word, not a verb, that opens 3 or more
|
|
70
|
+
* names and is followed by a verb in 3 or more of them. kubectl_create leads with
|
|
71
|
+
* "create" although kubectl opens 10 of 23 names, under the 60% that makes it common.
|
|
72
|
+
* percent_change keeps "percent": a prefix of one or two names is not a namespace.
|
|
73
|
+
*/
|
|
74
|
+
export function prefixWords(names) {
|
|
75
|
+
const out = commonWords(names);
|
|
76
|
+
const opened = new Map();
|
|
77
|
+
for (const name of names) {
|
|
78
|
+
const [first, second] = words(name);
|
|
79
|
+
if (first === undefined || VERBS.has(first) || second === undefined || !VERBS.has(second))
|
|
80
|
+
continue;
|
|
81
|
+
opened.set(first, (opened.get(first) ?? 0) + 1);
|
|
82
|
+
}
|
|
83
|
+
for (const [w, n] of opened)
|
|
84
|
+
if (n >= 3)
|
|
85
|
+
out.add(w);
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
67
88
|
/** The tool's verb: the first word that is one, skipping prefixes like `API-`. */
|
|
68
89
|
export function verbOf(name) {
|
|
69
90
|
return words(name).find((w) => VERBS.has(w));
|
|
@@ -74,9 +95,16 @@ export function verbOf(name) {
|
|
|
74
95
|
* in its object (score_status_update scores; "update" is what it scores).
|
|
75
96
|
*/
|
|
76
97
|
export function leadingVerb(name, common = new Set()) {
|
|
77
|
-
const
|
|
78
|
-
|
|
98
|
+
const rest = words(name).filter((w, i, all) => all.slice(0, i + 1).some((x) => !common.has(x)));
|
|
99
|
+
const [first, second] = rest;
|
|
100
|
+
if (first !== undefined && VERBS.has(first))
|
|
101
|
+
return first;
|
|
102
|
+
// Object first, then the verb (browserbase_session_create, session_close): only a word
|
|
103
|
+
// that can't be read as a noun, so percent_change still computes a change.
|
|
104
|
+
return first !== undefined && second !== undefined && ONLY_VERBS.has(second) ? second : undefined;
|
|
79
105
|
}
|
|
106
|
+
/** Verbs that aren't also nouns in tool names: not change, update, set, post, patch, copy or plan. */
|
|
107
|
+
const ONLY_VERBS = new Set(['create', 'add', 'insert', 'upsert', 'modify', 'delete', 'remove', 'purge', 'destroy', 'revoke', 'cancel', 'close', 'send', 'publish', 'submit', 'upload', 'rename', 'assign', 'invite', 'approve', 'get', 'list', 'fetch', 'retrieve', 'describe']);
|
|
80
108
|
/** The nouns in a tool name, singular: `list_team_audits` → team, audit. */
|
|
81
109
|
export function nouns(name) {
|
|
82
110
|
return words(name)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "toolmenu",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
|
|
5
5
|
"author": "Nikša Perović (https://niksa.me)",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"linter",
|
|
21
21
|
"agents",
|
|
22
22
|
"prompt-caching",
|
|
23
|
-
"cli"
|
|
23
|
+
"cli",
|
|
24
|
+
"mcp-server"
|
|
24
25
|
],
|
|
25
26
|
"license": "MIT",
|
|
26
27
|
"bugs": {
|
|
@@ -28,17 +29,17 @@
|
|
|
28
29
|
},
|
|
29
30
|
"homepage": "https://github.com/niksa90/toolmenu#readme",
|
|
30
31
|
"dependencies": {
|
|
31
|
-
"@modelcontextprotocol/client": "^2.
|
|
32
|
+
"@modelcontextprotocol/client": "^2.4.0",
|
|
33
|
+
"@modelcontextprotocol/server": "^2.4.0",
|
|
32
34
|
"ajv": "^8.20.0",
|
|
33
35
|
"ajv-formats": "^3.0.1",
|
|
34
36
|
"js-tiktoken": "^1.0.21",
|
|
35
|
-
"yaml": "^2.9.1"
|
|
37
|
+
"yaml": "^2.9.1",
|
|
38
|
+
"zod": "^4.6.7"
|
|
36
39
|
},
|
|
37
40
|
"devDependencies": {
|
|
38
|
-
"@modelcontextprotocol/server": "^2.1.0",
|
|
39
41
|
"@types/node": "^22.20.4",
|
|
40
|
-
"typescript": "^7.0.2"
|
|
41
|
-
"zod": "^4.6.5"
|
|
42
|
+
"typescript": "^7.0.2"
|
|
42
43
|
},
|
|
43
44
|
"type": "module",
|
|
44
45
|
"bin": {
|