toolmenu 0.18.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/dist/auto.js +9 -3
- package/dist/catalog.js +4 -0
- package/dist/cli.js +10 -3
- package/dist/clients.d.ts +7 -1
- package/dist/clients.js +29 -6
- package/dist/config.d.ts +2 -0
- package/dist/config.js +6 -2
- package/dist/diff.js +90 -0
- package/dist/explain.d.ts +8 -0
- package/dist/explain.js +44 -5
- package/dist/failures.d.ts +1 -1
- package/dist/failures.js +12 -2
- package/dist/help.js +2 -1
- package/dist/lists.d.ts +40 -0
- package/dist/lists.js +91 -0
- package/dist/options.d.ts +3 -0
- package/dist/options.js +1 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +19 -1
- package/dist/rules/clients.js +2 -2
- package/dist/rules/description.js +1 -1
- package/dist/rules/index.js +5 -1
- package/dist/rules/lists.d.ts +16 -0
- package/dist/rules/lists.js +102 -0
- package/dist/rules/rule.d.ts +2 -0
- package/dist/rules/search.js +2 -2
- package/dist/rules/spec.d.ts +2 -0
- package/dist/rules/spec.js +17 -8
- package/dist/rules/write.d.ts +7 -0
- package/dist/rules/write.js +41 -7
- package/dist/snapshot.d.ts +5 -0
- package/dist/snapshot.js +33 -2
- package/dist/types.d.ts +4 -0
- package/dist/words.d.ts +6 -0
- package/dist/words.js +9 -0
- package/package.json +1 -1
package/dist/lists.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Prompts and resources: the other lists a client loads up front (2026-07-28:
|
|
3
|
+
* cacheable like tools/list, SEP-2549, and no longer varying per connection).
|
|
4
|
+
* snapshot --lists tools,prompts,resources reads them; the rules in
|
|
5
|
+
* rules/lists.ts check them the way the menu rules check tools.
|
|
6
|
+
*/
|
|
7
|
+
import { quietExclusions } from './connect.js';
|
|
8
|
+
export const LIST_KINDS = ['prompts', 'resources', 'resourceTemplates'];
|
|
9
|
+
/** The --lists words: `resources` reads both resources/list and resources/templates/list. */
|
|
10
|
+
export const LIST_WORDS = ['tools', 'prompts', 'resources'];
|
|
11
|
+
/** Each list's JSON-RPC method, result key, the key that names an item, and its words. */
|
|
12
|
+
export const LIST_INFO = {
|
|
13
|
+
prompts: { method: 'prompts/list', key: 'prompts', id: 'name', one: 'prompt', capability: 'prompts' },
|
|
14
|
+
resources: { method: 'resources/list', key: 'resources', id: 'uri', one: 'resource', capability: 'resources' },
|
|
15
|
+
resourceTemplates: { method: 'resources/templates/list', key: 'resourceTemplates', id: 'uriTemplate', one: 'resource template', capability: 'resources' },
|
|
16
|
+
};
|
|
17
|
+
/** Pages read at most: a resources list can be a whole file tree. */
|
|
18
|
+
export const MAX_PAGES = 50;
|
|
19
|
+
/** Read one list, page by page, as the server sends it. */
|
|
20
|
+
export async function readList(connection, kind, timeoutMs) {
|
|
21
|
+
const info = LIST_INFO[kind];
|
|
22
|
+
const pages = [];
|
|
23
|
+
let cursor;
|
|
24
|
+
try {
|
|
25
|
+
for (let i = 0; i < MAX_PAGES; i++) {
|
|
26
|
+
const mark = connection.wire.responses.length;
|
|
27
|
+
const params = cursor ? { cursor } : undefined;
|
|
28
|
+
const call = () => {
|
|
29
|
+
const c = connection.client;
|
|
30
|
+
const fn = kind === 'prompts' ? 'listPrompts' : kind === 'resources' ? 'listResources' : 'listResourceTemplates';
|
|
31
|
+
return c[fn].call(connection.client, params, { timeout: timeoutMs, cacheMode: 'bypass' });
|
|
32
|
+
};
|
|
33
|
+
let failed;
|
|
34
|
+
await quietExclusions(call).catch((e) => (failed = e));
|
|
35
|
+
// The page as the server sent it: the SDK may reject a result toolmenu still checks.
|
|
36
|
+
const page = connection.wire.since(mark, info.method).at(-1)?.result;
|
|
37
|
+
if (!page)
|
|
38
|
+
throw failed ?? new Error(`no ${info.method} result`);
|
|
39
|
+
pages.push(page);
|
|
40
|
+
cursor = typeof page.nextCursor === 'string' && page.nextCursor ? page.nextCursor : undefined;
|
|
41
|
+
if (!cursor)
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
const code = error?.code;
|
|
47
|
+
return { kind, items: [], pages, listMeta: {}, error: (error instanceof Error ? error.message : String(error)).split('\n')[0].replace(/^MCP error -?\d+:\s*/, ''), ...(typeof code === 'number' ? { code } : {}) };
|
|
48
|
+
}
|
|
49
|
+
const first = pages[0] ?? {};
|
|
50
|
+
return {
|
|
51
|
+
kind,
|
|
52
|
+
items: pages.flatMap((p) => (Array.isArray(p[info.key]) ? p[info.key] : [])),
|
|
53
|
+
pages,
|
|
54
|
+
listMeta: {
|
|
55
|
+
...(typeof first.ttlMs === 'number' ? { ttlMs: first.ttlMs } : {}),
|
|
56
|
+
...(typeof first.cacheScope === 'string' ? { cacheScope: first.cacheScope } : {}),
|
|
57
|
+
},
|
|
58
|
+
...(cursor ? { truncated: true } : {}),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** The list kinds the --lists words ask for that the server declares a capability for, and the ones it doesn't. */
|
|
62
|
+
export function kindsToRead(words, capabilities) {
|
|
63
|
+
const wanted = [...(words.includes('prompts') ? ['prompts'] : []), ...(words.includes('resources') ? ['resources', 'resourceTemplates'] : [])];
|
|
64
|
+
return {
|
|
65
|
+
read: wanted.filter((k) => capabilities[LIST_INFO[k].capability] !== undefined),
|
|
66
|
+
undeclared: wanted.filter((k) => capabilities[LIST_INFO[k].capability] === undefined),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** Two reads of one list: what changed between them, by item id and in order. */
|
|
70
|
+
export function listChanges(kind, before, after) {
|
|
71
|
+
const id = (x) => String(x[LIST_INFO[kind].id] ?? JSON.stringify(x));
|
|
72
|
+
const a = new Map(before.map((x) => [id(x), JSON.stringify(x)]));
|
|
73
|
+
const b = new Map(after.map((x) => [id(x), JSON.stringify(x)]));
|
|
74
|
+
const out = [];
|
|
75
|
+
for (const k of a.keys())
|
|
76
|
+
if (!b.has(k))
|
|
77
|
+
out.push(`${k}: gone`);
|
|
78
|
+
for (const k of b.keys())
|
|
79
|
+
if (!a.has(k))
|
|
80
|
+
out.push(`${k}: new`);
|
|
81
|
+
for (const [k, v] of b)
|
|
82
|
+
if (a.has(k) && a.get(k) !== v)
|
|
83
|
+
out.push(`${k}: ${sameKeys(a.get(k), v) ? 'key order changed' : 'changed'}`);
|
|
84
|
+
if (!out.length && [...a.keys()].join('\n') !== [...b.keys()].join('\n'))
|
|
85
|
+
out.push('same items, different order');
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
function sameKeys(x, y) {
|
|
89
|
+
const sort = (v) => (Array.isArray(v) ? v.map(sort) : v && typeof v === 'object' ? Object.fromEntries(Object.entries(v).sort(([p], [q]) => p.localeCompare(q)).map(([k, w]) => [k, sort(w)])) : v);
|
|
90
|
+
return JSON.stringify(sort(JSON.parse(x))) === JSON.stringify(sort(JSON.parse(y)));
|
|
91
|
+
}
|
package/dist/options.d.ts
CHANGED
package/dist/options.js
CHANGED
|
@@ -14,6 +14,7 @@ export const CLI_OPTIONS = {
|
|
|
14
14
|
env: { type: 'string', multiple: true },
|
|
15
15
|
as: { type: 'string', multiple: true },
|
|
16
16
|
client: { type: 'string' },
|
|
17
|
+
lists: { type: 'string' },
|
|
17
18
|
timeout: { type: 'string' },
|
|
18
19
|
protocol: { type: 'string' },
|
|
19
20
|
elicitation: { type: 'string' },
|
package/dist/report.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { type Scenario, type SessionResult } from './session.js';
|
|
|
5
5
|
import { type Finding, type Menu, type Severity } from './types.js';
|
|
6
6
|
export type Format = 'text' | 'json' | 'github' | 'markdown';
|
|
7
7
|
export declare function counts(findings: Finding[]): Record<Severity, number>;
|
|
8
|
-
export declare function formatSnapshot(menu: Menu, findings: Finding[], format: Format, outPath?: string, cost?: ClientCost): string;
|
|
8
|
+
export declare function formatSnapshot(menu: Menu, findings: Finding[], format: Format, outPath?: string, cost?: ClientCost, notOffered?: string[]): string;
|
|
9
9
|
export declare function formatDiff(d: DiffResult, beforeTools: number, afterTools: number, format: Format): string;
|
|
10
10
|
export declare function formatHistory(h: HistoryResult, format: Format, outDir: string, csvPath?: string): string;
|
|
11
11
|
export declare function formatSession(s: SessionResult, format: Format): string;
|
package/dist/report.js
CHANGED
|
@@ -12,6 +12,14 @@ export function counts(findings) {
|
|
|
12
12
|
function plural(n, word) {
|
|
13
13
|
return `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
14
14
|
}
|
|
15
|
+
/** snapshot --lists: "3 prompts · 5 resources · 2 resource templates", or nothing. */
|
|
16
|
+
function otherLists(menu) {
|
|
17
|
+
return [
|
|
18
|
+
menu.prompts ? plural(menu.prompts.length, 'prompt') : '',
|
|
19
|
+
menu.resources ? plural(menu.resources.length, 'resource') : '',
|
|
20
|
+
menu.resourceTemplates ? plural(menu.resourceTemplates.length, 'resource template') : '',
|
|
21
|
+
].filter(Boolean).join(' · ');
|
|
22
|
+
}
|
|
15
23
|
/** --client: "as Claude Code: up front ~N tokens (…) · on demand ~M per tool loaded (largest: x, ~K)". */
|
|
16
24
|
function costLine(cost) {
|
|
17
25
|
const parts = [
|
|
@@ -20,7 +28,7 @@ function costLine(cost) {
|
|
|
20
28
|
].filter(Boolean);
|
|
21
29
|
return `as ${cost.label}: ${parts.join(' · ')}`;
|
|
22
30
|
}
|
|
23
|
-
export function formatSnapshot(menu, findings, format, outPath, cost) {
|
|
31
|
+
export function formatSnapshot(menu, findings, format, outPath, cost, notOffered = []) {
|
|
24
32
|
if (format === 'json') {
|
|
25
33
|
return JSON.stringify({
|
|
26
34
|
command: 'snapshot',
|
|
@@ -30,6 +38,10 @@ export function formatSnapshot(menu, findings, format, outPath, cost) {
|
|
|
30
38
|
listMeta: menu.listMeta,
|
|
31
39
|
breakdown: breakdown(menu),
|
|
32
40
|
...(cost ? { client: cost } : {}),
|
|
41
|
+
...(menu.prompts ? { prompts: menu.prompts.length } : {}),
|
|
42
|
+
...(menu.resources ? { resources: menu.resources.length } : {}),
|
|
43
|
+
...(menu.resourceTemplates ? { resourceTemplates: menu.resourceTemplates.length } : {}),
|
|
44
|
+
...(notOffered.length ? { notOffered } : {}),
|
|
33
45
|
written: outPath ?? null,
|
|
34
46
|
counts: counts(findings),
|
|
35
47
|
findings,
|
|
@@ -44,6 +56,7 @@ export function formatSnapshot(menu, findings, format, outPath, cost) {
|
|
|
44
56
|
'',
|
|
45
57
|
`Protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
46
58
|
...(cost ? ['', `${costLine(cost)}. _${cost.basis}._`] : []),
|
|
59
|
+
...(otherLists(menu) ? ['', `Also listed: ${otherLists(menu)}`] : []),
|
|
47
60
|
'',
|
|
48
61
|
...mdFindings(findings),
|
|
49
62
|
...mdBreakdown(menu),
|
|
@@ -56,6 +69,11 @@ export function formatSnapshot(menu, findings, format, outPath, cost) {
|
|
|
56
69
|
];
|
|
57
70
|
if (cost)
|
|
58
71
|
lines.push(` ${costLine(cost)}`, ` ${cost.basis}`);
|
|
72
|
+
const others = otherLists(menu);
|
|
73
|
+
// Lists asked for that the server doesn't offer: said once, not a finding.
|
|
74
|
+
const caps = [...new Set(notOffered.map((k) => (k === 'prompts' ? 'prompts' : 'resources')))];
|
|
75
|
+
if (others || caps.length)
|
|
76
|
+
lines.push(` ${[others ? `also listed: ${others}` : '', caps.length ? `no ${caps.join(' or ')} offered (not declared)` : ''].filter(Boolean).join(' · ')}`);
|
|
59
77
|
if (outPath)
|
|
60
78
|
lines.push(` wrote ${outPath}`);
|
|
61
79
|
lines.push('');
|
package/dist/rules/clients.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { nameProblem, profileOf } from '../clients.js';
|
|
2
2
|
import { count, kept } from './rule.js';
|
|
3
3
|
/**
|
|
4
4
|
* --client: tool names the client's API doesn't take. MCP allows names its APIs
|
|
@@ -10,7 +10,7 @@ export const clientName = {
|
|
|
10
10
|
severity: 'warn',
|
|
11
11
|
summary: "Tool names the client's API doesn't accept as they are",
|
|
12
12
|
run(ctx) {
|
|
13
|
-
const profile = ctx.client
|
|
13
|
+
const profile = profileOf(ctx.client);
|
|
14
14
|
if (!profile?.namePattern)
|
|
15
15
|
return [];
|
|
16
16
|
const bad = kept(ctx.menu.tools, ctx)
|
|
@@ -25,7 +25,7 @@ const CLAUDE_CODE_UPFRONT = 4096;
|
|
|
25
25
|
export function resolveLimit(setting) {
|
|
26
26
|
if (typeof setting === 'number')
|
|
27
27
|
return { limit: setting, who: 'your client', assumed: false, perTool: () => setting };
|
|
28
|
-
const client = setting !== undefined && setting
|
|
28
|
+
const client = setting !== undefined && Object.hasOwn(CLIENT_LIMITS, setting) ? setting : DEFAULT_CLIENT;
|
|
29
29
|
const limit = CLIENT_LIMITS[client];
|
|
30
30
|
const upfront = (tool) => tool._meta?.['anthropic/alwaysLoad'] === true;
|
|
31
31
|
return {
|
package/dist/rules/index.js
CHANGED
|
@@ -3,11 +3,12 @@ import { buried, cut, lateInstruction } from './description.js';
|
|
|
3
3
|
import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
5
|
import { clientName } from './clients.js';
|
|
6
|
+
import { listSchema, listVariance } from './lists.js';
|
|
6
7
|
import { handleLifetime } from './handles.js';
|
|
7
8
|
import { authVariance, cacheScopePublic } from './identities.js';
|
|
8
9
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
9
10
|
import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
|
|
10
|
-
import { noDryRun, unannotated } from './write.js';
|
|
11
|
+
import { noDryRun, readOnlyMismatch, unannotated } from './write.js';
|
|
11
12
|
import { gated } from './gated.js';
|
|
12
13
|
import { xMcpHeader } from './headers.js';
|
|
13
14
|
import { schemaRef } from './refs.js';
|
|
@@ -31,6 +32,7 @@ export const MENU_RULES = [
|
|
|
31
32
|
lateInstruction,
|
|
32
33
|
noDryRun,
|
|
33
34
|
unannotated,
|
|
35
|
+
readOnlyMismatch,
|
|
34
36
|
schema,
|
|
35
37
|
xMcpHeader,
|
|
36
38
|
schemaRef,
|
|
@@ -39,6 +41,8 @@ export const MENU_RULES = [
|
|
|
39
41
|
deprecated,
|
|
40
42
|
cacheHints,
|
|
41
43
|
authVariance,
|
|
44
|
+
listVariance,
|
|
45
|
+
listSchema,
|
|
42
46
|
cacheScopePublic,
|
|
43
47
|
serverInfo,
|
|
44
48
|
];
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type ListKind, type ListRead } from '../lists.js';
|
|
2
|
+
import { type Rule } from './rule.js';
|
|
3
|
+
/** What snapshot read of the other lists, for these rules. */
|
|
4
|
+
export interface ListsContext {
|
|
5
|
+
/** Each list read twice on one connection, and once from a second process (stdio) or connection (HTTP). */
|
|
6
|
+
read: {
|
|
7
|
+
kind: ListKind;
|
|
8
|
+
first: ListRead;
|
|
9
|
+
second?: ListRead;
|
|
10
|
+
probe?: ListRead;
|
|
11
|
+
}[];
|
|
12
|
+
/** Lists asked for that the server declares no capability for. */
|
|
13
|
+
undeclared: ListKind[];
|
|
14
|
+
}
|
|
15
|
+
export declare const listVariance: Rule;
|
|
16
|
+
export declare const listSchema: Rule;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { LIST_INFO, listChanges } from '../lists.js';
|
|
2
|
+
import { classifyFailure, SETUP_FAILURES } from '../failures.js';
|
|
3
|
+
import { count } from './rule.js';
|
|
4
|
+
import { resultValidator } from './spec.js';
|
|
5
|
+
const RESULT = { prompts: 'ListPromptsResult', resources: 'ListResourcesResult', resourceTemplates: 'ListResourceTemplatesResult' };
|
|
6
|
+
export const listVariance = {
|
|
7
|
+
id: 'lists/variance',
|
|
8
|
+
severity: 'warn',
|
|
9
|
+
summary: 'Prompts and resources lists are the same every time they are listed, and from a fresh process or connection',
|
|
10
|
+
run(ctx) {
|
|
11
|
+
const findings = [];
|
|
12
|
+
for (const r of ctx.lists?.read ?? []) {
|
|
13
|
+
const info = LIST_INFO[r.kind];
|
|
14
|
+
if (r.first.error)
|
|
15
|
+
continue;
|
|
16
|
+
const again = r.second && !r.second.error ? listChanges(r.kind, r.first.items, r.second.items) : [];
|
|
17
|
+
if (again.length) {
|
|
18
|
+
findings.push({
|
|
19
|
+
message: `${info.method} differs between two calls on one connection: ${count(again.length, 'change')}. A client that caches it (ttlMs) or compares it to decide whether to refresh sees a new list every time.`,
|
|
20
|
+
detail: again.slice(0, 8).concat(again.length > 8 ? [`…and ${again.length - 8} more`] : []),
|
|
21
|
+
fix: `Build ${info.method} the same way every time: a stable order and no per-call values (timestamps, counters).`,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
const fresh = r.probe && !r.probe.error ? listChanges(r.kind, r.first.items, r.probe.items) : [];
|
|
25
|
+
if (fresh.length && !again.length) {
|
|
26
|
+
const where = ctx.transport === 'http' ? 'connection' : 'process';
|
|
27
|
+
findings.push({
|
|
28
|
+
message: `${info.method} from a second ${where} differs from the first: ${count(fresh.length, 'change')}. 2026-07-28 lists no longer vary per connection, so clients and caches treat one ${where}'s list as everyone's.`,
|
|
29
|
+
detail: fresh.slice(0, 8).concat(fresh.length > 8 ? [`…and ${fresh.length - 8} more`] : []),
|
|
30
|
+
fix: `Build ${info.method} from what's the same for every ${where}: no per-start ordering (a set, a hash map), no per-${where} state.`,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return findings;
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
export const listSchema = {
|
|
38
|
+
id: 'lists/schema',
|
|
39
|
+
severity: 'error',
|
|
40
|
+
summary: 'Prompts and resources list results match the official schema for their protocol version',
|
|
41
|
+
run(ctx) {
|
|
42
|
+
if (!ctx.protocolVersion)
|
|
43
|
+
return [];
|
|
44
|
+
const findings = [];
|
|
45
|
+
for (const r of ctx.lists?.read ?? []) {
|
|
46
|
+
const info = LIST_INFO[r.kind];
|
|
47
|
+
if (r.first.error) {
|
|
48
|
+
const said = r.first.error.replace(/[.\s]+$/, '');
|
|
49
|
+
const failure = classifyFailure(r.first.error, { code: r.first.code });
|
|
50
|
+
if (r.first.code === -32601 || /method not (?:found|supported|implemented)/i.test(r.first.error)) {
|
|
51
|
+
// A server that declares resources but answers one of their methods with Method not found.
|
|
52
|
+
findings.push({
|
|
53
|
+
severity: 'info',
|
|
54
|
+
message: `The server declares ${info.capability} but answers ${info.method} with Method not found (-32601), so a client that asks for ${info.key === 'resourceTemplates' ? 'resource templates' : info.key} gets an error, not an empty list.`,
|
|
55
|
+
fix: `Answer ${info.method} with an empty list if there are none.`,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
else if (SETUP_FAILURES.has(failure)) {
|
|
59
|
+
// Credentials, this machine, a missing app: a failure of the setup, not of the server.
|
|
60
|
+
findings.push({
|
|
61
|
+
severity: 'info',
|
|
62
|
+
message: `${info.method} couldn't be read here (${said}), so it wasn't checked: a setup problem, not the server's.`,
|
|
63
|
+
fix: `Rerun where ${info.method} can answer (credentials, the app or data it needs) to check it.`,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
findings.push({
|
|
68
|
+
severity: 'warn',
|
|
69
|
+
message: `The server declares ${info.capability}, but ${info.method} failed: ${said}. Clients that load ${info.key === 'resourceTemplates' ? 'resource templates' : info.key} up front get an error instead.`,
|
|
70
|
+
fix: `Answer ${info.method} (an empty list is fine), or stop declaring the ${info.capability} capability.`,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (r.first.truncated) {
|
|
76
|
+
// Not a defect: said so that no one reads the checks as covering the whole list.
|
|
77
|
+
findings.push({
|
|
78
|
+
severity: 'info',
|
|
79
|
+
message: `${info.method} has more than ${r.first.pages.length} pages; toolmenu read the first ${r.first.pages.length} (${r.first.items.length} items), so the checks and the menu file cover those only.`,
|
|
80
|
+
fix: 'Nothing to change for this; a client that loads the whole list up front pays for every page.',
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
const validate = resultValidator(ctx.protocolVersion, RESULT[r.kind]);
|
|
84
|
+
if (!validate)
|
|
85
|
+
continue;
|
|
86
|
+
r.first.pages.forEach((page, i) => {
|
|
87
|
+
if (validate(page))
|
|
88
|
+
return;
|
|
89
|
+
const errors = validate.errors ?? [];
|
|
90
|
+
const missing = [...new Set(errors.filter((e) => !e.instancePath && e.params && 'missingProperty' in e.params).map((e) => String(e.params.missingProperty)))];
|
|
91
|
+
findings.push({
|
|
92
|
+
message: `The ${info.method} result${r.first.pages.length > 1 ? ` (page ${i + 1})` : ''} doesn't match the official ${ctx.protocolVersion} schema, so a strict client may refuse it.`,
|
|
93
|
+
detail: errors.slice(0, 5).map((e) => `${e.instancePath || '(result)'} ${e.message}${e.params && 'missingProperty' in e.params ? `: ${e.params.missingProperty}` : ''}`).concat(errors.length > 5 ? [`…and ${errors.length - 5} more`] : []),
|
|
94
|
+
fix: missing.length
|
|
95
|
+
? `Add ${missing.join(', ')} to the ${info.method} result${missing.some((m) => m === 'ttlMs' || m === 'cacheScope') ? ` (${ctx.protocolVersion} requires cache hints on it, SEP-2549)` : ''}.`
|
|
96
|
+
: `Fix ${errors[0]?.instancePath || 'the result'} first (${errors[0]?.message ?? 'see above'}), then rerun.`,
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
return findings;
|
|
101
|
+
},
|
|
102
|
+
};
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface RuleContext {
|
|
|
28
28
|
server?: string;
|
|
29
29
|
/** snapshot --as: each identity's menu, the main one first (menu/auth-variance, spec/cache-scope-public). */
|
|
30
30
|
identities?: import('./identities.js').IdentityMenu[];
|
|
31
|
+
/** snapshot --lists: the prompts and resources lists read (lists/* rules). */
|
|
32
|
+
lists?: import('./lists.js').ListsContext;
|
|
31
33
|
/** --client: the client profile the menu is checked for (src/clients.ts). */
|
|
32
34
|
client?: string;
|
|
33
35
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
package/dist/rules/search.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { profileOf } from '../clients.js';
|
|
2
2
|
import { buildIndex, placeIn, placeOf, SEARCH_LIMIT } from '../search.js';
|
|
3
3
|
import { commonWords, VERBS, words } from '../words.js';
|
|
4
4
|
import { clip, count, kept, names, num } from './rule.js';
|
|
@@ -20,7 +20,7 @@ export const findable = {
|
|
|
20
20
|
summary: 'Each tool comes up when searched for by its own name, for clients that defer tools',
|
|
21
21
|
run(ctx) {
|
|
22
22
|
const tools = ctx.menu.tools;
|
|
23
|
-
const profile = ctx.client
|
|
23
|
+
const profile = profileOf(ctx.client);
|
|
24
24
|
if (profile) {
|
|
25
25
|
// A client that defers tools does it at any size; one whose search results
|
|
26
26
|
// aren't documented (OpenAI's spec doesn't say how many) isn't ranked against a guess.
|
package/dist/rules/spec.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { Rule } from './rule.js';
|
|
2
|
+
/** A validator for one result type ($defs name) of a protocol version, or null when this toolmenu has no schema for it. */
|
|
3
|
+
export declare function resultValidator(version: string, result?: string): import("ajv").ValidateFunction<unknown> | null;
|
|
2
4
|
export declare const schema: Rule;
|
|
3
5
|
export declare const toolsCapability: Rule;
|
|
4
6
|
export declare const duplicateName: Rule;
|
package/dist/rules/spec.js
CHANGED
|
@@ -7,24 +7,33 @@ import addFormatsImport from 'ajv-formats';
|
|
|
7
7
|
const addFormats = (addFormatsImport.default ?? addFormatsImport);
|
|
8
8
|
const SCHEMA_DIR = fileURLToPath(new URL('../../schemas/', import.meta.url));
|
|
9
9
|
const validators = new Map();
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/** A validator for one result type ($defs name) of a protocol version, or null when this toolmenu has no schema for it. */
|
|
11
|
+
export function resultValidator(version, result = 'ListToolsResult') {
|
|
12
|
+
const key = `${version}\0${result}`;
|
|
13
|
+
if (!validators.has(key)) {
|
|
12
14
|
const file = `${SCHEMA_DIR}${version}.json`;
|
|
13
15
|
if (!existsSync(file)) {
|
|
14
|
-
validators.set(
|
|
16
|
+
validators.set(key, null);
|
|
15
17
|
}
|
|
16
18
|
else {
|
|
17
19
|
const schema = JSON.parse(readFileSync(file, 'utf8'));
|
|
18
20
|
// 2025-11-25 and later are draft 2020-12 with $defs; earlier ones are draft-07 with definitions.
|
|
19
21
|
const modernDraft = String(schema.$schema).includes('2020-12');
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
const pool = modernDraft ? schema.$defs : schema.definitions;
|
|
23
|
+
if (!pool?.[result]) {
|
|
24
|
+
validators.set(key, null);
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
const ajv = modernDraft ? new Ajv2020({ strict: false, allErrors: true }) : new Ajv({ strict: false, allErrors: true });
|
|
28
|
+
addFormats(ajv);
|
|
29
|
+
ajv.addSchema(schema, 'mcp');
|
|
30
|
+
validators.set(key, ajv.compile({ $ref: `mcp#/${modernDraft ? '$defs' : 'definitions'}/${result}` }));
|
|
31
|
+
}
|
|
24
32
|
}
|
|
25
33
|
}
|
|
26
|
-
return validators.get(
|
|
34
|
+
return validators.get(key) ?? null;
|
|
27
35
|
}
|
|
36
|
+
const listToolsValidator = (version) => resultValidator(version, 'ListToolsResult');
|
|
28
37
|
export const schema = {
|
|
29
38
|
id: 'spec/schema',
|
|
30
39
|
severity: 'error',
|
package/dist/rules/write.d.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
import { type Rule } from './rule.js';
|
|
2
2
|
export declare const unannotated: Rule;
|
|
3
|
+
/**
|
|
4
|
+
* readOnlyHint: true on a tool whose name says it writes (create_, delete_, send_…).
|
|
5
|
+
* The spec's readOnlyHint means "the tool does not modify its environment", and
|
|
6
|
+
* clients and agents act on it: run it without asking, retry it, call it to explore.
|
|
7
|
+
* Seen on a hosted server: create_my_ideal_account, marked read-only.
|
|
8
|
+
*/
|
|
9
|
+
export declare const readOnlyMismatch: Rule;
|
|
3
10
|
export declare const noDryRun: Rule;
|
package/dist/rules/write.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
-
import { WRITE_VERBS,
|
|
1
|
+
import { WRITE_VERBS, commonWords, leadingVerb, nouns, words } from '../words.js';
|
|
2
2
|
import { count, kept, names } from './rule.js';
|
|
3
3
|
const DRY_RUN_PARAMS = /^(dry[_-]?run|preview|validate[_-]?only|simulate|plan|what[_-]?if)$/i;
|
|
4
4
|
const PREVIEW_WORDS = new Set(['preview', 'validate', 'plan', 'simulate', 'dry']);
|
|
5
5
|
/** Write verbs that only add something; the others delete or overwrite. A guess from the name. */
|
|
6
6
|
const ADDITIVE_VERBS = new Set(['create', 'add', 'insert', 'send', 'post', 'publish', 'submit', 'upload', 'assign', 'invite', 'approve']);
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Named like a write: the name leads with a write verb, after a prefix most of the
|
|
9
|
+
* menu's tools share. Not a verb further in: percent_change computes a change, and
|
|
10
|
+
* score_status_update scores one (both seen on scanned servers).
|
|
11
|
+
*/
|
|
12
|
+
function isWriteByName(tool, common = new Set()) {
|
|
13
|
+
return WRITE_VERBS.has(leadingVerb(tool.name, common) ?? '');
|
|
9
14
|
}
|
|
10
15
|
function hasAnnotations(tool) {
|
|
11
16
|
return !!tool.annotations && Object.keys(tool.annotations).length > 0;
|
|
@@ -17,10 +22,11 @@ export const unannotated = {
|
|
|
17
22
|
lesson: '02',
|
|
18
23
|
summary: 'Tools that look like writes carry annotations',
|
|
19
24
|
run(ctx) {
|
|
20
|
-
const
|
|
25
|
+
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
26
|
+
const bare = kept(ctx.menu.tools, ctx).filter((t) => isWriteByName(t, common) && !hasAnnotations(t));
|
|
21
27
|
if (bare.length === 0)
|
|
22
28
|
return [];
|
|
23
|
-
const additive = (t) => ADDITIVE_VERBS.has(
|
|
29
|
+
const additive = (t) => ADDITIVE_VERBS.has(leadingVerb(t.name, common));
|
|
24
30
|
// One finding, not one per tool: the same gap in 53 tools is one thing to fix.
|
|
25
31
|
if (bare.length === 1) {
|
|
26
32
|
const [tool] = bare;
|
|
@@ -29,12 +35,12 @@ export const unannotated = {
|
|
|
29
35
|
{
|
|
30
36
|
tool: tool.name,
|
|
31
37
|
confidence: 'unsure',
|
|
32
|
-
message: `${tool.name} is named like a write ("${
|
|
38
|
+
message: `${tool.name} is named like a write ("${leadingVerb(tool.name, common)}") and has no annotations. ${WHY}`,
|
|
33
39
|
fix: `Add annotations: { readOnlyHint: false, destructiveHint: ${!adds} } to ${tool.name} (${adds ? 'true if it can also delete or overwrite' : 'false if it only adds'}), or { readOnlyHint: true } if it changes nothing.`,
|
|
34
40
|
},
|
|
35
41
|
];
|
|
36
42
|
}
|
|
37
|
-
const verbs = [...new Set(bare.map((t) => `${
|
|
43
|
+
const verbs = [...new Set(bare.map((t) => `${leadingVerb(t.name, common)}…`))];
|
|
38
44
|
const overwrites = bare.filter((t) => !additive(t)).map((t) => t.name);
|
|
39
45
|
const adds = bare.filter(additive).map((t) => t.name);
|
|
40
46
|
return [
|
|
@@ -50,6 +56,34 @@ export const unannotated = {
|
|
|
50
56
|
];
|
|
51
57
|
},
|
|
52
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* readOnlyHint: true on a tool whose name says it writes (create_, delete_, send_…).
|
|
61
|
+
* The spec's readOnlyHint means "the tool does not modify its environment", and
|
|
62
|
+
* clients and agents act on it: run it without asking, retry it, call it to explore.
|
|
63
|
+
* Seen on a hosted server: create_my_ideal_account, marked read-only.
|
|
64
|
+
*/
|
|
65
|
+
export const readOnlyMismatch = {
|
|
66
|
+
id: 'write/read-only-mismatch',
|
|
67
|
+
severity: 'warn',
|
|
68
|
+
summary: 'Tools marked read-only whose names say they write',
|
|
69
|
+
run(ctx) {
|
|
70
|
+
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
71
|
+
const lead = (t) => leadingVerb(t.name, common);
|
|
72
|
+
const odd = kept(ctx.menu.tools, ctx).filter((t) => t.annotations?.readOnlyHint === true && WRITE_VERBS.has(lead(t) ?? ''));
|
|
73
|
+
if (!odd.length)
|
|
74
|
+
return [];
|
|
75
|
+
const one = odd.length === 1;
|
|
76
|
+
return [
|
|
77
|
+
{
|
|
78
|
+
...(one ? { tool: odd[0].name } : {}),
|
|
79
|
+
confidence: 'unsure',
|
|
80
|
+
message: `${one ? `${odd[0].name} is` : `${count(odd.length, 'tool')} are`} marked readOnlyHint: true but named like ${one ? 'a write' : 'writes'} (${names([...new Set(odd.map((t) => `${lead(t)}…`))], 6)}). readOnlyHint says a tool doesn't change anything, and clients act on it: one may run ${one ? 'it' : 'them'} without asking the user, or call ${one ? 'it' : 'them'} just to look. If ${one ? 'it changes' : 'they change'} anything, even state on the server, the hint is wrong.`,
|
|
81
|
+
...(one ? {} : { detail: [odd.map((t) => t.name).join(', ')] }),
|
|
82
|
+
fix: `Where the tool changes anything, set readOnlyHint: false (and destructiveHint to whether it deletes or overwrites); where it truly only reads, a name without the write verb says so to agents too.`,
|
|
83
|
+
},
|
|
84
|
+
];
|
|
85
|
+
},
|
|
86
|
+
};
|
|
53
87
|
export const noDryRun = {
|
|
54
88
|
id: 'write/no-dry-run',
|
|
55
89
|
severity: 'info',
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type Target } from './connect.js';
|
|
2
2
|
import { type Identity } from './probe.js';
|
|
3
|
+
import { type ListKind } from './lists.js';
|
|
3
4
|
import type { IdentityMenu } from './rules/identities.js';
|
|
4
5
|
import { type ClientCost } from './clients.js';
|
|
5
6
|
import { type RuleSettings } from './rules/index.js';
|
|
@@ -23,6 +24,8 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
23
24
|
* menu is the one saved and checked; the others are compared with it.
|
|
24
25
|
*/
|
|
25
26
|
identities?: Identity[];
|
|
27
|
+
/** --lists: which lists besides tools to read and check (prompts, resources). */
|
|
28
|
+
lists?: string[];
|
|
26
29
|
/** --client: check the menu for this client (src/clients.ts): its description cut, its search, its name format. */
|
|
27
30
|
client?: string;
|
|
28
31
|
}
|
|
@@ -33,6 +36,8 @@ export interface SnapshotResult {
|
|
|
33
36
|
identities?: IdentityMenu[];
|
|
34
37
|
/** --client: what the menu costs up front and per tool a search loads. */
|
|
35
38
|
cost?: ClientCost;
|
|
39
|
+
/** --lists: lists asked for that the server declares no capability for, so weren't read. */
|
|
40
|
+
notOffered?: ListKind[];
|
|
36
41
|
}
|
|
37
42
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
38
43
|
export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
package/dist/snapshot.js
CHANGED
|
@@ -2,7 +2,8 @@ import { listTools } from './connect.js';
|
|
|
2
2
|
import { patiently, tooMany } from './failures.js';
|
|
3
3
|
import { ConnectError } from './explain.js';
|
|
4
4
|
import { buildMenu } from './menu.js';
|
|
5
|
-
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
5
|
+
import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, PROBE_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
|
+
import { kindsToRead, readList } from './lists.js';
|
|
6
7
|
import { clientCost } from './clients.js';
|
|
7
8
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
8
9
|
import { catalogFindings, readCatalog } from './catalog.js';
|
|
@@ -18,6 +19,8 @@ export async function snapshot(given, options = {}) {
|
|
|
18
19
|
let menu;
|
|
19
20
|
let secondList;
|
|
20
21
|
let first;
|
|
22
|
+
const reads = [];
|
|
23
|
+
let undeclared = [];
|
|
21
24
|
try {
|
|
22
25
|
first = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
23
26
|
const second = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
@@ -29,6 +32,16 @@ export async function snapshot(given, options = {}) {
|
|
|
29
32
|
};
|
|
30
33
|
menu = buildMenu(first.tools, server, first.listMeta);
|
|
31
34
|
secondList = buildMenu(second.tools, server, second.listMeta).tools;
|
|
35
|
+
// --lists: prompts and resources, each read twice on this connection.
|
|
36
|
+
const kinds = kindsToRead(options.lists ?? [], connection.capabilities);
|
|
37
|
+
undeclared = kinds.undeclared;
|
|
38
|
+
for (const kind of kinds.read) {
|
|
39
|
+
const first = await readList(connection, kind, options.timeoutMs);
|
|
40
|
+
const second = first.error ? undefined : await readList(connection, kind, options.timeoutMs);
|
|
41
|
+
reads.push({ kind, first, ...(second ? { second } : {}) });
|
|
42
|
+
if (!first.error)
|
|
43
|
+
menu[kind] = first.items;
|
|
44
|
+
}
|
|
32
45
|
if (options.catalog) {
|
|
33
46
|
menu.catalog = await readCatalog(connection, menu.tools, { ...(typeof options.catalog === 'object' ? options.catalog : {}), timeoutMs: options.timeoutMs });
|
|
34
47
|
}
|
|
@@ -46,6 +59,23 @@ export async function snapshot(given, options = {}) {
|
|
|
46
59
|
// After the main process has exited: a server that holds a file or a port
|
|
47
60
|
// shouldn't have to share it with its own probe.
|
|
48
61
|
const probes = await probeVariance(target, menu.tools, options.processes ?? 2, options.timeoutMs ?? 30_000);
|
|
62
|
+
// The other lists from a second process (stdio) or connection (HTTP), as tools get.
|
|
63
|
+
if ((options.processes ?? 2) > 1 && reads.some((r) => !r.first.error)) {
|
|
64
|
+
try {
|
|
65
|
+
const probe = await connectPatiently(seeded(target, PROBE_SEED), { timeoutMs: options.timeoutMs });
|
|
66
|
+
try {
|
|
67
|
+
for (const r of reads)
|
|
68
|
+
if (!r.first.error)
|
|
69
|
+
r.probe = await readList(probe, r.kind, options.timeoutMs);
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
await probe.close().catch(() => { });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// The tool probe above says why a second process or connection failed.
|
|
77
|
+
}
|
|
78
|
+
}
|
|
49
79
|
// Each other identity's menu, from its own process (stdio) or connection (HTTP).
|
|
50
80
|
const identities = mainIdentity
|
|
51
81
|
? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
|
|
@@ -80,6 +110,7 @@ export async function snapshot(given, options = {}) {
|
|
|
80
110
|
serverInfo: connection.server,
|
|
81
111
|
server: serverArgs(target),
|
|
82
112
|
identities,
|
|
113
|
+
...(reads.length || undeclared.length ? { lists: { read: reads, undeclared } } : {}),
|
|
83
114
|
}, options);
|
|
84
115
|
// What --catalog called and what it got, under the same settings as the rules
|
|
85
116
|
// (rules: { "catalog/read": "off" }, ignore).
|
|
@@ -94,7 +125,7 @@ export async function snapshot(given, options = {}) {
|
|
|
94
125
|
findings.sort((a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
95
126
|
}
|
|
96
127
|
const cost = options.client ? clientCost(options.client, menu) : undefined;
|
|
97
|
-
return { menu, findings, ...(identities ? { identities } : {}), ...(cost ? { cost } : {}) };
|
|
128
|
+
return { menu, findings, ...(identities ? { identities } : {}), ...(cost ? { cost } : {}), ...(undeclared.length ? { notOffered: undeclared } : {}) };
|
|
98
129
|
}
|
|
99
130
|
const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
|
|
100
131
|
/**
|
package/dist/types.d.ts
CHANGED
|
@@ -62,6 +62,10 @@ export interface Menu {
|
|
|
62
62
|
ttlMs?: number;
|
|
63
63
|
cacheScope?: string;
|
|
64
64
|
};
|
|
65
|
+
/** snapshot --lists: the other lists as the server sent them (additive; readers that don't know them ignore them). */
|
|
66
|
+
prompts?: Record<string, unknown>[];
|
|
67
|
+
resources?: Record<string, unknown>[];
|
|
68
|
+
resourceTemplates?: Record<string, unknown>[];
|
|
65
69
|
/**
|
|
66
70
|
* 'session': every tool a session saw (session --union-out), in the order first
|
|
67
71
|
* seen, which is the unlock order, not an order any client was served.
|