toolmenu 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -20
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +53 -17
- package/dist/auto.d.ts +70 -6
- package/dist/auto.js +248 -17
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +62 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +80 -170
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +526 -114
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +9 -0
- package/dist/failures.js +17 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +92 -2
- package/dist/init.js +507 -20
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +85 -42
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +145 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +12 -4
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.js +33 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/session.d.ts +18 -8
- package/dist/session.js +484 -92
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +17 -2
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/rules/naming.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { commonWords, nouns, singular, words } from '../words.js';
|
|
1
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, WRITE_VERBS, commonWords, nouns, singular, verbOf, words } from '../words.js';
|
|
2
2
|
import { missingWords, negatedMention, routeScores } from '../routes.js';
|
|
3
|
+
import { kept, names } from './rule.js';
|
|
3
4
|
// Not `ref`: in real menus it's a git ref (GitHub ×4), a name with a meaning of its own.
|
|
4
5
|
const VAGUE_IDS = /^(id|ids|uuid|uuids|identifier|identifiers)$/i;
|
|
5
6
|
export const vagueId = {
|
|
@@ -8,20 +9,47 @@ export const vagueId = {
|
|
|
8
9
|
lesson: '04',
|
|
9
10
|
summary: 'ID parameters say which kind of thing they identify',
|
|
10
11
|
run(ctx) {
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
message: `${tool.name}.${param}: says "${param}" but not which kind of thing it identifies. Name it after the thing (e.g. form_id) so the agent doesn't pass the wrong ID.`,
|
|
18
|
-
});
|
|
19
|
-
}
|
|
12
|
+
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
13
|
+
const found = [];
|
|
14
|
+
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
15
|
+
const params = Object.keys(tool.inputSchema?.properties ?? {});
|
|
16
|
+
for (const param of params.filter((p) => VAGUE_IDS.test(p))) {
|
|
17
|
+
found.push({ tool: tool.name, param, where: `${tool.name}.${param}`, suggestion: betterName(tool.name, param, params, common) });
|
|
20
18
|
}
|
|
21
19
|
}
|
|
22
|
-
|
|
20
|
+
if (found.length === 0)
|
|
21
|
+
return [];
|
|
22
|
+
const one = found.length === 1;
|
|
23
|
+
const example = found.find((f) => f.suggestion) ?? found[0];
|
|
24
|
+
const tools = [...new Set(found.map((f) => f.tool))];
|
|
25
|
+
const said = names([...new Set(found.map((f) => `"${f.param}"`))].slice(0, 4));
|
|
26
|
+
// One finding: the same naming habit in 20 tools is one thing to change.
|
|
27
|
+
return [
|
|
28
|
+
{
|
|
29
|
+
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
30
|
+
confidence: 'unsure',
|
|
31
|
+
message: `${one ? `${found[0].where} is` : `${found.length} parameters are`} called just ${said}, which doesn't say which kind of thing ${one ? 'it identifies' : 'each one identifies'}. An agent holding several kinds of ID has only the description to pick the right one.`,
|
|
32
|
+
...(one ? {} : { detail: found.map((f) => f.where + (f.suggestion ? ` (${f.suggestion}?)` : '')) }),
|
|
33
|
+
fix: `Name ${one ? 'it' : 'each one'} after what it identifies${example.suggestion ? ` (${example.where} → ${example.suggestion})` : ' (form_id, not id)'}. If renaming would break callers, say in the parameter's description which kind of ID it takes and which tool returns it.`,
|
|
34
|
+
},
|
|
35
|
+
];
|
|
23
36
|
},
|
|
24
37
|
};
|
|
38
|
+
/** thing_id for get_thing.id, when the tool name has exactly one subject noun. */
|
|
39
|
+
function betterName(tool, param, params, common) {
|
|
40
|
+
// Only where the verb acts on the thing itself: get_monitor.id is a monitor's,
|
|
41
|
+
// check_crawl_status.id is a crawl job's, not a status's. Not on collection
|
|
42
|
+
// verbs: list_comments.id is usually the post's, not a comment's.
|
|
43
|
+
const verb = verbOf(tool);
|
|
44
|
+
if (!verb || COLLECTION_VERBS.has(verb) || !(LOOKUP_VERBS.has(verb) || WRITE_VERBS.has(verb)))
|
|
45
|
+
return undefined;
|
|
46
|
+
const subject = [...new Set(nouns(tool).filter((n) => !common.has(n)))];
|
|
47
|
+
if (subject.length !== 1)
|
|
48
|
+
return undefined;
|
|
49
|
+
const lower = param.toLowerCase();
|
|
50
|
+
// Follow the tool's own style: formId next to other camelCase parameters.
|
|
51
|
+
return params.some((p) => /[a-z][A-Z]/.test(p)) ? subject[0] + lower[0].toUpperCase() + lower.slice(1) : `${subject[0]}_${lower}`;
|
|
52
|
+
}
|
|
25
53
|
export const sharedWord = {
|
|
26
54
|
id: 'naming/shared-word',
|
|
27
55
|
severity: 'info',
|
|
@@ -59,8 +87,9 @@ export const sharedWord = {
|
|
|
59
87
|
}
|
|
60
88
|
}
|
|
61
89
|
}
|
|
62
|
-
|
|
63
|
-
|
|
90
|
+
const shown = [...clashes].filter((t) => !ctx.isIgnored?.(t));
|
|
91
|
+
if (shown.length > 1)
|
|
92
|
+
groups.push({ noun, tools: shown });
|
|
64
93
|
}
|
|
65
94
|
if (groups.length === 0)
|
|
66
95
|
return [];
|
|
@@ -69,7 +98,9 @@ export const sharedWord = {
|
|
|
69
98
|
const shown = groups.slice(0, 8);
|
|
70
99
|
return [
|
|
71
100
|
{
|
|
72
|
-
|
|
101
|
+
confidence: 'unsure',
|
|
102
|
+
message: `${groups.length === 1 ? `"${groups[0].noun}" appears` : `${groups.length} words appear`} in tool names that qualify ${groups.length === 1 ? 'it' : 'them'} differently, the way list_team_audits and get_audit_trail both say "audit". Where the word means two things, an agent asked about it can pick the wrong tool; where it means one, all is well.`,
|
|
103
|
+
fix: 'For each word that means two things, rename one side or pin the right tool for the word in routes.yml, so a later rename can\'t quietly undo it.',
|
|
73
104
|
detail: shown.map((g) => `"${g.noun}": ${g.tools.join(', ')}`).concat(groups.length > shown.length ? [`…and ${groups.length - shown.length} more (--json lists all)`] : []),
|
|
74
105
|
words: groups,
|
|
75
106
|
},
|
|
@@ -94,8 +125,17 @@ export const route = {
|
|
|
94
125
|
if (!names.has(name)) {
|
|
95
126
|
// A server that announces list changes may unlock the tool later.
|
|
96
127
|
findings.push(gated
|
|
97
|
-
? {
|
|
98
|
-
|
|
128
|
+
? {
|
|
129
|
+
tool: name,
|
|
130
|
+
severity: 'info',
|
|
131
|
+
message: `routes.yml "${keyword}" names ${name}, which isn't in the menu yet. The server announces menu changes, so it may appear after an unlock.`,
|
|
132
|
+
fix: `If ${name} was renamed or removed, update routes.yml; if it comes with an unlock, check this route against the menu after it (toolmenu session).`,
|
|
133
|
+
}
|
|
134
|
+
: {
|
|
135
|
+
tool: name,
|
|
136
|
+
message: `routes.yml "${keyword}" names ${name}, which isn't in the menu, so this route can't be checked. Renamed or removed?`,
|
|
137
|
+
fix: `Update "${keyword}" in routes.yml to the tool's current name, or take ${name} out of it.`,
|
|
138
|
+
});
|
|
99
139
|
}
|
|
100
140
|
}
|
|
101
141
|
const scores = routeScores(keyword, ctx.menu.tools);
|
|
@@ -113,8 +153,11 @@ export const route = {
|
|
|
113
153
|
findings.push({
|
|
114
154
|
tool: good,
|
|
115
155
|
message: missing.length > 0
|
|
116
|
-
? `"${keyword}" doesn't match ${good} at all: ${missing.map((w) => `"${w}"`).join(', ')} ${missing.length === 1 ? "isn't" : "aren't"} in its name or description.`
|
|
117
|
-
: `"${keyword}" doesn't match ${good} at all.`,
|
|
156
|
+
? `"${keyword}" doesn't match ${good} at all, the tool routes.yml says it must reach: ${missing.map((w) => `"${w}"`).join(', ')} ${missing.length === 1 ? "isn't" : "aren't"} in its name or description.`
|
|
157
|
+
: `"${keyword}" doesn't match ${good} at all, the tool routes.yml says it must reach.`,
|
|
158
|
+
fix: missing.length > 0
|
|
159
|
+
? `Use ${missing.map((w) => `"${w}"`).join(', ')} in the ${good} description, or change the route word in routes.yml to one it has.`
|
|
160
|
+
: `Change the route word in routes.yml to one the name or description of ${good} has.`,
|
|
118
161
|
});
|
|
119
162
|
continue;
|
|
120
163
|
}
|
|
@@ -123,7 +166,10 @@ export const route = {
|
|
|
123
166
|
if (badScore >= goodScore) {
|
|
124
167
|
findings.push({
|
|
125
168
|
tool: good,
|
|
126
|
-
message: `"${keyword}" matches ${bad} (score ${badScore}) at least as well as ${good} (score ${goodScore})
|
|
169
|
+
message: `"${keyword}" matches ${bad} (score ${badScore}) at least as well as ${good} (score ${goodScore}), so an agent could route it to the wrong tool.${why(bad)}${badScore === goodScore && !why(bad) ? ' Neither the names nor the descriptions separate them.' : ''}`,
|
|
170
|
+
fix: why(bad)
|
|
171
|
+
? `Say what ${bad} isn't for without the words "${keyword}" (name the other tool instead), or give ${good} more of them.`
|
|
172
|
+
: `Give ${good} words for "${keyword}" in its name or description that ${bad} doesn't have.`,
|
|
127
173
|
});
|
|
128
174
|
}
|
|
129
175
|
}
|
|
@@ -136,6 +182,7 @@ export const route = {
|
|
|
136
182
|
findings.push({
|
|
137
183
|
tool: bad,
|
|
138
184
|
message: `"${keyword}" matches ${bad} (score ${badScore}), which routes.yml says must not match it.${why(bad)}`,
|
|
185
|
+
fix: `Take "${keyword}" out of the name and description of ${bad}, or drop the route if the match is right after all.`,
|
|
139
186
|
});
|
|
140
187
|
}
|
|
141
188
|
}
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -23,6 +23,12 @@ export interface RuleContext {
|
|
|
23
23
|
descriptionLimit?: number | string;
|
|
24
24
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
|
25
25
|
fullDescriptions?: string[];
|
|
26
|
+
/**
|
|
27
|
+
* Set by runRules from the `ignore` setting. Rules that sum many tools up in
|
|
28
|
+
* one finding use it to leave ignored tools out of the summary; findings about
|
|
29
|
+
* one tool are dropped by runRules itself.
|
|
30
|
+
*/
|
|
31
|
+
isIgnored?: (tool: string) => boolean;
|
|
26
32
|
}
|
|
27
33
|
export type RuleFinding = Omit<Finding, 'rule' | 'severity'> & {
|
|
28
34
|
severity?: Severity;
|
|
@@ -37,3 +43,15 @@ export interface Rule {
|
|
|
37
43
|
summary: string;
|
|
38
44
|
run(ctx: RuleContext): RuleFinding[];
|
|
39
45
|
}
|
|
46
|
+
/** 2048 → "2,048". */
|
|
47
|
+
export declare function num(n: number): string;
|
|
48
|
+
/** "1 tool", "3 tools". */
|
|
49
|
+
export declare function count(n: number, one: string, many?: string): string;
|
|
50
|
+
/** "a, b and c"; the first `max` and "and N more" when there are more. */
|
|
51
|
+
export declare function names(items: string[], max?: number): string;
|
|
52
|
+
/** Text on one line, cut to `max` characters with an ellipsis. */
|
|
53
|
+
export declare function clip(text: string, max: number): string;
|
|
54
|
+
/** Tools the `ignore` setting doesn't drop, for findings that sum many tools up. */
|
|
55
|
+
export declare function kept<T extends {
|
|
56
|
+
name: string;
|
|
57
|
+
}>(tools: T[], ctx: RuleContext): T[];
|
package/dist/rules/rule.js
CHANGED
|
@@ -1 +1,25 @@
|
|
|
1
|
-
|
|
1
|
+
/** 2048 → "2,048". */
|
|
2
|
+
export function num(n) {
|
|
3
|
+
return n.toLocaleString('en-US');
|
|
4
|
+
}
|
|
5
|
+
/** "1 tool", "3 tools". */
|
|
6
|
+
export function count(n, one, many = `${one}s`) {
|
|
7
|
+
return `${num(n)} ${n === 1 ? one : many}`;
|
|
8
|
+
}
|
|
9
|
+
/** "a, b and c"; the first `max` and "and N more" when there are more. */
|
|
10
|
+
export function names(items, max = Infinity) {
|
|
11
|
+
const shown = items.slice(0, max);
|
|
12
|
+
const rest = items.length - shown.length;
|
|
13
|
+
if (rest > 0)
|
|
14
|
+
return `${shown.join(', ')} and ${num(rest)} more`;
|
|
15
|
+
return shown.length <= 1 ? shown.join('') : `${shown.slice(0, -1).join(', ')} and ${shown[shown.length - 1]}`;
|
|
16
|
+
}
|
|
17
|
+
/** Text on one line, cut to `max` characters with an ellipsis. */
|
|
18
|
+
export function clip(text, max) {
|
|
19
|
+
const flat = text.replace(/\s+/g, ' ').trim();
|
|
20
|
+
return flat.length > max ? flat.slice(0, max - 1) + '…' : flat;
|
|
21
|
+
}
|
|
22
|
+
/** Tools the `ignore` setting doesn't drop, for findings that sum many tools up. */
|
|
23
|
+
export function kept(tools, ctx) {
|
|
24
|
+
return ctx.isIgnored ? tools.filter((t) => !ctx.isIgnored(t.name)) : tools;
|
|
25
|
+
}
|
package/dist/rules/spec.js
CHANGED
|
@@ -34,20 +34,34 @@ export const schema = {
|
|
|
34
34
|
return [];
|
|
35
35
|
const validate = listToolsValidator(ctx.protocolVersion);
|
|
36
36
|
if (!validate) {
|
|
37
|
-
return [
|
|
37
|
+
return [
|
|
38
|
+
{
|
|
39
|
+
severity: 'info',
|
|
40
|
+
message: `This toolmenu has no official schema for protocol ${ctx.protocolVersion}, so the tools/list result wasn't checked against one.`,
|
|
41
|
+
fix: 'Nothing to change on the server; a newer toolmenu may bundle this version\'s schema.',
|
|
42
|
+
},
|
|
43
|
+
];
|
|
38
44
|
}
|
|
39
45
|
const findings = [];
|
|
40
46
|
if (ctx.clientError && ctx.pages.every((page) => validate(page))) {
|
|
41
|
-
findings.push({
|
|
47
|
+
findings.push({
|
|
48
|
+
message: `The official MCP SDK client rejected the tools/list result, so clients built on it list no tools. The result does match the ${ctx.protocolVersion} schema; the SDK said: ${ctx.clientError.split('\n')[0]}`,
|
|
49
|
+
fix: 'Fix what the SDK names above, then list the tools with the official client to confirm.',
|
|
50
|
+
});
|
|
42
51
|
}
|
|
43
52
|
ctx.pages.forEach((page, i) => {
|
|
44
53
|
if (validate(page))
|
|
45
54
|
return;
|
|
46
55
|
const errors = validate.errors ?? [];
|
|
56
|
+
const missing = [...new Set(errors.filter((e) => !e.instancePath && e.params && 'missingProperty' in e.params).map((e) => String(e.params.missingProperty)))];
|
|
57
|
+
const first = errors[0];
|
|
47
58
|
findings.push({
|
|
48
|
-
message: `tools/list result${ctx.pages.length > 1 ? ` (page ${i + 1})` : ''} doesn't match the ${ctx.protocolVersion} schema.`,
|
|
49
|
-
detail:
|
|
50
|
-
.concat(errors.length > 5 ? [`…and ${errors.length - 5} more`] : [])
|
|
59
|
+
message: `The tools/list result${ctx.pages.length > 1 ? ` (page ${i + 1})` : ''} doesn't match the official ${ctx.protocolVersion} schema${ctx.clientError ? ', and the official MCP SDK client rejects it, so clients built on it list no tools' : ', so a strict client may refuse it'}.`,
|
|
60
|
+
detail: errors.slice(0, 5).map((e) => `${e.instancePath || '(result)'} ${e.message}${e.params && 'missingProperty' in e.params ? `: ${e.params.missingProperty}` : ''}`)
|
|
61
|
+
.concat(errors.length > 5 ? [`…and ${errors.length - 5} more`] : []),
|
|
62
|
+
fix: missing.length
|
|
63
|
+
? `Add ${missing.join(', ')} to the tools/list result${missing.some((m) => m === 'ttlMs' || m === 'cacheScope') ? ` (${ctx.protocolVersion} requires cache hints on it, SEP-2549)` : ''}.`
|
|
64
|
+
: `Fix ${first?.instancePath || 'the result'} first (${first?.message ?? 'see above'}), then rerun.`,
|
|
51
65
|
});
|
|
52
66
|
});
|
|
53
67
|
return findings;
|
|
@@ -62,7 +76,8 @@ export const toolsCapability = {
|
|
|
62
76
|
return [];
|
|
63
77
|
return [
|
|
64
78
|
{
|
|
65
|
-
message: `The server doesn't declare the tools capability, so clients don't ask for its tools: the official MCP SDK client returns an empty list without sending tools/list${ctx.menu.tools.length ? '' : ', and that is the menu toolmenu got'}
|
|
79
|
+
message: `The server doesn't declare the tools capability, so clients don't ask for its tools: the official MCP SDK client returns an empty list without sending tools/list${ctx.menu.tools.length ? '' : ', and that is the menu toolmenu got'}.`,
|
|
80
|
+
fix: 'Declare "tools": {} in the capabilities the server sends at initialize.',
|
|
66
81
|
},
|
|
67
82
|
];
|
|
68
83
|
},
|
|
@@ -79,6 +94,7 @@ export const duplicateName = {
|
|
|
79
94
|
.map(([name, at]) => ({
|
|
80
95
|
tool: name,
|
|
81
96
|
message: `${at.length} tools are named ${name} (positions ${at.join(', ')}). A call names the tool, so only one of them can be reached; clients may keep either, or reject the whole list (Claude's API refuses duplicate tool names).`,
|
|
97
|
+
fix: `Give each ${name} its own name, or drop the copy if it's the same tool listed twice.`,
|
|
82
98
|
}));
|
|
83
99
|
},
|
|
84
100
|
};
|
|
@@ -92,6 +108,7 @@ export const discover = {
|
|
|
92
108
|
return [
|
|
93
109
|
{
|
|
94
110
|
message: `The server answered with protocol ${ctx.protocolVersion} and no working server/discover, so it isn't on 2026-07-28 yet. Rules for 2026-07-28 were skipped.`,
|
|
111
|
+
fix: 'Nothing to do on 2025; when the server moves to 2026-07-28, implement server/discover and rerun for the 2026 checks.',
|
|
95
112
|
},
|
|
96
113
|
];
|
|
97
114
|
},
|
|
@@ -103,7 +120,12 @@ export const deprecated = {
|
|
|
103
120
|
summary: 'Deprecated features still advertised',
|
|
104
121
|
run(ctx) {
|
|
105
122
|
if ('logging' in ctx.capabilities) {
|
|
106
|
-
return [
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
message: 'The server advertises the logging capability, which 2026-07-28 deprecates (SEP-2577), so clients may stop asking for its logs.',
|
|
126
|
+
fix: 'Stop advertising logging; log to stderr or through OpenTelemetry instead.',
|
|
127
|
+
},
|
|
128
|
+
];
|
|
107
129
|
}
|
|
108
130
|
return [];
|
|
109
131
|
},
|
|
@@ -118,12 +140,14 @@ export const cacheHints = {
|
|
|
118
140
|
const meta = ctx.menu.listMeta ?? {};
|
|
119
141
|
if (meta.ttlMs === 0) {
|
|
120
142
|
findings.push({
|
|
121
|
-
message: 'tools/list
|
|
143
|
+
message: 'tools/list says ttlMs: 0, so clients treat the list as stale the moment it arrives and may fetch it again before every use. Valid per spec, and the default in the official TypeScript SDK v2.',
|
|
144
|
+
fix: 'If the tool set is stable, set a positive ttlMs on tools/list (3600000 is an hour).',
|
|
122
145
|
});
|
|
123
146
|
}
|
|
124
147
|
if (meta.cacheScope === 'public' && ctx.usedAuth) {
|
|
125
148
|
findings.push({
|
|
126
|
-
message: 'tools/list
|
|
149
|
+
message: 'tools/list says cacheScope: "public" on a server that took credentials. Public lets shared caches serve the list to other callers, across authorization contexts. Fine only if the tool set never depends on who is asking.',
|
|
150
|
+
fix: 'If tools depend on the caller\'s account or scopes, set cacheScope: "private" on tools/list.',
|
|
127
151
|
});
|
|
128
152
|
}
|
|
129
153
|
return findings;
|
package/dist/rules/write.d.ts
CHANGED
package/dist/rules/write.js
CHANGED
|
@@ -1,24 +1,53 @@
|
|
|
1
1
|
import { WRITE_VERBS, nouns, verbOf, words } from '../words.js';
|
|
2
|
+
import { count, kept, names } from './rule.js';
|
|
2
3
|
const DRY_RUN_PARAMS = /^(dry[_-]?run|preview|validate[_-]?only|simulate|plan|what[_-]?if)$/i;
|
|
3
4
|
const PREVIEW_WORDS = new Set(['preview', 'validate', 'plan', 'simulate', 'dry']);
|
|
5
|
+
/** Write verbs that only add something; the others delete or overwrite. A guess from the name. */
|
|
6
|
+
const ADDITIVE_VERBS = new Set(['create', 'add', 'insert', 'send', 'post', 'publish', 'submit', 'upload', 'assign', 'invite', 'approve']);
|
|
4
7
|
function isWriteByName(tool) {
|
|
5
8
|
return WRITE_VERBS.has(verbOf(tool.name) ?? '');
|
|
6
9
|
}
|
|
7
10
|
function hasAnnotations(tool) {
|
|
8
11
|
return !!tool.annotations && Object.keys(tool.annotations).length > 0;
|
|
9
12
|
}
|
|
13
|
+
const WHY = 'Without readOnlyHint and destructiveHint, a client has to assume the worst of every write, and neither the client nor the agent can tell a harmless create from a delete when deciding what to confirm.';
|
|
10
14
|
export const unannotated = {
|
|
11
15
|
id: 'write/unannotated',
|
|
12
16
|
severity: 'warn',
|
|
13
17
|
lesson: '02',
|
|
14
18
|
summary: 'Tools that look like writes carry annotations',
|
|
15
19
|
run(ctx) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
const bare = kept(ctx.menu.tools, ctx).filter((t) => isWriteByName(t) && !hasAnnotations(t));
|
|
21
|
+
if (bare.length === 0)
|
|
22
|
+
return [];
|
|
23
|
+
const additive = (t) => ADDITIVE_VERBS.has(verbOf(t.name));
|
|
24
|
+
// One finding, not one per tool: the same gap in 53 tools is one thing to fix.
|
|
25
|
+
if (bare.length === 1) {
|
|
26
|
+
const [tool] = bare;
|
|
27
|
+
const adds = additive(tool);
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
tool: tool.name,
|
|
31
|
+
confidence: 'unsure',
|
|
32
|
+
message: `${tool.name} is named like a write ("${verbOf(tool.name)}") and has no annotations. ${WHY}`,
|
|
33
|
+
fix: `Add annotations: { readOnlyHint: false, destructiveHint: ${!adds} } to ${tool.name} (${adds ? 'true if it can also delete or overwrite' : 'false if it only adds'}), or { readOnlyHint: true } if it changes nothing.`,
|
|
34
|
+
},
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
const verbs = [...new Set(bare.map((t) => `${verbOf(t.name)}…`))];
|
|
38
|
+
const overwrites = bare.filter((t) => !additive(t)).map((t) => t.name);
|
|
39
|
+
const adds = bare.filter(additive).map((t) => t.name);
|
|
40
|
+
return [
|
|
41
|
+
{
|
|
42
|
+
confidence: 'unsure',
|
|
43
|
+
message: `${count(bare.length, 'tool')} are named like writes (${names(verbs, 6)}) and have no annotations. ${WHY}`,
|
|
44
|
+
detail: [
|
|
45
|
+
...(overwrites.length ? [`Delete or overwrite, going by the name (destructiveHint: true): ${overwrites.join(', ')}`] : []),
|
|
46
|
+
...(adds.length ? [`Only add, going by the name (destructiveHint: false): ${adds.join(', ')}`] : []),
|
|
47
|
+
],
|
|
48
|
+
fix: 'Add annotations to each: { readOnlyHint: false, destructiveHint: true } where it deletes or overwrites, { readOnlyHint: false, destructiveHint: false } where it only adds. The split above is a guess from the names: check each tool.',
|
|
49
|
+
},
|
|
50
|
+
];
|
|
22
51
|
},
|
|
23
52
|
};
|
|
24
53
|
export const noDryRun = {
|
|
@@ -28,7 +57,7 @@ export const noDryRun = {
|
|
|
28
57
|
summary: 'Destructive tools offer a way to look before leaping',
|
|
29
58
|
run(ctx) {
|
|
30
59
|
const missing = [];
|
|
31
|
-
for (const tool of ctx.menu.tools) {
|
|
60
|
+
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
32
61
|
const a = tool.annotations ?? {};
|
|
33
62
|
const destructive = a.destructiveHint === true || (a.readOnlyHint === false && a.destructiveHint !== false);
|
|
34
63
|
if (!destructive)
|
|
@@ -43,12 +72,15 @@ export const noDryRun = {
|
|
|
43
72
|
}
|
|
44
73
|
if (missing.length === 0)
|
|
45
74
|
return [];
|
|
75
|
+
const one = missing.length === 1;
|
|
46
76
|
// One finding, not one per tool: it's a design note, not a list of defects.
|
|
47
77
|
return [
|
|
48
78
|
{
|
|
49
|
-
...(
|
|
50
|
-
|
|
51
|
-
|
|
79
|
+
...(one ? { tool: missing[0] } : {}),
|
|
80
|
+
confidence: 'unsure',
|
|
81
|
+
message: `${one ? `${missing[0]} is` : `${count(missing.length, 'tool')} are`} annotated as destructive, and toolmenu found no dry_run/preview-style parameter and no preview tool for the same thing. Where a mistake is costly to undo, a dry run lets the agent see what would change while it can still back out.`,
|
|
82
|
+
...(one ? {} : { detail: [missing.join(', ')] }),
|
|
83
|
+
fix: `Where the effect is hard to undo, add a dry_run parameter${one ? ` to ${missing[0]}` : ''} that returns what would change without changing it. If it's cheap to undo, or a preview exists under another name, ignore this.`,
|
|
52
84
|
},
|
|
53
85
|
];
|
|
54
86
|
},
|
package/dist/session.d.ts
CHANGED
|
@@ -2,12 +2,18 @@ import { type ToolChange } from './compare.js';
|
|
|
2
2
|
import { type Target } from './connect.js';
|
|
3
3
|
import { type FailureClass } from './failures.js';
|
|
4
4
|
import type { Era, Finding, Menu, MenuTool, Severity } from './types.js';
|
|
5
|
+
import { type AutoSummary } from './auto.js';
|
|
5
6
|
export type Step = {
|
|
6
7
|
kind: 'list';
|
|
7
8
|
} | {
|
|
8
9
|
kind: 'call';
|
|
9
10
|
tool: string;
|
|
10
11
|
args: Record<string, unknown>;
|
|
12
|
+
/**
|
|
13
|
+
* --auto only, never from a scenario file: an unlock guessed from its schema
|
|
14
|
+
* alone. If its first such call changes nothing, the rest aren't made.
|
|
15
|
+
*/
|
|
16
|
+
tentative?: boolean;
|
|
11
17
|
} | {
|
|
12
18
|
kind: 'wait_for';
|
|
13
19
|
event: 'tools_list_changed';
|
|
@@ -16,6 +22,12 @@ export type Step = {
|
|
|
16
22
|
export interface Scenario {
|
|
17
23
|
allowWrites: boolean;
|
|
18
24
|
steps: Step[];
|
|
25
|
+
/**
|
|
26
|
+
* Tools the user says only read, although the server doesn't mark them
|
|
27
|
+
* readOnlyHint (`assume_read_only`, --assume-read-only). Called without
|
|
28
|
+
* allow_writes; a tool marked destructiveHint or readOnlyHint: false never is.
|
|
29
|
+
*/
|
|
30
|
+
assumeReadOnly?: string[];
|
|
19
31
|
}
|
|
20
32
|
/** Read and check a scenario file. Mistakes are reported before anything runs. */
|
|
21
33
|
export declare function loadScenario(path: string): Promise<Scenario>;
|
|
@@ -60,10 +72,7 @@ export interface SessionResult {
|
|
|
60
72
|
*/
|
|
61
73
|
union: Menu;
|
|
62
74
|
/** With --auto: which tools were called, and which weren't and why. */
|
|
63
|
-
auto?:
|
|
64
|
-
called: string[];
|
|
65
|
-
skipped: import('./auto.js').AutoPlan['skipped'];
|
|
66
|
-
};
|
|
75
|
+
auto?: AutoSummary;
|
|
67
76
|
}
|
|
68
77
|
export interface SessionOptions {
|
|
69
78
|
timeoutMs?: number;
|
|
@@ -77,15 +86,14 @@ export interface SessionOptions {
|
|
|
77
86
|
/** The union menu is kept as a baseline (--union-out): an unlock never called then leaves tools out of it. */
|
|
78
87
|
unionOut?: boolean;
|
|
79
88
|
/** session --auto: what the plan called and skipped, so a run that called nothing isn't a clean one. */
|
|
80
|
-
auto?:
|
|
81
|
-
called: string[];
|
|
82
|
-
skipped: import('./auto.js').AutoPlan['skipped'];
|
|
83
|
-
};
|
|
89
|
+
auto?: AutoSummary;
|
|
84
90
|
/** Waits before retrying a request refused for being too many (default 2, 4, 8, 16, 32 s: a per-minute limit clears within them). */
|
|
85
91
|
rateLimitWaitsMs?: number[];
|
|
86
92
|
}
|
|
93
|
+
/** `each`: this step's line when findings of one rule are said once for several steps. */
|
|
87
94
|
type Raw = Omit<Finding, 'severity'> & {
|
|
88
95
|
severity: Severity;
|
|
96
|
+
each?: string;
|
|
89
97
|
};
|
|
90
98
|
/** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
|
|
91
99
|
export declare function session(target: Target, scenario: Scenario, options?: SessionOptions): Promise<SessionResult>;
|
|
@@ -107,6 +115,8 @@ interface Unlocker {
|
|
|
107
115
|
param?: string;
|
|
108
116
|
/** Values to try for it, from the schema's enum. */
|
|
109
117
|
values: unknown[];
|
|
118
|
+
/** Its name or description says it unlocks tools (for this parameter); otherwise a guess from its schema. */
|
|
119
|
+
claims: boolean;
|
|
110
120
|
}
|
|
111
121
|
/**
|
|
112
122
|
* At most `max` characters, cut by code point at a word boundary, never inside a
|