toolmenu 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -20
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +53 -17
- package/dist/auto.d.ts +70 -6
- package/dist/auto.js +248 -17
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +62 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +80 -170
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +526 -114
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +9 -0
- package/dist/failures.js +17 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +92 -2
- package/dist/init.js +507 -20
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +85 -42
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +145 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +12 -4
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.js +33 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/session.d.ts +18 -8
- package/dist/session.js +484 -92
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +17 -2
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { clip, kept, num } from './rule.js';
|
|
1
2
|
/*
|
|
2
3
|
* Clients that cut tool descriptions, and where. Only reported cuts, with sources:
|
|
3
4
|
* - claude-code: 2,048 characters, "… [truncated]" appended, silently (anthropics/claude-code#87650);
|
|
@@ -8,14 +9,20 @@
|
|
|
8
9
|
*/
|
|
9
10
|
export const CLIENT_LIMITS = { 'claude-code': 2048, 'amazon-q': 10024 };
|
|
10
11
|
export const DEFAULT_CLIENT = 'claude-code';
|
|
11
|
-
/**
|
|
12
|
+
/**
|
|
13
|
+
* The cut in characters, the client to name in a message ("Claude Code", "your
|
|
14
|
+
* client"), and whether it's toolmenu's assumption rather than the user's setting.
|
|
15
|
+
*/
|
|
12
16
|
export function resolveLimit(setting) {
|
|
13
17
|
if (typeof setting === 'number')
|
|
14
|
-
return { limit: setting, who:
|
|
18
|
+
return { limit: setting, who: 'your client', assumed: false };
|
|
15
19
|
const client = setting ?? DEFAULT_CLIENT;
|
|
16
20
|
const limit = CLIENT_LIMITS[client] ?? CLIENT_LIMITS[DEFAULT_CLIENT];
|
|
17
|
-
|
|
18
|
-
|
|
21
|
+
return { limit, who: client === 'amazon-q' ? 'Amazon Q CLI' : 'Claude Code', assumed: setting === undefined };
|
|
22
|
+
}
|
|
23
|
+
/** The fix's last words when the cut is assumed, not set. */
|
|
24
|
+
function unlessOtherClient(assumed) {
|
|
25
|
+
return assumed ? ' If your client isn\'t Claude Code, set descriptionLimit to its cut.' : '';
|
|
19
26
|
}
|
|
20
27
|
/*
|
|
21
28
|
* An instruction tells the agent what to do or not do: "use X", "call this tool",
|
|
@@ -93,27 +100,28 @@ export const buried = {
|
|
|
93
100
|
lesson: 'truncated description',
|
|
94
101
|
summary: 'Instructions to the agent sit past the point where your client cuts descriptions',
|
|
95
102
|
run(ctx) {
|
|
96
|
-
const { limit, who } = resolveLimit(ctx.descriptionLimit);
|
|
97
|
-
const
|
|
103
|
+
const { limit, who, assumed } = resolveLimit(ctx.descriptionLimit);
|
|
104
|
+
const all = ctx.menu.tools.map((t) => t.name);
|
|
98
105
|
const findings = [];
|
|
99
106
|
for (const tool of ctx.menu.tools) {
|
|
100
107
|
const text = tool.description ?? '';
|
|
101
108
|
if (text.length <= limit || sentInFull(tool, ctx.fullDescriptions))
|
|
102
109
|
continue;
|
|
103
|
-
const past = pastTheCut(text,
|
|
110
|
+
const past = pastTheCut(text, all.filter((n) => n !== tool.name), limit);
|
|
104
111
|
if (past.length === 0)
|
|
105
112
|
continue;
|
|
106
113
|
const one = past.length === 1;
|
|
114
|
+
const first = past[0];
|
|
107
115
|
findings.push({
|
|
108
116
|
tool: tool.name,
|
|
109
|
-
|
|
117
|
+
// Which sentences are instructions is read from the words: a guess.
|
|
118
|
+
confidence: 'unsure',
|
|
119
|
+
message: `${tool.name}: ${one ? 'a sentence that reads as an instruction to the agent sits' : `${past.length} sentences that read as instructions to the agent sit`} past character ${num(limit)}, where ${who} cuts the description, so the model never sees ${one ? 'it' : 'them'}. The description is ${num(text.length)} characters; the ${one ? '' : 'first '}instruction ${first.straddles ? `starts at ${num(first.at)} and is cut in the middle` : `is at ${num(first.at)}`}.`,
|
|
110
120
|
detail: past
|
|
111
|
-
.slice(0,
|
|
112
|
-
.map((i) => {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
})
|
|
116
|
-
.concat(past.length > 3 ? [`…and ${past.length - 3} more`] : []),
|
|
121
|
+
.slice(0, 5)
|
|
122
|
+
.map((i) => `char ${num(i.at)}${i.straddles ? ' (cut mid-instruction)' : ''}: “${clip(sentenceAround(text, i.at), 200)}”`)
|
|
123
|
+
.concat(past.length > 5 ? [`…and ${past.length - 5} more`] : []),
|
|
124
|
+
fix: `Move “${clip(sentenceAround(text, first.at), 70)}”${one ? '' : ' and the rest above'} into the first ${num(limit)} characters of the ${tool.name} description, or list ${tool.name} in fullDescriptions if your client sends it uncut.${unlessOtherClient(assumed)}`,
|
|
117
125
|
});
|
|
118
126
|
}
|
|
119
127
|
return findings;
|
|
@@ -121,41 +129,76 @@ export const buried = {
|
|
|
121
129
|
};
|
|
122
130
|
/** A short cut to check against when the client's own cut isn't set: one real client cut at 280. */
|
|
123
131
|
const SHORT_CUT = 280;
|
|
132
|
+
/** “…the last words before ✂ the first words after…”, to find the cut in the text. */
|
|
133
|
+
function aroundTheCut(text, at) {
|
|
134
|
+
const before = text.slice(Math.max(0, at - 30), at).replace(/\s+/g, ' ');
|
|
135
|
+
const after = text.slice(at, at + 30).replace(/\s+/g, ' ');
|
|
136
|
+
return `“…${before}✂${after}…”`;
|
|
137
|
+
}
|
|
124
138
|
export const cut = {
|
|
125
139
|
id: 'description/cut',
|
|
126
140
|
severity: 'info',
|
|
127
141
|
lesson: 'truncated description',
|
|
128
|
-
summary: 'Descriptions longer than your client sends
|
|
142
|
+
summary: 'Descriptions longer than your client sends',
|
|
129
143
|
run(ctx) {
|
|
130
|
-
const { limit, who } = resolveLimit(ctx.descriptionLimit);
|
|
131
|
-
const
|
|
132
|
-
const
|
|
133
|
-
const tools = ctx.menu.tools.filter((t) => !sentInFull(t, ctx.fullDescriptions));
|
|
134
|
-
const findings = [];
|
|
135
|
-
const long = tools.filter((tool) => {
|
|
144
|
+
const { limit, who, assumed } = resolveLimit(ctx.descriptionLimit);
|
|
145
|
+
const all = ctx.menu.tools.map((t) => t.name);
|
|
146
|
+
const long = kept(ctx.menu.tools, ctx).filter((tool) => {
|
|
136
147
|
const text = tool.description ?? '';
|
|
137
|
-
return text.length > limit && pastTheCut(text,
|
|
148
|
+
return !sentInFull(tool, ctx.fullDescriptions) && text.length > limit && pastTheCut(text, all.filter((n) => n !== tool.name), limit).length === 0;
|
|
138
149
|
});
|
|
139
|
-
if (long.length)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
150
|
+
if (long.length === 0)
|
|
151
|
+
return [];
|
|
152
|
+
const one = long.length === 1;
|
|
153
|
+
// One summary: a long description is a question for its author, not a defect.
|
|
154
|
+
return [
|
|
155
|
+
{
|
|
156
|
+
...(one ? { tool: long[0].name } : {}),
|
|
157
|
+
message: `${one ? `The ${long[0].name} description is` : `${long.length} descriptions are`} longer than the ${num(limit)} characters ${who} sends. The model gets the start, with no sign that anything is missing, so a cut list or example can read as complete. Nothing past the cut reads as an instruction to the agent, so this is a check, not a bug.`,
|
|
158
|
+
detail: long.map((t) => {
|
|
159
|
+
const text = t.description ?? '';
|
|
160
|
+
return `${t.name}: ${num(text.length)} characters, ${num(text.length - limit)} past the cut, which falls at ${aroundTheCut(text, limit)}`;
|
|
161
|
+
}),
|
|
162
|
+
fix: `Check that what's past the cut ${one ? 'in' : 'in each of'} ${one ? long[0].name : 'these'} can be lost, and move anything the agent needs into the first ${num(limit)} characters.${unlessOtherClient(assumed)}`,
|
|
163
|
+
},
|
|
164
|
+
];
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
export const lateInstruction = {
|
|
168
|
+
id: 'description/late-instruction',
|
|
169
|
+
severity: 'info',
|
|
170
|
+
lesson: 'truncated description',
|
|
171
|
+
summary: 'Instructions to the agent that a client cutting descriptions short would hide',
|
|
172
|
+
run(ctx) {
|
|
173
|
+
// Only with the default cut: once descriptionLimit is set, description/buried checks the real one.
|
|
174
|
+
if (ctx.descriptionLimit !== undefined)
|
|
175
|
+
return [];
|
|
176
|
+
const { limit, who } = resolveLimit(undefined);
|
|
177
|
+
if (limit <= SHORT_CUT)
|
|
178
|
+
return [];
|
|
179
|
+
const all = ctx.menu.tools.map((t) => t.name);
|
|
180
|
+
const late = [];
|
|
181
|
+
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
182
|
+
const text = tool.description ?? '';
|
|
183
|
+
if (text.length <= SHORT_CUT || sentInFull(tool, ctx.fullDescriptions))
|
|
184
|
+
continue;
|
|
185
|
+
// Past the default cut is description/buried's.
|
|
186
|
+
const hidden = pastTheCut(text, all.filter((n) => n !== tool.name), SHORT_CUT).filter((i) => i.at < limit);
|
|
187
|
+
if (hidden.length)
|
|
188
|
+
late.push({ tool, at: hidden[0].at, sentence: sentenceAround(text, hidden[0].at), more: hidden.length - 1 });
|
|
155
189
|
}
|
|
156
|
-
|
|
190
|
+
if (late.length === 0)
|
|
191
|
+
return [];
|
|
192
|
+
const one = late.length === 1;
|
|
193
|
+
const client = who;
|
|
194
|
+
return [
|
|
195
|
+
{
|
|
196
|
+
...(one ? { tool: late[0].tool.name } : {}),
|
|
197
|
+
confidence: 'unsure',
|
|
198
|
+
message: `${one ? `The ${late[0].tool.name} description gives` : `${late.length} descriptions give`} the agent instructions after character ${SHORT_CUT}, where a client that cuts descriptions short never shows them (one real client cut at ${SHORT_CUT}). ${client} sends ${num(limit)} characters, so there they arrive.`,
|
|
199
|
+
detail: late.map((l) => `${l.tool.name}: char ${num(l.at)}: “${clip(l.sentence, 120)}”${l.more ? ` (and ${l.more} more after it)` : ''}`),
|
|
200
|
+
fix: `If your client cuts descriptions, set descriptionLimit to its cut and toolmenu checks against that. Otherwise, moving ${one ? 'the instruction' : 'each instruction'} into the first sentences is cheap insurance.`,
|
|
201
|
+
},
|
|
202
|
+
];
|
|
157
203
|
},
|
|
158
204
|
};
|
|
159
|
-
function list(tools, show) {
|
|
160
|
-
return tools.slice(0, 5).map(show).join(', ') + (tools.length > 5 ? `, and ${tools.length - 5} more` : '');
|
|
161
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { MenuTool, Severity } from '../types.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type Rule, type RuleFinding } from './rule.js';
|
|
3
3
|
export declare const nondeterministic: Rule;
|
|
4
4
|
/** What a second process or connection served, for the variance rules. */
|
|
5
5
|
export interface Probe {
|
|
@@ -16,6 +16,8 @@ export declare function varianceFinding(main: MenuTool[], other: MenuTool[], opt
|
|
|
16
16
|
transport: 'stdio' | 'http';
|
|
17
17
|
modern: boolean;
|
|
18
18
|
wrapper: boolean;
|
|
19
|
+
/** Places menu/nondeterministic already reports: said once, and left out of the explanation and fix. */
|
|
20
|
+
explained?: Set<string>;
|
|
19
21
|
}): (RuleFinding & {
|
|
20
22
|
rule: string;
|
|
21
23
|
severity: Severity;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { compareMenus } from '../compare.js';
|
|
2
|
-
import { describeToolDifference } from '../difference.js';
|
|
2
|
+
import { allDifferences, describeToolDifference, firstDifference } from '../difference.js';
|
|
3
3
|
import { serverWords } from '../failures.js';
|
|
4
|
+
import { toolDefinition } from '../menu.js';
|
|
5
|
+
import { count } from './rule.js';
|
|
4
6
|
const LABELS = {
|
|
5
7
|
added: 'tool added',
|
|
6
8
|
removed: 'tool removed',
|
|
@@ -12,6 +14,74 @@ const LABELS = {
|
|
|
12
14
|
other: 'definition changed',
|
|
13
15
|
serialization: 'key order changed (same content, different bytes)',
|
|
14
16
|
};
|
|
17
|
+
const EDITS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
18
|
+
const TIMESTAMP = /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
19
|
+
/** Where each edited tool first differs between two menus. */
|
|
20
|
+
function spots(before, after, changes) {
|
|
21
|
+
const byName = new Map(before.map((t) => [t.name, t]));
|
|
22
|
+
const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const t of after.filter((x) => edited.has(x.name))) {
|
|
25
|
+
const d = firstDifference(toolDefinition(byName.get(t.name)), toolDefinition(t));
|
|
26
|
+
const path = (d?.path ?? '').replace(/^inputSchema\.properties\./, '');
|
|
27
|
+
out.push({ tool: t.name, where: path ? `${t.name}.${path}` : t.name, before: d?.before, after: d?.after, reordered: !!d?.reordered });
|
|
28
|
+
}
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
/** Every value that differs inside each edited tool, not only the first. */
|
|
32
|
+
function everySpot(before, after, changes) {
|
|
33
|
+
const byName = new Map(before.map((t) => [t.name, t]));
|
|
34
|
+
const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
|
|
35
|
+
return after
|
|
36
|
+
.filter((x) => edited.has(x.name))
|
|
37
|
+
.flatMap((t) => allDifferences(toolDefinition(byName.get(t.name)), toolDefinition(t)).map((d) => {
|
|
38
|
+
const path = d.path.replace(/^inputSchema\.properties\./, '');
|
|
39
|
+
return { tool: t.name, where: path ? `${t.name}.${path}` : t.name, before: d.before, after: d.after, reordered: d.reordered };
|
|
40
|
+
}));
|
|
41
|
+
}
|
|
42
|
+
/** Every place two menus differ: whole-tool changes, and each differing value. */
|
|
43
|
+
function places(before, after) {
|
|
44
|
+
const changes = compareMenus(before, after);
|
|
45
|
+
return [...changes.filter((c) => !EDITS.has(c.kind)).map((c) => `${c.kind}:${c.tool}`), ...everySpot(before, after, changes).map((s) => s.where)];
|
|
46
|
+
}
|
|
47
|
+
const isTime = (s) => TIMESTAMP.test(String(s.before)) && TIMESTAMP.test(String(s.after));
|
|
48
|
+
/** "list_transactions.end_date.default" or "a.b, c.d and 3 more places". */
|
|
49
|
+
function placesOf(found) {
|
|
50
|
+
const shown = found.slice(0, 2).map((s) => s.where).join(', ');
|
|
51
|
+
return found.length > 2 ? `${shown} and ${found.length - 2} more ${found.length - 2 === 1 ? 'place' : 'places'}` : shown;
|
|
52
|
+
}
|
|
53
|
+
/** What differs, in words, after the first sentence of the message. */
|
|
54
|
+
function whatDiffers(changes, found) {
|
|
55
|
+
const time = found.filter(isTime);
|
|
56
|
+
if (time.length)
|
|
57
|
+
return `${placesOf(time)} ${time.length === 1 ? 'is a timestamp' : 'are timestamps'}, taken when the menu is built.`;
|
|
58
|
+
const sorted = found.filter((s) => s.reordered);
|
|
59
|
+
if (sorted.length)
|
|
60
|
+
return `${placesOf(sorted)} ${sorted.length === 1 ? 'holds' : 'hold'} the same items in a different order.`;
|
|
61
|
+
if (found.length)
|
|
62
|
+
return `It differs in ${placesOf(found)}.`;
|
|
63
|
+
const tools = [...new Set(changes.filter((c) => c.kind === 'added' || c.kind === 'removed').map((c) => c.tool))];
|
|
64
|
+
if (tools.length)
|
|
65
|
+
return `The tool set itself changed: ${tools.slice(0, 3).join(', ')}${tools.length > 3 ? ` and ${tools.length - 3} more` : ''}.`;
|
|
66
|
+
return '';
|
|
67
|
+
}
|
|
68
|
+
/** One step for the server author, for the most telling difference. */
|
|
69
|
+
function fixFor(changes, found, who) {
|
|
70
|
+
const time = found.find(isTime);
|
|
71
|
+
if (time)
|
|
72
|
+
return `Build ${time.where} from a fixed value, not the current time: leave it out, or say "defaults to now" in the description.`;
|
|
73
|
+
const sorted = found.filter((s) => s.reordered);
|
|
74
|
+
if (sorted.length)
|
|
75
|
+
return `Sort the items of ${sorted[0].where}${sorted.length > 1 ? ` (and the ${count(sorted.length - 1, 'other place')} listed)` : ''} when building the menu (sorted(), not a set or a map), so every ${who} lists them in the same order.`;
|
|
76
|
+
if (changes.some((c) => c.kind === 'added' || c.kind === 'removed')) {
|
|
77
|
+
return 'Serve the same tools on every list; if the set really depends on state, send notifications/tools/list_changed when it changes.';
|
|
78
|
+
}
|
|
79
|
+
if (changes.every((c) => c.kind === 'moved'))
|
|
80
|
+
return 'Return the tools in a fixed order: keep them in a list, or sort them by name before answering tools/list.';
|
|
81
|
+
if (changes.every((c) => c.kind === 'serialization' || c.kind === 'moved'))
|
|
82
|
+
return 'Build each tool object with its keys in a fixed order, so the bytes match as well as the content.';
|
|
83
|
+
return `Build ${found[0]?.where ?? 'the menu'} from fixed values: nothing from the clock, a random ID, a counter or the environment.`;
|
|
84
|
+
}
|
|
15
85
|
export const nondeterministic = {
|
|
16
86
|
id: 'menu/nondeterministic',
|
|
17
87
|
severity: 'error',
|
|
@@ -24,12 +94,16 @@ export const nondeterministic = {
|
|
|
24
94
|
if (changes.length === 0)
|
|
25
95
|
return [];
|
|
26
96
|
const onlyOrder = changes.every((c) => c.kind === 'moved');
|
|
97
|
+
const found = spots(ctx.menu.tools, ctx.secondList, changes);
|
|
98
|
+
const tools = [...new Set(changes.map((c) => c.tool))];
|
|
27
99
|
return [
|
|
28
100
|
{
|
|
101
|
+
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
29
102
|
message: onlyOrder
|
|
30
|
-
? 'Two identical tools/list calls returned the tools in a different order
|
|
31
|
-
:
|
|
103
|
+
? 'Two identical tools/list calls, back to back, returned the tools in a different order, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. The spec says servers SHOULD keep the order stable.'
|
|
104
|
+
: `Two identical tools/list calls, back to back, returned different menus, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. ${whatDiffers(changes, found)}`.trim(),
|
|
32
105
|
detail: detailFor(ctx.menu.tools, ctx.secondList, changes),
|
|
106
|
+
fix: fixFor(changes, found, 'call'),
|
|
33
107
|
},
|
|
34
108
|
];
|
|
35
109
|
},
|
|
@@ -53,15 +127,33 @@ export function varianceFinding(main, other, opts) {
|
|
|
53
127
|
const changes = compareMenus(main, other);
|
|
54
128
|
if (changes.length === 0)
|
|
55
129
|
return undefined;
|
|
56
|
-
const detail = detailFor(main, other, changes);
|
|
57
|
-
const reordered = detail.some((d) => d.includes('same ') && d.includes('different order'));
|
|
58
130
|
const stdio = opts.transport === 'stdio';
|
|
131
|
+
let why = changes;
|
|
132
|
+
let found = spots(main, other, changes);
|
|
133
|
+
let detail = detailFor(main, other, changes);
|
|
134
|
+
let lead = '';
|
|
135
|
+
if (opts.explained) {
|
|
136
|
+
const known = opts.explained;
|
|
137
|
+
const all = everySpot(main, other, changes);
|
|
138
|
+
const same = all.filter((s) => known.has(s.where));
|
|
139
|
+
found = all.filter((s) => !known.has(s.where));
|
|
140
|
+
why = changes.filter((c) => (EDITS.has(c.kind) ? found.some((s) => s.tool === c.tool) : !known.has(`${c.kind}:${c.tool}`)));
|
|
141
|
+
lead = `${same.length ? `${placesOf(same)} ${same.length === 1 ? 'is' : 'are'}` : 'Part of it is'} what menu/nondeterministic reports, and goes away with that fix. The rest differs only between ${stdio ? 'processes' : 'connections'}, so it has a cause of its own:`;
|
|
142
|
+
const lines = [
|
|
143
|
+
...why.filter((c) => !EDITS.has(c.kind)).map((c) => `${LABELS[c.kind]}: ${c.tool} (position ${c.position})`),
|
|
144
|
+
...found.map((f) => `${f.where}: ${f.reordered ? 'same items, different order' : 'differs'} (${short(f.before)} vs ${short(f.after)})`),
|
|
145
|
+
];
|
|
146
|
+
detail = lines.slice(0, 10).concat(lines.length > 10 ? [`…and ${lines.length - 10} more`] : []);
|
|
147
|
+
}
|
|
148
|
+
const reordered = found.some((s) => s.reordered);
|
|
59
149
|
const message = [
|
|
60
150
|
stdio
|
|
61
|
-
? 'A second server process, started the same way, served a different menu
|
|
62
|
-
: `A second connection with the same credentials got a different menu.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per
|
|
151
|
+
? 'A second server process, started the same way, served a different menu, so every restart, and every client that starts its own copy, gets a tool list that no cached prompt matches.'
|
|
152
|
+
: `A second connection with the same credentials got a different menu, so clients can't share a cached tool list.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per connection.' : ''}`,
|
|
153
|
+
lead,
|
|
154
|
+
whatDiffers(why, found),
|
|
63
155
|
reordered
|
|
64
|
-
? '
|
|
156
|
+
? 'Likely a set or a map iterated in hash order.' +
|
|
65
157
|
(stdio ? ' (toolmenu runs the first process with PYTHONHASHSEED=0 and the second with 1, so a Python set shows up every time; Go and Rust maps vary per process on their own.)' : '')
|
|
66
158
|
: '',
|
|
67
159
|
opts.wrapper
|
|
@@ -73,10 +165,27 @@ export function varianceFinding(main, other, opts) {
|
|
|
73
165
|
severity: stdio || opts.modern ? 'error' : 'warn',
|
|
74
166
|
message,
|
|
75
167
|
detail,
|
|
168
|
+
fix: fixFor(why, found, stdio ? 'process' : 'connection'),
|
|
76
169
|
};
|
|
77
170
|
}
|
|
78
|
-
|
|
171
|
+
/**
|
|
172
|
+
* The places menu/nondeterministic reports, when the second list differs too:
|
|
173
|
+
* a probe that differs only there has the same cause, and says so instead of
|
|
174
|
+
* repeating it.
|
|
175
|
+
*/
|
|
176
|
+
function listSpots(ctx) {
|
|
177
|
+
if (!ctx.secondList)
|
|
178
|
+
return undefined;
|
|
179
|
+
const found = places(ctx.menu.tools, ctx.secondList);
|
|
180
|
+
return found.length ? new Set(found) : undefined;
|
|
181
|
+
}
|
|
182
|
+
/** A value for a detail line, cut short. */
|
|
183
|
+
function short(v) {
|
|
184
|
+
const s = JSON.stringify(v) ?? String(v);
|
|
185
|
+
return s.length > 60 ? `${s.slice(0, 57)}…` : s;
|
|
186
|
+
}
|
|
79
187
|
function varianceRule(id, transport) {
|
|
188
|
+
const other = transport === 'stdio' ? 'server process' : 'connection';
|
|
80
189
|
return {
|
|
81
190
|
id,
|
|
82
191
|
severity: 'error',
|
|
@@ -88,13 +197,38 @@ function varianceRule(id, transport) {
|
|
|
88
197
|
const out = [];
|
|
89
198
|
for (const probe of ctx.probes) {
|
|
90
199
|
if (probe.error) {
|
|
91
|
-
out.push({
|
|
200
|
+
out.push({
|
|
201
|
+
severity: 'info',
|
|
202
|
+
message: `Couldn't ${transport === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
|
|
203
|
+
fix: transport === 'stdio'
|
|
204
|
+
? 'If the server can only run once at a time (a lock, a port), rerun with --processes 1 to skip this check.'
|
|
205
|
+
: 'If the server allows one connection at a time, rerun with --processes 1 to skip this check.',
|
|
206
|
+
});
|
|
92
207
|
continue;
|
|
93
208
|
}
|
|
94
209
|
const f = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper });
|
|
95
210
|
if (f) {
|
|
96
211
|
const { rule: _r, ...rest } = f;
|
|
97
|
-
|
|
212
|
+
// Same places as the second tools/list call: one cause, reported by menu/nondeterministic.
|
|
213
|
+
// Every differing value is compared, not only the first in each tool: a
|
|
214
|
+
// timestamp that changes every call can hide a set that changes per process.
|
|
215
|
+
const same = listSpots(ctx);
|
|
216
|
+
const here = places(ctx.menu.tools, probe.tools ?? []);
|
|
217
|
+
if (same && here.every((w) => same.has(w))) {
|
|
218
|
+
out.push({
|
|
219
|
+
...rest,
|
|
220
|
+
message: `A second ${other} served a different menu too, in the same ${here.length === 1 ? 'place' : 'places'} menu/nondeterministic reports: a menu that changes on every call changes on every ${transport === 'stdio' ? 'restart' : 'connection'} as well. Same cause, same fix.`,
|
|
221
|
+
fix: 'Fix menu/nondeterministic; this goes away with it.',
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
else if (same && here.some((w) => same.has(w))) {
|
|
225
|
+
// Partly the same cause: say that once, and explain and fix the rest.
|
|
226
|
+
const { rule: _r2, ...partial } = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper, explained: same });
|
|
227
|
+
out.push(partial);
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
out.push(rest);
|
|
231
|
+
}
|
|
98
232
|
break; // one finding: a second differing probe says nothing new
|
|
99
233
|
}
|
|
100
234
|
}
|
package/dist/rules/ids.d.ts
CHANGED
package/dist/rules/ids.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, commonWords, nouns, singular, words } from '../words.js';
|
|
1
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, article, commonWords, nouns, singular, words } from '../words.js';
|
|
2
|
+
import { kept } from './rule.js';
|
|
2
3
|
// form_id, form-id, formId, formID. Not uid or paid: the suffix needs a boundary.
|
|
3
4
|
// Not "key" either: in real menus it's as often a keyboard key as an identifier.
|
|
4
5
|
const ID_SUFFIX = /^(.+?)[_-](id|ids|uuid|uuids)$/i;
|
|
@@ -121,7 +122,7 @@ export const authoredIds = {
|
|
|
121
122
|
run(ctx) {
|
|
122
123
|
const byKind = new Map();
|
|
123
124
|
const common = commonWords(ctx.menu.tools.map((t) => t.name));
|
|
124
|
-
for (const tool of ctx.menu.tools) {
|
|
125
|
+
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
125
126
|
for (const { param, kind } of idParams(tool)) {
|
|
126
127
|
if (!kind || isReturnedBySomeTool(kind, tool, ctx.menu.tools, common))
|
|
127
128
|
continue;
|
|
@@ -139,10 +140,14 @@ export const authoredIds = {
|
|
|
139
140
|
return [...byKind].map(([kind, uses]) => {
|
|
140
141
|
const tools = [...new Set(uses.map((u) => u.tool))];
|
|
141
142
|
const where = uses.map((u) => `${u.tool}.${u.param}`);
|
|
143
|
+
const one = where.length === 1;
|
|
144
|
+
const a = article(kind);
|
|
142
145
|
return {
|
|
143
146
|
...(tools.length === 1 ? { tool: tools[0] } : {}),
|
|
144
|
-
|
|
145
|
-
|
|
147
|
+
confidence: 'unsure',
|
|
148
|
+
message: `${one ? `${where[0]} takes` : `${where.length} parameters take`} ${a} ${kind} ID, and no tool in the menu appears to return one: no list, search, get or create tool for "${kind}", and no output schema with ${a} ${kind} ID. If the agent can't find one anywhere, it has to guess.`,
|
|
149
|
+
...(one ? {} : { detail: [where.join(', ')] }),
|
|
150
|
+
fix: `If ${kind} IDs come from somewhere the agent can see (a URL the user pastes, another tool's text output), say so in the parameter's description; if not, add a tool that lists or finds them.`,
|
|
146
151
|
};
|
|
147
152
|
});
|
|
148
153
|
},
|
package/dist/rules/index.d.ts
CHANGED
|
@@ -2,6 +2,12 @@ import type { Finding, Severity } from '../types.js';
|
|
|
2
2
|
import type { Rule, RuleContext } from './rule.js';
|
|
3
3
|
export type { Rule, RuleContext } from './rule.js';
|
|
4
4
|
export declare const MENU_RULES: Rule[];
|
|
5
|
+
/**
|
|
6
|
+
* Rules split out of an older one. A setting for the old id still applies to the
|
|
7
|
+
* new one unless the new id is set itself: before 0.13, description/cut held
|
|
8
|
+
* the late-instruction hint too, and `"description/cut": "off"` turned both off.
|
|
9
|
+
*/
|
|
10
|
+
export declare const RULE_ALIASES: Record<string, string>;
|
|
5
11
|
export interface RuleSettings {
|
|
6
12
|
/** Per-rule severity override, or 'off'. */
|
|
7
13
|
rules?: Record<string, Severity | 'off'>;
|
package/dist/rules/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SEVERITY_RANK } from '../types.js';
|
|
2
|
-
import { buried, cut } from './description.js';
|
|
2
|
+
import { buried, cut, lateInstruction } from './description.js';
|
|
3
3
|
import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
5
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
@@ -16,6 +16,7 @@ export const MENU_RULES = [
|
|
|
16
16
|
authoredIds,
|
|
17
17
|
buried,
|
|
18
18
|
cut,
|
|
19
|
+
lateInstruction,
|
|
19
20
|
noDryRun,
|
|
20
21
|
unannotated,
|
|
21
22
|
schema,
|
|
@@ -24,17 +25,24 @@ export const MENU_RULES = [
|
|
|
24
25
|
deprecated,
|
|
25
26
|
cacheHints,
|
|
26
27
|
];
|
|
28
|
+
/**
|
|
29
|
+
* Rules split out of an older one. A setting for the old id still applies to the
|
|
30
|
+
* new one unless the new id is set itself: before 0.13, description/cut held
|
|
31
|
+
* the late-instruction hint too, and `"description/cut": "off"` turned both off.
|
|
32
|
+
*/
|
|
33
|
+
export const RULE_ALIASES = { 'description/late-instruction': 'description/cut' };
|
|
27
34
|
export function runRules(rules, ctx, settings = {}) {
|
|
28
35
|
const ignore = (settings.ignore ?? []).map(globToRegExp);
|
|
36
|
+
const isIgnored = (tool) => ignore.some((re) => re.test(tool));
|
|
29
37
|
const findings = [];
|
|
30
38
|
for (const rule of rules) {
|
|
31
|
-
const configured = settings.rules?.[rule.id];
|
|
39
|
+
const configured = settings.rules?.[rule.id] ?? (RULE_ALIASES[rule.id] ? settings.rules?.[RULE_ALIASES[rule.id]] : undefined);
|
|
32
40
|
if (configured === 'off')
|
|
33
41
|
continue;
|
|
34
42
|
if (rule.since && (!ctx.protocolVersion || ctx.protocolVersion < rule.since))
|
|
35
43
|
continue;
|
|
36
|
-
for (const f of rule.run(ctx)) {
|
|
37
|
-
if (f.tool &&
|
|
44
|
+
for (const f of rule.run({ ...ctx, isIgnored })) {
|
|
45
|
+
if (f.tool && isIgnored(f.tool))
|
|
38
46
|
continue;
|
|
39
47
|
findings.push({ rule: rule.id, ...f, severity: configured ?? f.severity ?? rule.severity });
|
|
40
48
|
}
|
package/dist/rules/naming.d.ts
CHANGED