toolmenu 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +76 -20
  2. package/dist/args.d.ts +31 -5
  3. package/dist/args.js +91 -14
  4. package/dist/auth.d.ts +4 -2
  5. package/dist/auth.js +53 -17
  6. package/dist/auto.d.ts +70 -6
  7. package/dist/auto.js +248 -17
  8. package/dist/breakdown.d.ts +2 -0
  9. package/dist/breakdown.js +62 -19
  10. package/dist/catalog.d.ts +87 -10
  11. package/dist/catalog.js +415 -58
  12. package/dist/cli.js +80 -170
  13. package/dist/config.d.ts +5 -1
  14. package/dist/config.js +14 -0
  15. package/dist/connect.d.ts +4 -0
  16. package/dist/connect.js +132 -11
  17. package/dist/diff.d.ts +23 -1
  18. package/dist/diff.js +526 -114
  19. package/dist/difference.d.ts +2 -0
  20. package/dist/difference.js +13 -12
  21. package/dist/explain.d.ts +103 -0
  22. package/dist/explain.js +434 -0
  23. package/dist/failures.d.ts +9 -0
  24. package/dist/failures.js +17 -0
  25. package/dist/help.d.ts +9 -0
  26. package/dist/help.js +198 -0
  27. package/dist/history.d.ts +13 -0
  28. package/dist/history.js +86 -6
  29. package/dist/init.d.ts +92 -2
  30. package/dist/init.js +507 -20
  31. package/dist/menu.js +10 -2
  32. package/dist/options.d.ts +135 -0
  33. package/dist/options.js +48 -0
  34. package/dist/report.d.ts +1 -1
  35. package/dist/report.js +229 -99
  36. package/dist/rules/description.d.ts +7 -2
  37. package/dist/rules/description.js +85 -42
  38. package/dist/rules/determinism.d.ts +3 -1
  39. package/dist/rules/determinism.js +145 -11
  40. package/dist/rules/ids.d.ts +1 -1
  41. package/dist/rules/ids.js +9 -4
  42. package/dist/rules/index.d.ts +6 -0
  43. package/dist/rules/index.js +12 -4
  44. package/dist/rules/naming.d.ts +1 -1
  45. package/dist/rules/naming.js +66 -19
  46. package/dist/rules/rule.d.ts +18 -0
  47. package/dist/rules/rule.js +25 -1
  48. package/dist/rules/spec.js +33 -9
  49. package/dist/rules/write.d.ts +1 -1
  50. package/dist/rules/write.js +42 -10
  51. package/dist/session.d.ts +18 -8
  52. package/dist/session.js +484 -92
  53. package/dist/snapshot.d.ts +2 -2
  54. package/dist/snapshot.js +17 -2
  55. package/dist/types.d.ts +14 -0
  56. package/dist/words.d.ts +5 -0
  57. package/dist/words.js +14 -0
  58. package/package.json +1 -1
@@ -1,3 +1,4 @@
1
+ import { clip, kept, num } from './rule.js';
1
2
  /*
2
3
  * Clients that cut tool descriptions, and where. Only reported cuts, with sources:
3
4
  * - claude-code: 2,048 characters, "… [truncated]" appended, silently (anthropics/claude-code#87650);
@@ -8,14 +9,20 @@
8
9
  */
9
10
  export const CLIENT_LIMITS = { 'claude-code': 2048, 'amazon-q': 10024 };
10
11
  export const DEFAULT_CLIENT = 'claude-code';
11
- /** The cut in characters, and how to name it in a message. */
12
+ /**
13
+ * The cut in characters, the client to name in a message ("Claude Code", "your
14
+ * client"), and whether it's toolmenu's assumption rather than the user's setting.
15
+ */
12
16
  export function resolveLimit(setting) {
13
17
  if (typeof setting === 'number')
14
- return { limit: setting, who: `your client (descriptionLimit ${setting.toLocaleString('en-US')})` };
18
+ return { limit: setting, who: 'your client', assumed: false };
15
19
  const client = setting ?? DEFAULT_CLIENT;
16
20
  const limit = CLIENT_LIMITS[client] ?? CLIENT_LIMITS[DEFAULT_CLIENT];
17
- const name = client === 'amazon-q' ? 'Amazon Q CLI' : 'Claude Code';
18
- return { limit, who: `${name} (${limit.toLocaleString('en-US')} characters${setting === undefined ? ', the default: set descriptionLimit for your client' : ''})` };
21
+ return { limit, who: client === 'amazon-q' ? 'Amazon Q CLI' : 'Claude Code', assumed: setting === undefined };
22
+ }
23
+ /** The fix's last words when the cut is assumed, not set. */
24
+ function unlessOtherClient(assumed) {
25
+ return assumed ? ' If your client isn\'t Claude Code, set descriptionLimit to its cut.' : '';
19
26
  }
20
27
  /*
21
28
  * An instruction tells the agent what to do or not do: "use X", "call this tool",
@@ -93,27 +100,28 @@ export const buried = {
93
100
  lesson: 'truncated description',
94
101
  summary: 'Instructions to the agent sit past the point where your client cuts descriptions',
95
102
  run(ctx) {
96
- const { limit, who } = resolveLimit(ctx.descriptionLimit);
97
- const names = ctx.menu.tools.map((t) => t.name);
103
+ const { limit, who, assumed } = resolveLimit(ctx.descriptionLimit);
104
+ const all = ctx.menu.tools.map((t) => t.name);
98
105
  const findings = [];
99
106
  for (const tool of ctx.menu.tools) {
100
107
  const text = tool.description ?? '';
101
108
  if (text.length <= limit || sentInFull(tool, ctx.fullDescriptions))
102
109
  continue;
103
- const past = pastTheCut(text, names.filter((n) => n !== tool.name), limit);
110
+ const past = pastTheCut(text, all.filter((n) => n !== tool.name), limit);
104
111
  if (past.length === 0)
105
112
  continue;
106
113
  const one = past.length === 1;
114
+ const first = past[0];
107
115
  findings.push({
108
116
  tool: tool.name,
109
- message: `${tool.name}: the description is ${text.length.toLocaleString('en-US')} characters, and ${who} cuts it at ${limit.toLocaleString('en-US')}. ${one ? 'An instruction to the agent is' : `${past.length} instructions to the agent are`} past the cut${past[0].straddles ? `, the first cut mid-instruction (it starts at ${past[0].at.toLocaleString('en-US')})` : `, the first at ${past[0].at.toLocaleString('en-US')}`}, so the model never sees ${one ? 'it' : 'them'}. Move ${one ? 'it' : 'them'} into the first ${limit.toLocaleString('en-US')} characters, or list the tool in fullDescriptions if your client sends it uncut.`,
117
+ // Which sentences are instructions is read from the words: a guess.
118
+ confidence: 'unsure',
119
+ message: `${tool.name}: ${one ? 'a sentence that reads as an instruction to the agent sits' : `${past.length} sentences that read as instructions to the agent sit`} past character ${num(limit)}, where ${who} cuts the description, so the model never sees ${one ? 'it' : 'them'}. The description is ${num(text.length)} characters; the ${one ? '' : 'first '}instruction ${first.straddles ? `starts at ${num(first.at)} and is cut in the middle` : `is at ${num(first.at)}`}.`,
110
120
  detail: past
111
- .slice(0, 3)
112
- .map((i) => {
113
- const sentence = sentenceAround(text, i.at);
114
- return `char ${i.at.toLocaleString('en-US')}${i.straddles ? ' (cut mid-instruction)' : ''}: “${sentence.length > 200 ? sentence.slice(0, 197) + '…' : sentence}”`;
115
- })
116
- .concat(past.length > 3 ? [`…and ${past.length - 3} more`] : []),
121
+ .slice(0, 5)
122
+ .map((i) => `char ${num(i.at)}${i.straddles ? ' (cut mid-instruction)' : ''}: “${clip(sentenceAround(text, i.at), 200)}”`)
123
+ .concat(past.length > 5 ? [`…and ${past.length - 5} more`] : []),
124
+ fix: `Move “${clip(sentenceAround(text, first.at), 70)}”${one ? '' : ' and the rest above'} into the first ${num(limit)} characters of the ${tool.name} description, or list ${tool.name} in fullDescriptions if your client sends it uncut.${unlessOtherClient(assumed)}`,
117
125
  });
118
126
  }
119
127
  return findings;
@@ -121,41 +129,76 @@ export const buried = {
121
129
  };
122
130
  /** A short cut to check against when the client's own cut isn't set: one real client cut at 280. */
123
131
  const SHORT_CUT = 280;
132
+ /** “…the last words before ✂ the first words after…”, to find the cut in the text. */
133
+ function aroundTheCut(text, at) {
134
+ const before = text.slice(Math.max(0, at - 30), at).replace(/\s+/g, ' ');
135
+ const after = text.slice(at, at + 30).replace(/\s+/g, ' ');
136
+ return `“…${before}✂${after}…”`;
137
+ }
124
138
  export const cut = {
125
139
  id: 'description/cut',
126
140
  severity: 'info',
127
141
  lesson: 'truncated description',
128
- summary: 'Descriptions longer than your client sends, and instructions a shorter cut would hide',
142
+ summary: 'Descriptions longer than your client sends',
129
143
  run(ctx) {
130
- const { limit, who } = resolveLimit(ctx.descriptionLimit);
131
- const names = ctx.menu.tools.map((t) => t.name);
132
- const others = (tool) => names.filter((n) => n !== tool.name);
133
- const tools = ctx.menu.tools.filter((t) => !sentInFull(t, ctx.fullDescriptions));
134
- const findings = [];
135
- const long = tools.filter((tool) => {
144
+ const { limit, who, assumed } = resolveLimit(ctx.descriptionLimit);
145
+ const all = ctx.menu.tools.map((t) => t.name);
146
+ const long = kept(ctx.menu.tools, ctx).filter((tool) => {
136
147
  const text = tool.description ?? '';
137
- return text.length > limit && pastTheCut(text, others(tool), limit).length === 0;
148
+ return !sentInFull(tool, ctx.fullDescriptions) && text.length > limit && pastTheCut(text, all.filter((n) => n !== tool.name), limit).length === 0;
138
149
  });
139
- if (long.length) {
140
- findings.push({
141
- message: `${long.length === 1 ? '1 description is' : `${long.length} descriptions are`} longer than ${who} sends, so the model gets a prefix that can read as complete: ${list(long, (t) => `${t.name} (${(t.description ?? '').length.toLocaleString('en-US')})`)}. Nothing past the cut reads as an instruction, but check what's there.`,
142
- });
143
- }
144
- // With the default cut, say what a shorter client cut would hide.
145
- if (ctx.descriptionLimit === undefined && limit > SHORT_CUT) {
146
- const hidden = tools.filter((tool) => {
147
- const text = tool.description ?? '';
148
- return text.length > SHORT_CUT && pastTheCut(text, others(tool), SHORT_CUT).some((i) => i.at < limit);
149
- });
150
- if (hidden.length) {
151
- findings.push({
152
- message: `${hidden.length === 1 ? '1 description gives' : `${hidden.length} descriptions give`} the agent instructions after character ${SHORT_CUT}: ${list(hidden, (t) => t.name)}. ${who.split(' (')[0]} sends ${limit.toLocaleString('en-US')} characters, so they arrive there, but a client that cuts shorter never shows them (one real client cut at ${SHORT_CUT}). If yours cuts descriptions, set descriptionLimit to its cut.`,
153
- });
154
- }
150
+ if (long.length === 0)
151
+ return [];
152
+ const one = long.length === 1;
153
+ // One summary: a long description is a question for its author, not a defect.
154
+ return [
155
+ {
156
+ ...(one ? { tool: long[0].name } : {}),
157
+ message: `${one ? `The ${long[0].name} description is` : `${long.length} descriptions are`} longer than the ${num(limit)} characters ${who} sends. The model gets the start, with no sign that anything is missing, so a cut list or example can read as complete. Nothing past the cut reads as an instruction to the agent, so this is a check, not a bug.`,
158
+ detail: long.map((t) => {
159
+ const text = t.description ?? '';
160
+ return `${t.name}: ${num(text.length)} characters, ${num(text.length - limit)} past the cut, which falls at ${aroundTheCut(text, limit)}`;
161
+ }),
162
+ fix: `Check that what's past the cut ${one ? 'in' : 'in each of'} ${one ? long[0].name : 'these'} can be lost, and move anything the agent needs into the first ${num(limit)} characters.${unlessOtherClient(assumed)}`,
163
+ },
164
+ ];
165
+ },
166
+ };
167
+ export const lateInstruction = {
168
+ id: 'description/late-instruction',
169
+ severity: 'info',
170
+ lesson: 'truncated description',
171
+ summary: 'Instructions to the agent that a client cutting descriptions short would hide',
172
+ run(ctx) {
173
+ // Only with the default cut: once descriptionLimit is set, description/buried checks the real one.
174
+ if (ctx.descriptionLimit !== undefined)
175
+ return [];
176
+ const { limit, who } = resolveLimit(undefined);
177
+ if (limit <= SHORT_CUT)
178
+ return [];
179
+ const all = ctx.menu.tools.map((t) => t.name);
180
+ const late = [];
181
+ for (const tool of kept(ctx.menu.tools, ctx)) {
182
+ const text = tool.description ?? '';
183
+ if (text.length <= SHORT_CUT || sentInFull(tool, ctx.fullDescriptions))
184
+ continue;
185
+ // Past the default cut is description/buried's.
186
+ const hidden = pastTheCut(text, all.filter((n) => n !== tool.name), SHORT_CUT).filter((i) => i.at < limit);
187
+ if (hidden.length)
188
+ late.push({ tool, at: hidden[0].at, sentence: sentenceAround(text, hidden[0].at), more: hidden.length - 1 });
155
189
  }
156
- return findings;
190
+ if (late.length === 0)
191
+ return [];
192
+ const one = late.length === 1;
193
+ const client = who;
194
+ return [
195
+ {
196
+ ...(one ? { tool: late[0].tool.name } : {}),
197
+ confidence: 'unsure',
198
+ message: `${one ? `The ${late[0].tool.name} description gives` : `${late.length} descriptions give`} the agent instructions after character ${SHORT_CUT}, where a client that cuts descriptions short never shows them (one real client cut at ${SHORT_CUT}). ${client} sends ${num(limit)} characters, so there they arrive.`,
199
+ detail: late.map((l) => `${l.tool.name}: char ${num(l.at)}: “${clip(l.sentence, 120)}”${l.more ? ` (and ${l.more} more after it)` : ''}`),
200
+ fix: `If your client cuts descriptions, set descriptionLimit to its cut and toolmenu checks against that. Otherwise, moving ${one ? 'the instruction' : 'each instruction'} into the first sentences is cheap insurance.`,
201
+ },
202
+ ];
157
203
  },
158
204
  };
159
- function list(tools, show) {
160
- return tools.slice(0, 5).map(show).join(', ') + (tools.length > 5 ? `, and ${tools.length - 5} more` : '');
161
- }
@@ -1,5 +1,5 @@
1
1
  import type { MenuTool, Severity } from '../types.js';
2
- import type { Rule, RuleFinding } from './rule.js';
2
+ import { type Rule, type RuleFinding } from './rule.js';
3
3
  export declare const nondeterministic: Rule;
4
4
  /** What a second process or connection served, for the variance rules. */
5
5
  export interface Probe {
@@ -16,6 +16,8 @@ export declare function varianceFinding(main: MenuTool[], other: MenuTool[], opt
16
16
  transport: 'stdio' | 'http';
17
17
  modern: boolean;
18
18
  wrapper: boolean;
19
+ /** Places menu/nondeterministic already reports: said once, and left out of the explanation and fix. */
20
+ explained?: Set<string>;
19
21
  }): (RuleFinding & {
20
22
  rule: string;
21
23
  severity: Severity;
@@ -1,6 +1,8 @@
1
1
  import { compareMenus } from '../compare.js';
2
- import { describeToolDifference } from '../difference.js';
2
+ import { allDifferences, describeToolDifference, firstDifference } from '../difference.js';
3
3
  import { serverWords } from '../failures.js';
4
+ import { toolDefinition } from '../menu.js';
5
+ import { count } from './rule.js';
4
6
  const LABELS = {
5
7
  added: 'tool added',
6
8
  removed: 'tool removed',
@@ -12,6 +14,74 @@ const LABELS = {
12
14
  other: 'definition changed',
13
15
  serialization: 'key order changed (same content, different bytes)',
14
16
  };
17
+ const EDITS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
18
+ const TIMESTAMP = /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
19
+ /** Where each edited tool first differs between two menus. */
20
+ function spots(before, after, changes) {
21
+ const byName = new Map(before.map((t) => [t.name, t]));
22
+ const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
23
+ const out = [];
24
+ for (const t of after.filter((x) => edited.has(x.name))) {
25
+ const d = firstDifference(toolDefinition(byName.get(t.name)), toolDefinition(t));
26
+ const path = (d?.path ?? '').replace(/^inputSchema\.properties\./, '');
27
+ out.push({ tool: t.name, where: path ? `${t.name}.${path}` : t.name, before: d?.before, after: d?.after, reordered: !!d?.reordered });
28
+ }
29
+ return out;
30
+ }
31
+ /** Every value that differs inside each edited tool, not only the first. */
32
+ function everySpot(before, after, changes) {
33
+ const byName = new Map(before.map((t) => [t.name, t]));
34
+ const edited = new Set(changes.filter((c) => EDITS.has(c.kind)).map((c) => c.tool));
35
+ return after
36
+ .filter((x) => edited.has(x.name))
37
+ .flatMap((t) => allDifferences(toolDefinition(byName.get(t.name)), toolDefinition(t)).map((d) => {
38
+ const path = d.path.replace(/^inputSchema\.properties\./, '');
39
+ return { tool: t.name, where: path ? `${t.name}.${path}` : t.name, before: d.before, after: d.after, reordered: d.reordered };
40
+ }));
41
+ }
42
+ /** Every place two menus differ: whole-tool changes, and each differing value. */
43
+ function places(before, after) {
44
+ const changes = compareMenus(before, after);
45
+ return [...changes.filter((c) => !EDITS.has(c.kind)).map((c) => `${c.kind}:${c.tool}`), ...everySpot(before, after, changes).map((s) => s.where)];
46
+ }
47
+ const isTime = (s) => TIMESTAMP.test(String(s.before)) && TIMESTAMP.test(String(s.after));
48
+ /** "list_transactions.end_date.default" or "a.b, c.d and 3 more places". */
49
+ function placesOf(found) {
50
+ const shown = found.slice(0, 2).map((s) => s.where).join(', ');
51
+ return found.length > 2 ? `${shown} and ${found.length - 2} more ${found.length - 2 === 1 ? 'place' : 'places'}` : shown;
52
+ }
53
+ /** What differs, in words, after the first sentence of the message. */
54
+ function whatDiffers(changes, found) {
55
+ const time = found.filter(isTime);
56
+ if (time.length)
57
+ return `${placesOf(time)} ${time.length === 1 ? 'is a timestamp' : 'are timestamps'}, taken when the menu is built.`;
58
+ const sorted = found.filter((s) => s.reordered);
59
+ if (sorted.length)
60
+ return `${placesOf(sorted)} ${sorted.length === 1 ? 'holds' : 'hold'} the same items in a different order.`;
61
+ if (found.length)
62
+ return `It differs in ${placesOf(found)}.`;
63
+ const tools = [...new Set(changes.filter((c) => c.kind === 'added' || c.kind === 'removed').map((c) => c.tool))];
64
+ if (tools.length)
65
+ return `The tool set itself changed: ${tools.slice(0, 3).join(', ')}${tools.length > 3 ? ` and ${tools.length - 3} more` : ''}.`;
66
+ return '';
67
+ }
68
+ /** One step for the server author, for the most telling difference. */
69
+ function fixFor(changes, found, who) {
70
+ const time = found.find(isTime);
71
+ if (time)
72
+ return `Build ${time.where} from a fixed value, not the current time: leave it out, or say "defaults to now" in the description.`;
73
+ const sorted = found.filter((s) => s.reordered);
74
+ if (sorted.length)
75
+ return `Sort the items of ${sorted[0].where}${sorted.length > 1 ? ` (and the ${count(sorted.length - 1, 'other place')} listed)` : ''} when building the menu (sorted(), not a set or a map), so every ${who} lists them in the same order.`;
76
+ if (changes.some((c) => c.kind === 'added' || c.kind === 'removed')) {
77
+ return 'Serve the same tools on every list; if the set really depends on state, send notifications/tools/list_changed when it changes.';
78
+ }
79
+ if (changes.every((c) => c.kind === 'moved'))
80
+ return 'Return the tools in a fixed order: keep them in a list, or sort them by name before answering tools/list.';
81
+ if (changes.every((c) => c.kind === 'serialization' || c.kind === 'moved'))
82
+ return 'Build each tool object with its keys in a fixed order, so the bytes match as well as the content.';
83
+ return `Build ${found[0]?.where ?? 'the menu'} from fixed values: nothing from the clock, a random ID, a counter or the environment.`;
84
+ }
15
85
  export const nondeterministic = {
16
86
  id: 'menu/nondeterministic',
17
87
  severity: 'error',
@@ -24,12 +94,16 @@ export const nondeterministic = {
24
94
  if (changes.length === 0)
25
95
  return [];
26
96
  const onlyOrder = changes.every((c) => c.kind === 'moved');
97
+ const found = spots(ctx.menu.tools, ctx.secondList, changes);
98
+ const tools = [...new Set(changes.map((c) => c.tool))];
27
99
  return [
28
100
  {
101
+ ...(tools.length === 1 ? { tool: tools[0] } : {}),
29
102
  message: onlyOrder
30
- ? 'Two identical tools/list calls returned the tools in a different order. Every conversation can miss the prompt cache. The spec says servers SHOULD return a deterministic order.'
31
- : 'Two identical tools/list calls returned different menus. Every conversation can miss the prompt cache.',
103
+ ? 'Two identical tools/list calls, back to back, returned the tools in a different order, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. The spec says servers SHOULD keep the order stable.'
104
+ : `Two identical tools/list calls, back to back, returned different menus, so a cached prompt with the tool list rarely matches the next one: every conversation can miss the prompt cache. ${whatDiffers(changes, found)}`.trim(),
32
105
  detail: detailFor(ctx.menu.tools, ctx.secondList, changes),
106
+ fix: fixFor(changes, found, 'call'),
33
107
  },
34
108
  ];
35
109
  },
@@ -53,15 +127,33 @@ export function varianceFinding(main, other, opts) {
53
127
  const changes = compareMenus(main, other);
54
128
  if (changes.length === 0)
55
129
  return undefined;
56
- const detail = detailFor(main, other, changes);
57
- const reordered = detail.some((d) => d.includes('same ') && d.includes('different order'));
58
130
  const stdio = opts.transport === 'stdio';
131
+ let why = changes;
132
+ let found = spots(main, other, changes);
133
+ let detail = detailFor(main, other, changes);
134
+ let lead = '';
135
+ if (opts.explained) {
136
+ const known = opts.explained;
137
+ const all = everySpot(main, other, changes);
138
+ const same = all.filter((s) => known.has(s.where));
139
+ found = all.filter((s) => !known.has(s.where));
140
+ why = changes.filter((c) => (EDITS.has(c.kind) ? found.some((s) => s.tool === c.tool) : !known.has(`${c.kind}:${c.tool}`)));
141
+ lead = `${same.length ? `${placesOf(same)} ${same.length === 1 ? 'is' : 'are'}` : 'Part of it is'} what menu/nondeterministic reports, and goes away with that fix. The rest differs only between ${stdio ? 'processes' : 'connections'}, so it has a cause of its own:`;
142
+ const lines = [
143
+ ...why.filter((c) => !EDITS.has(c.kind)).map((c) => `${LABELS[c.kind]}: ${c.tool} (position ${c.position})`),
144
+ ...found.map((f) => `${f.where}: ${f.reordered ? 'same items, different order' : 'differs'} (${short(f.before)} vs ${short(f.after)})`),
145
+ ];
146
+ detail = lines.slice(0, 10).concat(lines.length > 10 ? [`…and ${lines.length - 10} more`] : []);
147
+ }
148
+ const reordered = found.some((s) => s.reordered);
59
149
  const message = [
60
150
  stdio
61
- ? 'A second server process, started the same way, served a different menu. Every restart, and every client that starts its own copy, gets a tool list that can\'t reuse a cached prompt.'
62
- : `A second connection with the same credentials got a different menu.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per-connection.' : ''} Clients can't share a cached menu.`,
151
+ ? 'A second server process, started the same way, served a different menu, so every restart, and every client that starts its own copy, gets a tool list that no cached prompt matches.'
152
+ : `A second connection with the same credentials got a different menu, so clients can't share a cached tool list.${opts.modern ? ' On 2026-07-28 the tool set MUST NOT vary per connection.' : ''}`,
153
+ lead,
154
+ whatDiffers(why, found),
63
155
  reordered
64
- ? 'The same items came out in a different order: likely a set or map iterated in hash order.' +
156
+ ? 'Likely a set or a map iterated in hash order.' +
65
157
  (stdio ? ' (toolmenu runs the first process with PYTHONHASHSEED=0 and the second with 1, so a Python set shows up every time; Go and Rust maps vary per process on their own.)' : '')
66
158
  : '',
67
159
  opts.wrapper
@@ -73,10 +165,27 @@ export function varianceFinding(main, other, opts) {
73
165
  severity: stdio || opts.modern ? 'error' : 'warn',
74
166
  message,
75
167
  detail,
168
+ fix: fixFor(why, found, stdio ? 'process' : 'connection'),
76
169
  };
77
170
  }
78
- const EDITS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
171
+ /**
172
+ * The places menu/nondeterministic reports, when the second list differs too:
173
+ * a probe that differs only there has the same cause, and says so instead of
174
+ * repeating it.
175
+ */
176
+ function listSpots(ctx) {
177
+ if (!ctx.secondList)
178
+ return undefined;
179
+ const found = places(ctx.menu.tools, ctx.secondList);
180
+ return found.length ? new Set(found) : undefined;
181
+ }
182
+ /** A value for a detail line, cut short. */
183
+ function short(v) {
184
+ const s = JSON.stringify(v) ?? String(v);
185
+ return s.length > 60 ? `${s.slice(0, 57)}…` : s;
186
+ }
79
187
  function varianceRule(id, transport) {
188
+ const other = transport === 'stdio' ? 'server process' : 'connection';
80
189
  return {
81
190
  id,
82
191
  severity: 'error',
@@ -88,13 +197,38 @@ function varianceRule(id, transport) {
88
197
  const out = [];
89
198
  for (const probe of ctx.probes) {
90
199
  if (probe.error) {
91
- out.push({ severity: 'info', message: `Couldn't ${transport === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}` });
200
+ out.push({
201
+ severity: 'info',
202
+ message: `Couldn't ${transport === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
203
+ fix: transport === 'stdio'
204
+ ? 'If the server can only run once at a time (a lock, a port), rerun with --processes 1 to skip this check.'
205
+ : 'If the server allows one connection at a time, rerun with --processes 1 to skip this check.',
206
+ });
92
207
  continue;
93
208
  }
94
209
  const f = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper });
95
210
  if (f) {
96
211
  const { rule: _r, ...rest } = f;
97
- out.push(rest);
212
+ // Same places as the second tools/list call: one cause, reported by menu/nondeterministic.
213
+ // Every differing value is compared, not only the first in each tool: a
214
+ // timestamp that changes every call can hide a set that changes per process.
215
+ const same = listSpots(ctx);
216
+ const here = places(ctx.menu.tools, probe.tools ?? []);
217
+ if (same && here.every((w) => same.has(w))) {
218
+ out.push({
219
+ ...rest,
220
+ message: `A second ${other} served a different menu too, in the same ${here.length === 1 ? 'place' : 'places'} menu/nondeterministic reports: a menu that changes on every call changes on every ${transport === 'stdio' ? 'restart' : 'connection'} as well. Same cause, same fix.`,
221
+ fix: 'Fix menu/nondeterministic; this goes away with it.',
222
+ });
223
+ }
224
+ else if (same && here.some((w) => same.has(w))) {
225
+ // Partly the same cause: say that once, and explain and fix the rest.
226
+ const { rule: _r2, ...partial } = varianceFinding(ctx.menu.tools, probe.tools ?? [], { transport, modern: ctx.era === 'modern', wrapper: !!ctx.wrapper, explained: same });
227
+ out.push(partial);
228
+ }
229
+ else {
230
+ out.push(rest);
231
+ }
98
232
  break; // one finding: a second differing probe says nothing new
99
233
  }
100
234
  }
@@ -1,5 +1,5 @@
1
1
  import type { MenuTool } from '../types.js';
2
- import type { Rule } from './rule.js';
2
+ import { type Rule } from './rule.js';
3
3
  interface IdParam {
4
4
  param: string;
5
5
  /** Which kind of thing it identifies (singular noun), when we can tell. */
package/dist/rules/ids.js CHANGED
@@ -1,4 +1,5 @@
1
- import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, commonWords, nouns, singular, words } from '../words.js';
1
+ import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, article, commonWords, nouns, singular, words } from '../words.js';
2
+ import { kept } from './rule.js';
2
3
  // form_id, form-id, formId, formID. Not uid or paid: the suffix needs a boundary.
3
4
  // Not "key" either: in real menus it's as often a keyboard key as an identifier.
4
5
  const ID_SUFFIX = /^(.+?)[_-](id|ids|uuid|uuids)$/i;
@@ -121,7 +122,7 @@ export const authoredIds = {
121
122
  run(ctx) {
122
123
  const byKind = new Map();
123
124
  const common = commonWords(ctx.menu.tools.map((t) => t.name));
124
- for (const tool of ctx.menu.tools) {
125
+ for (const tool of kept(ctx.menu.tools, ctx)) {
125
126
  for (const { param, kind } of idParams(tool)) {
126
127
  if (!kind || isReturnedBySomeTool(kind, tool, ctx.menu.tools, common))
127
128
  continue;
@@ -139,10 +140,14 @@ export const authoredIds = {
139
140
  return [...byKind].map(([kind, uses]) => {
140
141
  const tools = [...new Set(uses.map((u) => u.tool))];
141
142
  const where = uses.map((u) => `${u.tool}.${u.param}`);
143
+ const one = where.length === 1;
144
+ const a = article(kind);
142
145
  return {
143
146
  ...(tools.length === 1 ? { tool: tools[0] } : {}),
144
- message: `${where.length === 1 ? `${where[0]} looks like` : `${where.length} parameters look like`} a ${kind} ID the agent must supply, but no tool appears to return one (no list/search/get tool for "${kind}", no output schema with a ${kind} ID). The agent may invent it, unless it comes from somewhere toolmenu can't see (a URL, another tool's text output). Heuristic.`,
145
- ...(where.length > 1 ? { detail: [where.slice(0, 8).join(', ') + (where.length > 8 ? `, and ${where.length - 8} more` : '')] } : {}),
147
+ confidence: 'unsure',
148
+ message: `${one ? `${where[0]} takes` : `${where.length} parameters take`} ${a} ${kind} ID, and no tool in the menu appears to return one: no list, search, get or create tool for "${kind}", and no output schema with ${a} ${kind} ID. If the agent can't find one anywhere, it has to guess.`,
149
+ ...(one ? {} : { detail: [where.join(', ')] }),
150
+ fix: `If ${kind} IDs come from somewhere the agent can see (a URL the user pastes, another tool's text output), say so in the parameter's description; if not, add a tool that lists or finds them.`,
146
151
  };
147
152
  });
148
153
  },
@@ -2,6 +2,12 @@ import type { Finding, Severity } from '../types.js';
2
2
  import type { Rule, RuleContext } from './rule.js';
3
3
  export type { Rule, RuleContext } from './rule.js';
4
4
  export declare const MENU_RULES: Rule[];
5
+ /**
6
+ * Rules split out of an older one. A setting for the old id still applies to the
7
+ * new one unless the new id is set itself: before 0.13, description/cut held
8
+ * the late-instruction hint too, and `"description/cut": "off"` turned both off.
9
+ */
10
+ export declare const RULE_ALIASES: Record<string, string>;
5
11
  export interface RuleSettings {
6
12
  /** Per-rule severity override, or 'off'. */
7
13
  rules?: Record<string, Severity | 'off'>;
@@ -1,5 +1,5 @@
1
1
  import { SEVERITY_RANK } from '../types.js';
2
- import { buried, cut } from './description.js';
2
+ import { buried, cut, lateInstruction } from './description.js';
3
3
  import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
4
4
  import { authoredIds } from './ids.js';
5
5
  import { route, sharedWord, vagueId } from './naming.js';
@@ -16,6 +16,7 @@ export const MENU_RULES = [
16
16
  authoredIds,
17
17
  buried,
18
18
  cut,
19
+ lateInstruction,
19
20
  noDryRun,
20
21
  unannotated,
21
22
  schema,
@@ -24,17 +25,24 @@ export const MENU_RULES = [
24
25
  deprecated,
25
26
  cacheHints,
26
27
  ];
28
+ /**
29
+ * Rules split out of an older one. A setting for the old id still applies to the
30
+ * new one unless the new id is set itself: before 0.13, description/cut held
31
+ * the late-instruction hint too, and `"description/cut": "off"` turned both off.
32
+ */
33
+ export const RULE_ALIASES = { 'description/late-instruction': 'description/cut' };
27
34
  export function runRules(rules, ctx, settings = {}) {
28
35
  const ignore = (settings.ignore ?? []).map(globToRegExp);
36
+ const isIgnored = (tool) => ignore.some((re) => re.test(tool));
29
37
  const findings = [];
30
38
  for (const rule of rules) {
31
- const configured = settings.rules?.[rule.id];
39
+ const configured = settings.rules?.[rule.id] ?? (RULE_ALIASES[rule.id] ? settings.rules?.[RULE_ALIASES[rule.id]] : undefined);
32
40
  if (configured === 'off')
33
41
  continue;
34
42
  if (rule.since && (!ctx.protocolVersion || ctx.protocolVersion < rule.since))
35
43
  continue;
36
- for (const f of rule.run(ctx)) {
37
- if (f.tool && ignore.some((re) => re.test(f.tool)))
44
+ for (const f of rule.run({ ...ctx, isIgnored })) {
45
+ if (f.tool && isIgnored(f.tool))
38
46
  continue;
39
47
  findings.push({ rule: rule.id, ...f, severity: configured ?? f.severity ?? rule.severity });
40
48
  }
@@ -1,4 +1,4 @@
1
- import type { Rule } from './rule.js';
1
+ import { type Rule } from './rule.js';
2
2
  export declare const vagueId: Rule;
3
3
  export declare const sharedWord: Rule;
4
4
  export declare const route: Rule;