toolmenu 0.13.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,77 @@
1
+ import { unlockers } from '../session.js';
2
+ import { singular, VERBS, words } from '../words.js';
3
+ import { names } from './rule.js';
4
+ /**
5
+ * A server that unlocks tools and says its list can change may have served only
6
+ * its starting set. A baseline made from that misses every tool behind the unlock,
7
+ * so a diff can't see their breaking changes (a real server: 18 of 115 tools).
8
+ */
9
+ export const gated = {
10
+ id: 'menu/gated',
11
+ severity: 'warn',
12
+ summary: 'The menu may be only the starting set of a server that unlocks more tools',
13
+ run(ctx) {
14
+ const listChanged = ctx.capabilities.tools?.listChanged === true;
15
+ if (!listChanged)
16
+ return [];
17
+ // Only an unlock whose parameter lists what it unlocks (an enum of toolsets or
18
+ // domains), the kind session --auto walks: "Enables Cross-Region Restore on a
19
+ // vault" turns on a feature, not tools (Azure, 418 tools).
20
+ const found = unlockers(ctx.menu.tools).filter((u) => u.param && u.values.length);
21
+ if (!found.length)
22
+ return [];
23
+ // A value whose tools are already listed adds nothing: the same server started
24
+ // with every toolset still has its unlock. Where the tools are named
25
+ // namespace-first (content_get, group_list, contentGetEntry), a value matches a
26
+ // tool whose name starts with the value's words (pull_requests → pull_requests_list):
27
+ // an always-on home_getGroups or search_groups isn't the groups domain's. Where
28
+ // they're named verb-first (list_team_audits), every word of the value in the
29
+ // tool's name will do.
30
+ const unlocking = new Set(found.map((u) => u.tool.name));
31
+ const tools = ctx.menu.tools.filter((t) => !unlocking.has(t.name));
32
+ const namespaced = tools.filter((t) => !VERBS.has(singular(words(t.name)[0] ?? ''))).length * 2 > tools.length;
33
+ const startsWith = (name, want) => {
34
+ const have = words(name).map(singular);
35
+ return want.every((w, i) => have[i] === w);
36
+ };
37
+ const missing = (value) => {
38
+ const want = words(String(value)).map(singular);
39
+ if (!want.length)
40
+ return false;
41
+ return namespaced
42
+ ? !tools.some((t) => startsWith(t.name, want))
43
+ : !tools.some((t) => {
44
+ const have = new Set(words(t.name).map(singular));
45
+ return want.every((w) => have.has(w));
46
+ });
47
+ };
48
+ const gaps = found.map((u) => ({ u, missing: u.values.filter(missing).map(String) })).filter((g) => g.missing.length);
49
+ if (!gaps.length)
50
+ return [];
51
+ const { u: first, missing: absent } = gaps[0];
52
+ const more = gaps.slice(1).map((g) => g.u.tool.name);
53
+ // Most values matched: the menu is likely complete, with a few toolsets named for
54
+ // something else (insights → report_get). A hint, not a warning.
55
+ if (absent.length * 2 <= first.values.length) {
56
+ return [
57
+ {
58
+ tool: first.tool.name,
59
+ severity: 'info',
60
+ confidence: 'unsure',
61
+ message: `${first.tool.name} unlocks tools and the server says its tool list can change (listChanged); ${absent.length} of ${first.values.length} values (${names(absent, 5)}) match no tool by name. If their tools are named differently, the menu is complete; if not, it lacks them.`,
62
+ detail: [`unlock: ${first.tool.name} (${first.param}: ${names(first.values.map(String), 12)}); no tool named for: ${names(absent, 8)}${more.length ? `; also ${names(more, 4)}` : ''}`],
63
+ fix: `If ${names(absent, 3)} ${absent.length === 1 ? 'has' : 'have'} tools under other names, nothing to do. Otherwise commit every tool a session sees as the baseline: toolmenu session --auto --union-out menu.json ${ctx.server ?? '-- <the command that starts the server>'}.`,
64
+ },
65
+ ];
66
+ }
67
+ return [
68
+ {
69
+ tool: first.tool.name,
70
+ confidence: 'unsure',
71
+ message: `${first.tool.name} looks like it unlocks more tools, and the server says its tool list can change (listChanged), but no tool in this menu matches ${names(absent, 5)}, so this menu of ${ctx.menu.tools.length} tools may be only the starting set. A baseline made from it misses the tools behind the unlock, and a diff against it can't see their breaking changes.`,
72
+ detail: [`unlock: ${first.tool.name} (${first.param}: ${names(first.values.map(String), 8)}); no tool named for: ${names(absent, 8)}${more.length ? `; also ${names(more, 4)}` : ''}`],
73
+ fix: `Commit every tool a session sees as the baseline instead: toolmenu session --auto --union-out menu.json ${ctx.server ?? '-- <the command that starts the server>'} (it calls each value of ${first.tool.name}). If the server already lists every tool under other names, turn this off with rules: { "menu/gated": "off" }.`,
74
+ },
75
+ ];
76
+ },
77
+ };
@@ -1,15 +1,18 @@
1
1
  import { SEVERITY_RANK } from '../types.js';
2
2
  import { buried, cut, lateInstruction } from './description.js';
3
- import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
3
+ import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
4
4
  import { authoredIds } from './ids.js';
5
5
  import { route, sharedWord, vagueId } from './naming.js';
6
- import { cacheHints, deprecated, discover, duplicateName, schema, toolsCapability } from './spec.js';
6
+ import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
7
7
  import { noDryRun, unannotated } from './write.js';
8
+ import { gated } from './gated.js';
8
9
  export const MENU_RULES = [
9
10
  nondeterministic,
10
11
  duplicateName,
12
+ gated,
11
13
  processVariance,
12
14
  connectionVariance,
15
+ hostSpecific,
13
16
  vagueId,
14
17
  sharedWord,
15
18
  route,
@@ -24,6 +27,7 @@ export const MENU_RULES = [
24
27
  discover,
25
28
  deprecated,
26
29
  cacheHints,
30
+ serverInfo,
27
31
  ];
28
32
  /**
29
33
  * Rules split out of an older one. A setting for the old id still applies to the
@@ -1,4 +1,4 @@
1
- import { COLLECTION_VERBS, LOOKUP_VERBS, WRITE_VERBS, commonWords, nouns, singular, verbOf, words } from '../words.js';
1
+ import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, WRITE_VERBS, commonWords, nouns, singular, verbOf, words } from '../words.js';
2
2
  import { missingWords, negatedMention, routeScores } from '../routes.js';
3
3
  import { kept, names } from './rule.js';
4
4
  // Not `ref`: in real menus it's a git ref (GitHub ×4), a name with a meaning of its own.
@@ -57,10 +57,27 @@ export const sharedWord = {
57
57
  summary: 'The same noun used for different things across tools',
58
58
  run(ctx) {
59
59
  const byNoun = new Map();
60
- const common = commonWords(ctx.menu.tools.map((t) => t.name));
61
- const subject = (name) => nouns(name).filter((n) => !common.has(n));
60
+ // Compared in the singular: "maps" in every name is as common as "map".
61
+ const common = new Set([...commonWords(ctx.menu.tools.map((t) => t.name))].map(singular));
62
+ const subject = (name) => nouns(name).filter((n) => !common.has(singular(n)));
63
+ // A word that opens two or more tool names is a namespace (content_*, maps_*):
64
+ // where it opens the name it isn't the tool's subject, so it isn't compared as
65
+ // a shared word there. It still tells qualifiers apart (content_get_entry vs
66
+ // context_get_entry both say "entry").
67
+ const opens = new Map();
68
+ for (const tool of ctx.menu.tools) {
69
+ const first = singular(words(tool.name)[0] ?? '');
70
+ if (first)
71
+ opens.set(first, (opens.get(first) ?? 0) + 1);
72
+ }
73
+ const namespace = (name) => {
74
+ const first = singular(words(name)[0] ?? '');
75
+ return (opens.get(first) ?? 0) >= 2 && !VERBS.has(first) ? first : undefined;
76
+ };
62
77
  for (const tool of ctx.menu.tools) {
63
78
  for (const noun of new Set(subject(tool.name))) {
79
+ if (noun === namespace(tool.name))
80
+ continue;
64
81
  byNoun.set(noun, [...(byNoun.get(noun) ?? []), tool]);
65
82
  }
66
83
  }
@@ -71,6 +88,12 @@ export const sharedWord = {
71
88
  for (const [noun, tools] of byNoun) {
72
89
  if (tools.length > 1 && tools.every((t) => singular(firstWord(t.name) ?? '') === noun))
73
90
  continue;
91
+ // In every tool's name of a menu of three or more, it's the server's one subject
92
+ // (DeepWiki: ask_wiki_question, read_wiki_contents, read_wiki_structure), not two
93
+ // things sharing a word. Two tools are too few to tell: list_team_audits and
94
+ // get_audit_trail are the clash this rule exists for.
95
+ if (tools.length >= 3 && tools.length === ctx.menu.tools.length)
96
+ continue;
74
97
  // Tools where the noun is qualified differently and neither qualifier set
75
98
  // contains the other: list_team_audits vs get_audit_trail.
76
99
  const qualifiers = tools.map((t) => ({ tool: t.name, rest: new Set(subject(t.name).filter((n) => n !== noun)) }));
@@ -19,6 +19,13 @@ export interface RuleContext {
19
19
  probes?: import('./determinism.js').Probe[];
20
20
  /** The stdio command is a container wrapper (docker run …): env vars may not reach the server. */
21
21
  wrapper?: boolean;
22
+ /** serverInfo as the handshake sent it, when there was one (snapshot): spec/server-info. */
23
+ serverInfo?: {
24
+ name?: string;
25
+ version?: string;
26
+ };
27
+ /** How this run reached the server, as the command line ends: `-- node server.js` or the URL. For next steps. */
28
+ server?: string;
22
29
  /** Where clients are assumed to cut tool descriptions (description/buried). */
23
30
  descriptionLimit?: number | string;
24
31
  /** Tools (names or globs) the client sends uncut, so description/buried skips them. */
@@ -4,4 +4,10 @@ export declare const toolsCapability: Rule;
4
4
  export declare const duplicateName: Rule;
5
5
  export declare const discover: Rule;
6
6
  export declare const deprecated: Rule;
7
+ /**
8
+ * serverInfo { name, version } is required in the handshake (both protocol
9
+ * generations). Without them reports say "unknown server", and clients and logs
10
+ * can't tell which server or release answered (Netlify sends neither).
11
+ */
12
+ export declare const serverInfo: Rule;
7
13
  export declare const cacheHints: Rule;
@@ -130,6 +130,29 @@ export const deprecated = {
130
130
  return [];
131
131
  },
132
132
  };
133
+ /**
134
+ * serverInfo { name, version } is required in the handshake (both protocol
135
+ * generations). Without them reports say "unknown server", and clients and logs
136
+ * can't tell which server or release answered (Netlify sends neither).
137
+ */
138
+ export const serverInfo = {
139
+ id: 'spec/server-info',
140
+ severity: 'warn',
141
+ summary: 'The handshake names the server and its version',
142
+ run(ctx) {
143
+ if (!ctx.serverInfo)
144
+ return [];
145
+ const missing = [!ctx.serverInfo.name ? 'name' : '', !ctx.serverInfo.version ? 'version' : ''].filter(Boolean);
146
+ if (!missing.length)
147
+ return [];
148
+ return [
149
+ {
150
+ message: `The server's handshake has no serverInfo ${missing.join(' or ')}, which the spec requires. Clients, logs and this report can't tell which ${missing.length === 2 ? 'server or release' : missing[0] === 'name' ? 'server' : 'release'} answered${ctx.serverInfo.name ? ` (${ctx.serverInfo.name})` : ''}.`,
151
+ fix: 'Set serverInfo { name, version } where the server is created, the version from its package.',
152
+ },
153
+ ];
154
+ },
155
+ };
133
156
  export const cacheHints = {
134
157
  id: 'spec/cache-hints',
135
158
  severity: 'info',
@@ -56,11 +56,16 @@ export const noDryRun = {
56
56
  lesson: '02',
57
57
  summary: 'Destructive tools offer a way to look before leaping',
58
58
  run(ctx) {
59
+ // Annotated destructive (destructiveHint: true), or destructive only because the
60
+ // spec says so when a tool says readOnlyHint: false and nothing more: told apart,
61
+ // since "hover" counted as destructive is a missing annotation, not a missing dry run.
59
62
  const missing = [];
63
+ const byDefault = [];
60
64
  for (const tool of kept(ctx.menu.tools, ctx)) {
61
65
  const a = tool.annotations ?? {};
62
- const destructive = a.destructiveHint === true || (a.readOnlyHint === false && a.destructiveHint !== false);
63
- if (!destructive)
66
+ const annotated = a.destructiveHint === true;
67
+ const defaulted = !annotated && a.readOnlyHint === false && a.destructiveHint === undefined;
68
+ if (!annotated && !defaulted)
64
69
  continue;
65
70
  const hasParam = Object.keys(tool.inputSchema?.properties ?? {}).some((p) => DRY_RUN_PARAMS.test(p));
66
71
  const subject = new Set(nouns(tool.name));
@@ -68,19 +73,32 @@ export const noDryRun = {
68
73
  words(other.name).some((w) => PREVIEW_WORDS.has(w)) &&
69
74
  nouns(other.name).some((n) => subject.has(n)));
70
75
  if (!hasParam && !hasPreviewTool)
71
- missing.push(tool.name);
76
+ (annotated ? missing : byDefault).push(tool.name);
72
77
  }
73
- if (missing.length === 0)
78
+ if (missing.length === 0 && byDefault.length === 0)
74
79
  return [];
80
+ const defaultLine = `Destructive only by default (readOnlyHint: false and no destructiveHint): ${byDefault.join(', ')}`;
81
+ if (missing.length === 0) {
82
+ const one = byDefault.length === 1;
83
+ return [
84
+ {
85
+ ...(one ? { tool: byDefault[0] } : {}),
86
+ confidence: 'unsure',
87
+ message: `${one ? `${byDefault[0]} is` : `${count(byDefault.length, 'tool')} are`} destructive only by default: ${one ? 'it says' : 'they say'} readOnlyHint: false and nothing about destructiveHint, which the spec reads as destructive, and toolmenu found no dry_run/preview-style parameter or preview tool. Clients then confirm ${one ? 'it' : 'them'} like a delete.`,
88
+ ...(one ? {} : { detail: [byDefault.join(', ')] }),
89
+ fix: `Set destructiveHint on each: false where it only reads, adds or changes something cheap to undo, true where it deletes or overwrites (and then consider a dry_run parameter).`,
90
+ },
91
+ ];
92
+ }
75
93
  const one = missing.length === 1;
76
94
  // One finding, not one per tool: it's a design note, not a list of defects.
77
95
  return [
78
96
  {
79
- ...(one ? { tool: missing[0] } : {}),
97
+ ...(one && !byDefault.length ? { tool: missing[0] } : {}),
80
98
  confidence: 'unsure',
81
99
  message: `${one ? `${missing[0]} is` : `${count(missing.length, 'tool')} are`} annotated as destructive, and toolmenu found no dry_run/preview-style parameter and no preview tool for the same thing. Where a mistake is costly to undo, a dry run lets the agent see what would change while it can still back out.`,
82
- ...(one ? {} : { detail: [missing.join(', ')] }),
83
- fix: `Where the effect is hard to undo, add a dry_run parameter${one ? ` to ${missing[0]}` : ''} that returns what would change without changing it. If it's cheap to undo, or a preview exists under another name, ignore this.`,
100
+ ...(one && !byDefault.length ? {} : { detail: [...(one ? [] : [missing.join(', ')]), ...(byDefault.length ? [defaultLine] : [])] }),
101
+ fix: `Where the effect is hard to undo, add a dry_run parameter${one ? ` to ${missing[0]}` : ''} that returns what would change without changing it. If it's cheap to undo, or a preview exists under another name, ignore this.${byDefault.length ? ` For the tools destructive only by default, set destructiveHint: false on the ones that don't delete or overwrite.` : ''}`,
84
102
  },
85
103
  ];
86
104
  },
package/dist/session.d.ts CHANGED
@@ -43,6 +43,11 @@ export interface StepRecord {
43
43
  note?: string;
44
44
  /** Why the call failed, when it did (a tool error or a failed request). */
45
45
  failure?: FailureClass;
46
+ /**
47
+ * What a call step's call did: the tool answered, answered with an error
48
+ * (isError), the request failed, or it wasn't sent (refused, not in the menu).
49
+ */
50
+ outcome?: 'answered' | 'tool-error' | 'failed' | 'not-sent';
46
51
  changed: boolean;
47
52
  listChanged: number;
48
53
  scope?: Scope;
@@ -76,6 +81,11 @@ export interface SessionResult {
76
81
  }
77
82
  export interface SessionOptions {
78
83
  timeoutMs?: number;
84
+ /**
85
+ * --auto: steps for tools that appear mid-session, run right after the step
86
+ * that brought them. Called once per change, with only the new tools.
87
+ */
88
+ replan?: (added: MenuTool[]) => Step[];
79
89
  /** How long to wait for a list_changed notification after a change. */
80
90
  noticeGraceMs?: number;
81
91
  rules?: Record<string, Severity | 'off'>;
@@ -91,12 +101,16 @@ export interface SessionOptions {
91
101
  rateLimitWaitsMs?: number[];
92
102
  }
93
103
  /** `each`: this step's line when findings of one rule are said once for several steps. */
94
- type Raw = Omit<Finding, 'severity'> & {
104
+ /** `each`: this step's line when repeats merge. `cause`: what makes two edits one cause. Neither is reported. */
105
+ export type Raw = Omit<Finding, 'severity'> & {
95
106
  severity: Severity;
96
107
  each?: string;
108
+ cause?: string;
97
109
  };
98
110
  /** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
99
111
  export declare function session(target: Target, scenario: Scenario, options?: SessionOptions): Promise<SessionResult>;
112
+ /** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
113
+ export declare function stepsText(steps: number[]): string;
100
114
  /** Turn a menu change into findings, with the cost of each. */
101
115
  export declare function changeFindings(before: MenuTool[], after: MenuTool[], changes: ToolChange[], step: number, origin?: string): Raw[];
102
116
  /**
@@ -107,6 +121,7 @@ export declare function changeFindings(before: MenuTool[], after: MenuTool[], ch
107
121
  * a fresh process starts there anyway (an unlock undone reads as "seen" by it).
108
122
  */
109
123
  export declare function scopeOf(changes: ToolChange[], next: MenuTool[], probe: MenuTool[], transport: Target['kind'], baseline?: MenuTool[]): Scope;
124
+ export declare function mergeRepeats(raw: Raw[]): Raw[];
110
125
  export type { Era };
111
126
  interface Unlocker {
112
127
  tool: MenuTool;
package/dist/session.js CHANGED
@@ -5,10 +5,10 @@ import { listTools } from './connect.js';
5
5
  import { buildMenu, toolDefinition } from './menu.js';
6
6
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
7
7
  import { varianceFinding } from './rules/determinism.js';
8
- import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
8
+ import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
9
  import { SEVERITY_RANK } from './types.js';
10
10
  import { leadingJson } from './catalog.js';
11
- import { describeToolDifference } from './difference.js';
11
+ import { allDifferences, describeToolDifference } from './difference.js';
12
12
  import { autoNextStep, neededValues, writeSign } from './auto.js';
13
13
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
14
14
  const LIST_CHANGED = 'notifications/tools/list_changed';
@@ -182,7 +182,8 @@ export async function session(target, scenario, options = {}) {
182
182
  };
183
183
  const callWaited = { ms: 0 };
184
184
  const listWaited = { ms: 0 };
185
- const assumed = new Set(scenario.assumeReadOnly ?? []);
185
+ // Read live: --auto vouches for tools that appear mid-session too.
186
+ const assumed = { has: (tool) => (scenario.assumeReadOnly ?? []).includes(tool) };
186
187
  // Called on the user's word (assume_read_only), by step.
187
188
  const onWord = [];
188
189
  // The same failure from the same tool at several steps is one finding that lists them.
@@ -230,6 +231,7 @@ export async function session(target, scenario, options = {}) {
230
231
  if (!tool) {
231
232
  record.status = 'failed';
232
233
  record.reason = 'missing';
234
+ record.outcome = 'not-sent';
233
235
  record.note = `${step.tool} isn't in the menu at this point`;
234
236
  const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
235
237
  const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
@@ -245,6 +247,7 @@ export async function session(target, scenario, options = {}) {
245
247
  else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
246
248
  record.status = 'refused';
247
249
  record.reason = 'refused';
250
+ record.outcome = 'not-sent';
248
251
  record.note = 'not marked readOnlyHint';
249
252
  raw.push({
250
253
  rule: 'session/refused',
@@ -275,6 +278,7 @@ export async function session(target, scenario, options = {}) {
275
278
  error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
276
279
  result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
277
280
  });
281
+ record.outcome = result.isError ? 'tool-error' : 'answered';
278
282
  if (result.isError) {
279
283
  const text = errorText(result);
280
284
  if (RATE_LIMITED.test(text)) {
@@ -294,6 +298,7 @@ export async function session(target, scenario, options = {}) {
294
298
  }
295
299
  catch (error) {
296
300
  record.status = 'failed';
301
+ record.outcome = 'failed';
297
302
  record.note = serverWords(error, 200);
298
303
  if (lost(record.note, index)) {
299
304
  done();
@@ -406,6 +411,10 @@ export async function session(target, scenario, options = {}) {
406
411
  }
407
412
  }
408
413
  }
414
+ // --auto: the tools this step brought get their calls now, while they're listed.
415
+ const added = new Set(changes.filter((c) => c.kind === 'added').map((c) => c.tool));
416
+ if (options.replan && added.size)
417
+ scenario.steps.splice(i + 1, 0, ...options.replan(next.tools.filter((t) => added.has(t.name))));
409
418
  current = next;
410
419
  see(next);
411
420
  }
@@ -449,7 +458,8 @@ export async function session(target, scenario, options = {}) {
449
458
  steps,
450
459
  findings: settle(mergeRepeats(raw), options),
451
460
  ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
452
- union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
461
+ // Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
462
+ union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
453
463
  };
454
464
  }
455
465
  finally {
@@ -562,11 +572,11 @@ function nothingCalled(auto) {
562
572
  const unmarked = by('unmarked');
563
573
  const writes = by('not read-only');
564
574
  if (needs.length)
565
- parts.push(`${needs.length} need values the schema doesn't give`);
575
+ parts.push(`${needs.length} ${needs.length === 1 ? 'needs' : 'need'} values the schema doesn't give`);
566
576
  if (unmarked.length)
567
- parts.push(`${unmarked.length} aren't marked readOnlyHint`);
577
+ parts.push(`${unmarked.length} ${unmarked.length === 1 ? "isn't" : "aren't"} marked readOnlyHint`);
568
578
  if (openWorld.length)
569
- parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
579
+ parts.push(`${openWorld.length} marked openWorldHint (${openWorld.length === 1 ? 'it' : 'they'} may cost API credits)`);
570
580
  if (writes.length)
571
581
  parts.push(`${writes.length} marked or named as writes`);
572
582
  const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
@@ -607,9 +617,12 @@ function sameError(failures) {
607
617
  }
608
618
  // The server says the feature isn't there, or isn't for this account: no argument
609
619
  // fixes that. A guess from its words.
610
- const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\bpermission\b|\bforbidden\b|\bnot allowed\b|\brole\b/i;
620
+ // Only definite statements: a hedge in the server's own advice ("the account may lack
621
+ // permission") says nothing about this call, and a bare "permission" or "role" turns up in
622
+ // ordinary validation messages.
623
+ const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\b(?:permission|access)s? denied\b|\binsufficient (?:permissions?|privileges?)\b|\b(?:don't|doesn't|do not|does not) have (?:the )?(?:required |necessary )?(?:permissions?|access)\b|\bforbidden\b|\bnot allowed\b/i;
611
624
  /** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
612
- function stepsText(steps) {
625
+ export function stepsText(steps) {
613
626
  if (steps.length === 1)
614
627
  return `step ${steps[0]}`;
615
628
  const runs = [];
@@ -637,7 +650,9 @@ function failureFindings(failures, auto) {
637
650
  const calls = [f, ...(f.others ?? [])];
638
651
  const tools = [...new Set(calls.map((c) => c.tool))];
639
652
  const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
640
- const said = f.text ? quotedSentence(clip(f.text, 160)) : 'no error text.';
653
+ const words = errorWords(f.text);
654
+ const shown = clip(words, 160);
655
+ const said = f.text ? quotedSentence(shown) : 'no error text.';
641
656
  const firstArg = Object.keys(f.args)[0];
642
657
  const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
643
658
  // No argument can fix a call to a tool that takes none, or a feature the server
@@ -671,6 +686,8 @@ function failureFindings(failures, auto) {
671
686
  step: f.steps[0],
672
687
  ...(many ? { steps: f.steps } : {}),
673
688
  tool: f.tool,
689
+ // The server's whole text, when the message above shows less of it.
690
+ ...(f.text && shown !== f.text ? { serverText: f.text } : {}),
674
691
  ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
675
692
  message: f.rule === 'session/tool-error'
676
693
  ? tools.length > 1
@@ -806,7 +823,7 @@ export function changeFindings(before, after, changes, step, origin) {
806
823
  const cost = !brk
807
824
  ? []
808
825
  : [
809
- `the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}; any change to the tool list invalidates the cached prompt, so the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it are processed again`,
826
+ `cost: the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it; the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}`,
810
827
  ];
811
828
  const why = origin ? [origin] : [];
812
829
  const added = changes.filter((c) => c.kind === 'added');
@@ -870,11 +887,17 @@ export function changeFindings(before, after, changes, step, origin) {
870
887
  const b = after.find((t) => t.name === tool);
871
888
  // What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
872
889
  const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
890
+ // One cause: the same tool, the same changed values and the same origin (a
891
+ // timestamp in a description, rewritten at every call). A change that touches
892
+ // anything more is a change of its own.
893
+ const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
873
894
  out.push({
874
895
  rule: 'session/edit',
875
896
  severity: 'error',
876
897
  step,
877
898
  tool,
899
+ cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
900
+ each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
878
901
  message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
879
902
  detail: [...seen, ...cost, ...why],
880
903
  fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
@@ -944,11 +967,12 @@ function scopeFindings(scope, modern, step) {
944
967
  * Findings that say the same thing at every step are said once, naming the steps:
945
968
  * stdio's side-effect warning, and an unlock per domain, where each value appends
946
969
  * tools for this connection only (9 steps made 18 near-identical blocks). What
947
- * differs per step, the tools and their cost, goes in the detail. Mid-inserts,
948
- * removals and edits stay per step: each is its own change.
970
+ * differs per step, the tools and their cost, goes in the detail. Mid-inserts
971
+ * and removals stay per step: each is its own change. Edits merge only when the
972
+ * tool, the changed values and their origin are the same: one cause.
949
973
  */
950
974
  const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
951
- function mergeRepeats(raw) {
975
+ export function mergeRepeats(raw) {
952
976
  let out = raw;
953
977
  for (const rule of REPEATS) {
954
978
  const same = out.filter((f) => f.rule === rule);
@@ -968,6 +992,21 @@ function mergeRepeats(raw) {
968
992
  const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
969
993
  out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
970
994
  }
995
+ // The same edit at several steps is one cause: one finding naming the steps,
996
+ // with each step's position and cost.
997
+ const byCause = new Map();
998
+ for (const f of out)
999
+ if (f.rule === 'session/edit' && f.cause)
1000
+ byCause.set(f.cause, [...(byCause.get(f.cause) ?? []), f]);
1001
+ for (const same of byCause.values()) {
1002
+ if (same.length < 2)
1003
+ continue;
1004
+ const steps = same.map((f) => f.step);
1005
+ // The per-step value and cost are in each step's line; keep the rest (the origin) once.
1006
+ const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.startsWith('cost: ') && !f.each.endsWith(` · ${d}`))))];
1007
+ const merged = { ...same[0], steps, message: `${stepsText(steps).replace(/^s/, 'S')}: ${same[0].message}`, detail: [...same.map((f) => f.each), ...why] };
1008
+ out = out.filter((f) => !same.includes(f) || f === same[0]).map((f) => (f === same[0] ? merged : f));
1009
+ }
971
1010
  return out;
972
1011
  }
973
1012
  function describe(changes) {
@@ -977,7 +1016,7 @@ function settle(raw, options) {
977
1016
  const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
978
1017
  return raw
979
1018
  .filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
980
- .map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
1019
+ .map(({ each: _each, cause: _cause, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
981
1020
  .sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
982
1021
  }
983
1022
  /*
@@ -22,3 +22,9 @@ export interface SnapshotResult {
22
22
  }
23
23
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
24
24
  export declare function snapshot(target: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
25
+ /**
26
+ * How this run reached the server, as the end of a command line, for next steps:
27
+ * the options that got it past the door (names only; values are the user's) and
28
+ * then `-- command args` or the URL.
29
+ */
30
+ export declare function serverArgs(target: Target): string;
package/dist/snapshot.js CHANGED
@@ -6,6 +6,7 @@ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded
6
6
  import { MENU_RULES, runRules } from './rules/index.js';
7
7
  import { catalogFindings, readCatalog } from './catalog.js';
8
8
  import { SEVERITY_RANK } from './types.js';
9
+ import { shellWord } from './init.js';
9
10
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
10
11
  export async function snapshot(target, options = {}) {
11
12
  // Connecting and listing are read-only: a refusal for being too many is waited
@@ -57,6 +58,8 @@ export async function snapshot(target, options = {}) {
57
58
  transport: target.kind,
58
59
  probes,
59
60
  wrapper: isContainerWrapper(target),
61
+ serverInfo: connection.server,
62
+ server: serverArgs(target),
60
63
  }, options);
61
64
  // What --catalog called and what it got, under the same settings as the rules
62
65
  // (rules: { "catalog/read": "off" }, ignore).
@@ -72,3 +75,16 @@ export async function snapshot(target, options = {}) {
72
75
  }
73
76
  return { menu, findings };
74
77
  }
78
+ /**
79
+ * How this run reached the server, as the end of a command line, for next steps:
80
+ * the options that got it past the door (names only; values are the user's) and
81
+ * then `-- command args` or the URL.
82
+ */
83
+ export function serverArgs(target) {
84
+ if (target.kind === 'stdio') {
85
+ const env = Object.keys(target.env ?? {}).map((k) => `--env ${k}=<value>`);
86
+ return [...env, `-- ${[target.command, ...target.args].map(shellWord).join(' ')}`].join(' ');
87
+ }
88
+ const headers = Object.keys(target.headers ?? {}).map((k) => `--header "${k}: <value>"`);
89
+ return [...headers, ...(target.noAuth ? ['--no-auth'] : []), shellWord(target.url)].join(' ');
90
+ }
package/dist/types.d.ts CHANGED
@@ -24,6 +24,8 @@ export interface Finding {
24
24
  * where, not what it means. Absent: toolmenu observed it directly.
25
25
  */
26
26
  confidence?: 'unsure';
27
+ /** For `session` tool errors: the tool's whole error text, when the message shows only part of it. */
28
+ serverText?: string;
27
29
  }
28
30
  /** A tool exactly as the server returned it, plus toolmenu's token estimate. */
29
31
  export interface MenuTool {
@@ -60,6 +62,11 @@ export interface Menu {
60
62
  ttlMs?: number;
61
63
  cacheScope?: string;
62
64
  };
65
+ /**
66
+ * 'session': every tool a session saw (session --union-out), in the order first
67
+ * seen, which is the unlock order, not an order any client was served.
68
+ */
69
+ from?: 'session';
63
70
  /** Operations behind a search tool (snapshot --catalog). Not part of the menu or its tokens. */
64
71
  catalog?: import('./catalog.js').Catalog;
65
72
  }
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.13.0",
3
+ "version": "0.14.1",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
- "author": "Niksa (https://niksa.me)",
5
+ "author": "Nikša Perović (https://niksa.me)",
6
6
  "main": "dist/index.js",
7
7
  "scripts": {
8
8
  "build": "tsc -p tsconfig.json",