toolmenu 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/session.js CHANGED
@@ -6,12 +6,13 @@ import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from '.
6
6
  import { buildMenu, toolDefinition } from './menu.js';
7
7
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
8
8
  import { varianceFinding } from './rules/determinism.js';
9
- import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
9
+ import { classifyFailure, deniedAnswer, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
10
10
  import { SEVERITY_RANK } from './types.js';
11
11
  import { leadingJson } from './catalog.js';
12
12
  import { allDifferences, describeToolDifference } from './difference.js';
13
13
  import { autoNextStep, neededValues, writeSign } from './auto.js';
14
14
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
15
+ import { handleFromResult } from './rules/handles.js';
15
16
  const LIST_CHANGED = 'notifications/tools/list_changed';
16
17
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
17
18
  const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
@@ -78,6 +79,9 @@ export async function session(target, scenario, options = {}) {
78
79
  const raw = [];
79
80
  // 2026-07-28: every call that asked for input, for the input findings at the end.
80
81
  const inputs = [];
82
+ // --create-handles: the handles creating tools returned, by kind, and what went wrong on the way.
83
+ const created = new Map();
84
+ const roundtrips = [];
81
85
  // --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
82
86
  const handleErrors = [];
83
87
  // The main process runs with a pinned hash seed, and so does every scope probe:
@@ -255,7 +259,7 @@ export async function session(target, scenario, options = {}) {
255
259
  fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
256
260
  });
257
261
  }
258
- else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
262
+ else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word && !(step.creates && tool.annotations?.destructiveHint !== true)) {
259
263
  record.status = 'refused';
260
264
  record.reason = 'refused';
261
265
  record.outcome = 'not-sent';
@@ -275,6 +279,11 @@ export async function session(target, scenario, options = {}) {
275
279
  }
276
280
  if (word)
277
281
  onWord.push({ tool: step.tool, step: index });
282
+ // --create-handles: the handle the creating tool returned, in place of a made-up one.
283
+ const real = step.probe?.fromCreated ? created.get(step.probe.kind) : undefined;
284
+ const args = real && step.probe ? { ...step.args, [step.probe.param]: real.value } : step.args;
285
+ if (real)
286
+ record.label = stepLabel({ ...step, args });
278
287
  if (record.status === 'ok')
279
288
  try {
280
289
  // Sent again after a refusal only when that repeats nothing: the transport's
@@ -285,7 +294,7 @@ export async function session(target, scenario, options = {}) {
285
294
  callWaited.ms = 0;
286
295
  // 2026-07-28: a tool that asks for input gets every elicitation declined
287
296
  // and is called again (mrtr.ts); on 2025 it's a plain call.
288
- const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: step.args }, { timeoutMs, elicitation: answer }), {
297
+ const call = await patiently(() => callWithInput(conn, { name: step.tool, arguments: args }, { timeoutMs, elicitation: answer }), {
289
298
  waits,
290
299
  waited: callWaited,
291
300
  // A 429 means the server didn't run that request, but after an input round the
@@ -309,12 +318,29 @@ export async function session(target, scenario, options = {}) {
309
318
  record.outcome = result.isError ? 'tool-error' : 'answered';
310
319
  if (call.input.rounds && !result.isError)
311
320
  record.note = inputNote(call.input, answer);
321
+ if (step.creates) {
322
+ // The handle this result carries, for the probes after it.
323
+ const value = result.isError ? undefined : handleFromResult(result, step.creates);
324
+ if (value)
325
+ created.set(step.creates, { tool: step.tool, value });
326
+ else
327
+ roundtrips.push({ step: index, tool: step.tool, kind: step.creates, problem: result.isError ? `returned an error: ${clip(errorText(result), 120)}` : 'returned no handle toolmenu could find' });
328
+ record.note = value ? `created a ${step.creates} (${clip(value, 40)}) for the calls after it` : `--create-handles: ${roundtrips.at(-1).problem}`;
329
+ }
312
330
  const setup = step.probe && result.isError ? classifyFailure(errorText(result), { hadArguments: true }) : undefined;
313
331
  if (step.probe && setup && SETUP_FAILURES.has(setup)) {
314
332
  // Missing credentials, this machine: the call never got as far as the
315
333
  // handle, so it says nothing about it. Counted with the other setup failures.
316
334
  record.failure = setup;
317
- record.note = `called with a made-up ${step.probe.kind}, but it failed before the ${step.probe.kind} mattered: ${clip(errorText(result), 100)}`;
335
+ record.note = `called with ${real ? `the ${step.probe.kind} ${real.tool} just created` : `a made-up ${step.probe.kind}`}, but it failed before the ${step.probe.kind} mattered: ${clip(errorText(result), 100)}`;
336
+ }
337
+ else if (step.probe && real) {
338
+ // The handle the creating tool just returned: an error here is the round trip broken.
339
+ record.note = result.isError
340
+ ? `called with the ${step.probe.kind} ${real.tool} just created; it answered with an error: ${clip(errorText(result), 100)}`
341
+ : `called with the ${step.probe.kind} ${real.tool} just created; it answered`;
342
+ if (result.isError)
343
+ roundtrips.push({ step: index, tool: step.tool, kind: step.probe.kind, creator: real.tool, problem: `answered with an error: ${clip(errorText(result), 160)}` });
318
344
  }
319
345
  else if (step.probe) {
320
346
  // A handle that can't exist: a tool error is the answer the spec asks for.
@@ -345,6 +371,17 @@ export async function session(target, scenario, options = {}) {
345
371
  fail('session/tool-error', 'warn', index, step, text, record.failure);
346
372
  }
347
373
  }
374
+ else if (!call.input.rounds && !step.creates) {
375
+ // An auth error passed on as an answer (not isError): the call reached the
376
+ // server but not the API behind it, so it tested no more than a refused one.
377
+ // A tool that takes a URL fetches it: what its answer says (a status, "Forbidden") is the page's.
378
+ const fetches = Object.keys(tool?.inputSchema?.properties ?? {}).some((p) => /^(?:url|uri|href|endpoint|link)s?$/i.test(p) || /_url$|Url$/.test(p));
379
+ const denied = fetches ? undefined : deniedAnswer(result);
380
+ if (denied) {
381
+ record.failure = 'auth';
382
+ record.note = `the tool answered, not as an error, with the API's ${clip(denied, 120)}`;
383
+ }
384
+ }
348
385
  }
349
386
  }
350
387
  catch (error) {
@@ -377,7 +414,11 @@ export async function session(target, scenario, options = {}) {
377
414
  }
378
415
  const probeFailure = step.probe ? classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: true }) : undefined;
379
416
  // Not a setup failure (credentials, this machine, -32021): those never got as far as the handle.
380
- if (step.probe && typeof code === 'number' && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
417
+ if (step.probe && real && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
418
+ roundtrips.push({ step: index, tool: step.tool, kind: step.probe.kind, creator: real.tool, problem: `failed: ${clip(record.note ?? '', 160)}` });
419
+ record.note = `called with the ${step.probe.kind} ${real.tool} just created; it failed: ${clip(record.note ?? '', 120)}`;
420
+ }
421
+ else if (step.probe && typeof code === 'number' && probeFailure && !SETUP_FAILURES.has(probeFailure)) {
381
422
  // The server's protocol error to a handle that can't exist: the model gets no reason it can act on.
382
423
  if (!handleErrors.some((e) => e.tool === step.tool))
383
424
  handleErrors.push({ step: index, tool: step.tool, probe: step.probe, value: step.args[step.probe.param], message: record.note, code });
@@ -420,6 +461,31 @@ export async function session(target, scenario, options = {}) {
420
461
  done();
421
462
  break;
422
463
  }
464
+ if (/\btimed out after\b.*\bnever answered\b/.test(why)) {
465
+ // The server is running but answers nothing, tools/list included: every later
466
+ // step would wait out its timeout too (fulcra-context-mcp polls a device-code
467
+ // sign-in inside the call). One finding, and the run stops, as for a rate limit.
468
+ limitedAt = index;
469
+ // The call's own timeout at this step is the same cause: said by this finding.
470
+ for (const [key, f] of failures) {
471
+ if (!f.timedOutMs || f.steps.at(-1) !== index)
472
+ continue;
473
+ f.steps.pop();
474
+ if (!f.steps.length)
475
+ failures.delete(key);
476
+ }
477
+ const last = scenario.steps.length;
478
+ raw.push({
479
+ rule: 'session/step-failed',
480
+ severity: 'error',
481
+ step: index,
482
+ ...(step.kind === 'call' ? { tool: step.tool } : {}),
483
+ message: `The server stopped answering at step ${index} (${label}): after it, ${why.replace(/\.$/, '')}. The run stopped there${index < last ? `: steps ${index + 1}–${last} weren't run` : ''}.`,
484
+ fix: 'A tool that waits for something outside the call (a sign-in in a browser, a device code, input on a terminal) blocks every request after it. The server\'s stderr may say what it waits for: sign in before the run, or pass credentials (--env), and rerun.',
485
+ });
486
+ done();
487
+ break;
488
+ }
423
489
  raw.push({
424
490
  rule: 'session/step-failed',
425
491
  severity: 'error',
@@ -502,6 +568,7 @@ export async function session(target, scenario, options = {}) {
502
568
  raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
503
569
  raw.push(...inputFindings(inputs, answer));
504
570
  raw.push(...handleErrorFindings(handleErrors));
571
+ raw.push(...roundtripFindings(roundtrips));
505
572
  if (explained.length)
506
573
  raw.push(knownVariance(explained));
507
574
  if (onWord.length)
@@ -1456,6 +1523,30 @@ function inputNote(input, answer) {
1456
1523
  return asked;
1457
1524
  }
1458
1525
  }
1526
+ /**
1527
+ * --create-handles: a handle a creating tool returned that its own tools then refuse, or
1528
+ * a creating tool whose result carries no handle to find.
1529
+ */
1530
+ function roundtripFindings(problems) {
1531
+ return problems.map((p) => p.creator
1532
+ ? {
1533
+ rule: 'session/handle-roundtrip',
1534
+ severity: 'warn',
1535
+ confidence: 'unsure',
1536
+ step: p.step,
1537
+ tool: p.tool,
1538
+ message: `${p.tool} was given the ${p.kind} ${p.creator} had just created, and ${p.problem}. A model that starts a ${p.kind} and uses it right away hits the same error.`,
1539
+ fix: `Check that ${p.tool} accepts the ${p.kind} handle ${p.creator} returns, in the same form (the same field, no extra prefix or quoting).`,
1540
+ }
1541
+ : {
1542
+ rule: 'session/handle-roundtrip',
1543
+ severity: 'info',
1544
+ step: p.step,
1545
+ tool: p.tool,
1546
+ message: `--create-handles called ${p.tool}, which ${p.problem}, so the tools that take a ${p.kind} were called with a made-up one instead.`,
1547
+ fix: `Return the new ${p.kind}'s handle where a model finds it: a ${p.kind}_id field in structuredContent (declared in outputSchema), or at least "${p.kind}_id: …" in the text.`,
1548
+ });
1549
+ }
1459
1550
  /**
1460
1551
  * Handles that can't exist answered with a protocol error (2026-07-28, SEP-2567):
1461
1552
  * "a call against an expired or unknown handle should return a tool execution
@@ -41,6 +41,16 @@ export interface SnapshotResult {
41
41
  }
42
42
  /** Establish the menu: connect, list the tools twice, run the menu rules. */
43
43
  export declare function snapshot(given: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
44
+ /**
45
+ * The target with every credential it carries swapped for one that can't be valid,
46
+ * same scheme: Bearer stays Bearer, Basic gets a made-up user and password, an API-key
47
+ * header a made-up key. A stored login (OAuth) becomes an invalid Bearer token.
48
+ */
49
+ export declare function withInvalidCredential(target: Extract<Target, {
50
+ kind: 'http';
51
+ }>): Extract<Target, {
52
+ kind: 'http';
53
+ }>;
44
54
  /**
45
55
  * How this run reached the server, as the end of a command line, for next steps:
46
56
  * the options that got it past the door (names only; values are the user's) and
package/dist/snapshot.js CHANGED
@@ -4,6 +4,8 @@ import { ConnectError } from './explain.js';
4
4
  import { buildMenu } from './menu.js';
5
5
  import { asIdentity, connectPatiently, isContainerWrapper, MAIN_SEED, PROBE_SEED, probeMenu, probeVariance, seeded } from './probe.js';
6
6
  import { kindsToRead, readList } from './lists.js';
7
+ import { isCredentialHeader } from './explain.js';
8
+ import { httpStatus } from './failures.js';
7
9
  import { clientCost } from './clients.js';
8
10
  import { MENU_RULES, runRules } from './rules/index.js';
9
11
  import { catalogFindings, readCatalog } from './catalog.js';
@@ -76,6 +78,9 @@ export async function snapshot(given, options = {}) {
76
78
  // The tool probe above says why a second process or connection failed.
77
79
  }
78
80
  }
81
+ // A credential that can't be valid, in place of the one this run used (HTTP): the
82
+ // server should refuse it, not serve a menu (rules/auth.ts).
83
+ const invalid = target.kind === 'http' && connection.usedAuth && (options.processes ?? 2) > 1 ? await invalidCredentialProbe(target, options.timeoutMs ?? 30_000) : undefined;
79
84
  // Each other identity's menu, from its own process (stdio) or connection (HTTP).
80
85
  const identities = mainIdentity
81
86
  ? [{ name: mainIdentity.name, tools: menu.tools, tokens: menu.totalTokens, cacheScope: menu.listMeta?.cacheScope }]
@@ -110,6 +115,7 @@ export async function snapshot(given, options = {}) {
110
115
  serverInfo: connection.server,
111
116
  server: serverArgs(target),
112
117
  identities,
118
+ ...(invalid ? { invalidCredential: invalid } : {}),
113
119
  ...(reads.length || undeclared.length ? { lists: { read: reads, undeclared } } : {}),
114
120
  }, options);
115
121
  // What --catalog called and what it got, under the same settings as the rules
@@ -127,6 +133,41 @@ export async function snapshot(given, options = {}) {
127
133
  const cost = options.client ? clientCost(options.client, menu) : undefined;
128
134
  return { menu, findings, ...(identities ? { identities } : {}), ...(cost ? { cost } : {}), ...(undeclared.length ? { notOffered: undeclared } : {}) };
129
135
  }
136
+ /**
137
+ * The target with every credential it carries swapped for one that can't be valid,
138
+ * same scheme: Bearer stays Bearer, Basic gets a made-up user and password, an API-key
139
+ * header a made-up key. A stored login (OAuth) becomes an invalid Bearer token.
140
+ */
141
+ export function withInvalidCredential(target) {
142
+ const headers = {};
143
+ let replaced = false;
144
+ for (const [k, v] of Object.entries(target.headers ?? {})) {
145
+ if (k.toLowerCase() === 'authorization') {
146
+ const scheme = /^\s*(\w+)\s/.exec(v)?.[1] ?? 'Bearer';
147
+ headers[k] = /^basic$/i.test(scheme) ? `Basic ${Buffer.from('toolmenu-invalid:toolmenu-invalid').toString('base64')}` : `${scheme} toolmenu-invalid-credential`;
148
+ replaced = true;
149
+ }
150
+ else if (isCredentialHeader(k)) {
151
+ headers[k] = 'toolmenu-invalid-credential';
152
+ replaced = true;
153
+ }
154
+ else
155
+ headers[k] = v;
156
+ }
157
+ if (!replaced)
158
+ headers.Authorization = 'Bearer toolmenu-invalid-credential';
159
+ return { ...target, headers, noAuth: true };
160
+ }
161
+ async function invalidCredentialProbe(target, timeoutMs) {
162
+ try {
163
+ const m = await probeMenu(withInvalidCredential(target), timeoutMs);
164
+ return { tools: m.tools };
165
+ }
166
+ catch (error) {
167
+ const status = httpStatus(error);
168
+ return status !== undefined ? { status } : { error: firstLine(error) };
169
+ }
170
+ }
130
171
  const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split('\n')[0].trim();
131
172
  /**
132
173
  * How this run reached the server, as the end of a command line, for next steps:
package/dist/words.d.ts CHANGED
@@ -23,3 +23,5 @@ export declare function verbOf(name: string): string | undefined;
23
23
  export declare function leadingVerb(name: string, common?: Set<string>): string | undefined;
24
24
  /** The nouns in a tool name, singular: `list_team_audits` → team, audit. */
25
25
  export declare function nouns(name: string): string[];
26
+ /** Levenshtein distance: edits that turn one string into the other. */
27
+ export declare function editDistance(a: string, b: string): number;
package/dist/words.js CHANGED
@@ -84,3 +84,17 @@ export function nouns(name) {
84
84
  .map(singular)
85
85
  .filter((w) => !GENERIC.has(w));
86
86
  }
87
+ /** Levenshtein distance: edits that turn one string into the other. */
88
+ export function editDistance(a, b) {
89
+ const row = Array.from({ length: b.length + 1 }, (_, i) => i);
90
+ for (let i = 1; i <= a.length; i++) {
91
+ let diagonal = row[0];
92
+ row[0] = i;
93
+ for (let j = 1; j <= b.length; j++) {
94
+ const above = row[j];
95
+ row[j] = Math.min(row[j] + 1, row[j - 1] + 1, diagonal + (a[i - 1] === b[j - 1] ? 0 : 1));
96
+ diagonal = above;
97
+ }
98
+ }
99
+ return row[b.length];
100
+ }
package/package.json CHANGED
@@ -1,11 +1,11 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.20.1",
3
+ "version": "0.21.0",
4
4
  "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
5
  "author": "Nikša Perović (https://niksa.me)",
6
6
  "main": "dist/index.js",
7
7
  "scripts": {
8
- "build": "tsc -p tsconfig.json",
8
+ "build": "tsc -p tsconfig.json && node scripts/rule-docs.mjs",
9
9
  "test": "npm run build && node --test --test-reporter=spec test/*.test.mjs",
10
10
  "typecheck": "tsc -p tsconfig.json --noEmit",
11
11
  "prepublishOnly": "npm run build"