toolmenu 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +312 -35
  2. package/dist/args.d.ts +14 -2
  3. package/dist/args.js +44 -1
  4. package/dist/auth-check.d.ts +27 -0
  5. package/dist/auth-check.js +177 -0
  6. package/dist/auth.d.ts +7 -0
  7. package/dist/auth.js +29 -1
  8. package/dist/auto.d.ts +8 -0
  9. package/dist/auto.js +9 -2
  10. package/dist/breakdown.js +14 -10
  11. package/dist/cli.js +171 -56
  12. package/dist/clients.d.ts +13 -2
  13. package/dist/clients.js +65 -11
  14. package/dist/config.js +6 -0
  15. package/dist/diff.d.ts +35 -1
  16. package/dist/diff.js +57 -12
  17. package/dist/failures.js +4 -3
  18. package/dist/help.d.ts +1 -1
  19. package/dist/help.js +45 -17
  20. package/dist/history.js +5 -0
  21. package/dist/html.d.ts +29 -0
  22. package/dist/html.js +360 -0
  23. package/dist/index.d.ts +3 -2
  24. package/dist/index.js +3 -2
  25. package/dist/init.d.ts +14 -0
  26. package/dist/init.js +20 -6
  27. package/dist/junit.d.ts +17 -0
  28. package/dist/junit.js +87 -0
  29. package/dist/mcp-env.d.ts +6 -0
  30. package/dist/mcp-env.js +6 -0
  31. package/dist/mcp.d.ts +10 -0
  32. package/dist/mcp.js +86 -15
  33. package/dist/menu.d.ts +6 -0
  34. package/dist/menu.js +12 -1
  35. package/dist/options.d.ts +13 -0
  36. package/dist/options.js +4 -0
  37. package/dist/report.d.ts +14 -0
  38. package/dist/report.js +57 -51
  39. package/dist/review.d.ts +48 -0
  40. package/dist/review.js +212 -0
  41. package/dist/rule-docs.json +49 -4
  42. package/dist/rules/clients.d.ts +11 -0
  43. package/dist/rules/clients.js +65 -2
  44. package/dist/rules/determinism.d.ts +3 -0
  45. package/dist/rules/determinism.js +44 -7
  46. package/dist/rules/ids.js +5 -1
  47. package/dist/rules/index.js +5 -2
  48. package/dist/rules/output.d.ts +21 -0
  49. package/dist/rules/output.js +49 -0
  50. package/dist/rules/search.js +1 -1
  51. package/dist/rules/spec.d.ts +19 -0
  52. package/dist/rules/spec.js +86 -1
  53. package/dist/sanitize.d.ts +42 -0
  54. package/dist/sanitize.js +142 -0
  55. package/dist/sarif.d.ts +2 -0
  56. package/dist/sarif.js +2 -2
  57. package/dist/scan.d.ts +9 -2
  58. package/dist/scan.js +46 -24
  59. package/dist/session.d.ts +17 -0
  60. package/dist/session.js +119 -13
  61. package/dist/snapshot.js +2 -1
  62. package/package.json +5 -3
package/dist/session.js CHANGED
@@ -5,7 +5,7 @@ import { listTools } from './connect.js';
5
5
  import { AfterInputError, callWithInput, MAX_CONTINUATIONS, MAX_ROUNDS } from './mrtr.js';
6
6
  import { buildMenu, toolDefinition } from './menu.js';
7
7
  import { connectPatiently, isContainerWrapper, MAIN_SEED, probeSameEra, probeVariance, seeded } from './probe.js';
8
- import { varianceFinding } from './rules/determinism.js';
8
+ import { outsideModel, varianceFinding } from './rules/determinism.js';
9
9
  import { classifyFailure, errorWords, structuredError, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
10
10
  import { SEVERITY_RANK } from './types.js';
11
11
  import { leadingJson } from './catalog.js';
@@ -13,6 +13,7 @@ import { allDifferences, describeToolDifference } from './difference.js';
13
13
  import { autoNextStep, neededValues, writeSign } from './auto.js';
14
14
  import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
15
15
  import { handleFromResult } from './rules/handles.js';
16
+ import { OUTPUT_SCHEMA_SINCE, outputCheck } from './rules/output.js';
16
17
  const LIST_CHANGED = 'notifications/tools/list_changed';
17
18
  /** A server saying the session is gone: "Session not found or expired" (toolception). */
18
19
  const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
@@ -72,6 +73,36 @@ export function stepLabel(step) {
72
73
  return `call ${step.tool}${Object.keys(step.args).length ? ' ' + JSON.stringify(step.args) : ''}`;
73
74
  return `wait_for ${step.event}`;
74
75
  }
76
+ /**
77
+ * What a step's changes did to each tool of the menu after it, by position (two tools
78
+ * can share a name). An added tool is appended when only added tools follow it. The
79
+ * tools it removed (from `before`, the menu before the step) follow, marked removed.
80
+ */
81
+ export function stepMenu(tools, changes = [], before = []) {
82
+ const added = new Set();
83
+ const moved = new Set();
84
+ const edited = new Set();
85
+ const removed = [];
86
+ for (const c of changes) {
87
+ if (c.kind === 'added')
88
+ added.add(c.position);
89
+ else if (c.kind === 'moved')
90
+ moved.add(c.position);
91
+ else if (c.kind === 'removed')
92
+ removed.push({ name: c.tool, tokens: before[c.position]?.tokens ?? 0, change: 'removed' });
93
+ else if (c.kind !== 'serialization')
94
+ edited.add(c.position);
95
+ }
96
+ // Where the trailing run of added tools starts: those are appended, earlier ones inserted.
97
+ let tail = tools.length;
98
+ while (tail > 0 && added.has(tail - 1))
99
+ tail--;
100
+ const menu = tools.map((t, i) => {
101
+ const change = added.has(i) ? (i >= tail ? 'appended' : 'inserted') : moved.has(i) ? 'moved' : edited.has(i) ? 'edited' : undefined;
102
+ return { name: t.name, tokens: t.tokens, ...(change ? { change } : {}) };
103
+ });
104
+ return [...menu, ...removed];
105
+ }
75
106
  /** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
76
107
  export async function session(target, scenario, options = {}) {
77
108
  const timeoutMs = options.timeoutMs ?? 30_000;
@@ -84,6 +115,8 @@ export async function session(target, scenario, options = {}) {
84
115
  const roundtrips = [];
85
116
  // HTTP errors passed on as plain answers, not isError (session/error-not-flagged).
86
117
  const unflagged = [];
118
+ // Successful results that break the tool's own outputSchema (session/output-schema).
119
+ const outputs = [];
87
120
  // --auto's calls with a made-up handle that got a protocol error back (session/handle-error).
88
121
  const handleErrors = [];
89
122
  // The main process runs with a pinned hash seed, and so does every scope probe:
@@ -112,7 +145,10 @@ export async function session(target, scenario, options = {}) {
112
145
  listening = false;
113
146
  }
114
147
  }
115
- const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
148
+ const menuOf = async (c) => {
149
+ const listed = await listTools(c, { timeoutMs });
150
+ return buildMenu(listed.tools, serverOf(c), listed.listMeta);
151
+ };
116
152
  const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
117
153
  // A refusal that outlasts the waits, or a write's that isn't sent again, ends
118
154
  // the run with one finding, not one per step. `tool`: the write not sent again.
@@ -222,6 +258,10 @@ export async function session(target, scenario, options = {}) {
222
258
  const tried = new Set();
223
259
  const quiet = new Set();
224
260
  const stopped = new Map();
261
+ // Where a wait_for starts counting list_changed: from the start of the last call,
262
+ // or from the end of the last wait_for. The notification a call causes usually
263
+ // arrives while the call step runs, before the wait_for after it begins.
264
+ let waitFrom = conn.wire.notifications.length;
225
265
  for (const [i, step] of scenario.steps.entries()) {
226
266
  const index = i + 1;
227
267
  if (step.kind === 'call' && step.tentative) {
@@ -235,6 +275,8 @@ export async function session(target, scenario, options = {}) {
235
275
  tried.add(step.tool);
236
276
  const label = stepLabel(step);
237
277
  const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
278
+ let stepChanges = [];
279
+ let stepBefore = [];
238
280
  const mark = conn.wire.notifications.length;
239
281
  let word = false;
240
282
  const done = () => {
@@ -243,6 +285,7 @@ export async function session(target, scenario, options = {}) {
243
285
  steps.push(record);
244
286
  };
245
287
  if (step.kind === 'call') {
288
+ waitFrom = mark;
246
289
  const tool = current.tools.find((t) => t.name === step.tool);
247
290
  const write = tool ? writeSign(tool) : undefined;
248
291
  word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
@@ -319,6 +362,12 @@ export async function session(target, scenario, options = {}) {
319
362
  else {
320
363
  const result = call.result;
321
364
  record.outcome = result.isError ? 'tool-error' : 'answered';
365
+ // A tool that declares an outputSchema promises its successful results (2025-06-18 on).
366
+ if (!result.isError && tool?.outputSchema && (conn.protocolVersion ?? '') >= OUTPUT_SCHEMA_SINCE && !outputs.some((o) => o.tool === step.tool)) {
367
+ const checked = outputCheck(tool.outputSchema, result);
368
+ if (checked)
369
+ outputs.push({ step: index, tool: step.tool, ...checked });
370
+ }
322
371
  if (call.input.rounds && !result.isError)
323
372
  record.note = inputNote(call.input, answer);
324
373
  if (step.creates) {
@@ -431,7 +480,11 @@ export async function session(target, scenario, options = {}) {
431
480
  record.note = `called with a made-up ${step.probe.kind}; it answered with a protocol error (${code}): ${clip(record.note, 120)}`;
432
481
  }
433
482
  else {
434
- record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
483
+ // A 401 or 403 on the call itself is the credentials, whatever the words: Google's
484
+ // hosted servers list tools to anyone and answer each call with 401 and a
485
+ // WWW-Authenticate for that tool.
486
+ const status = httpStatus(error);
487
+ record.failure = status === 401 || status === 403 ? 'auth' : classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
435
488
  // The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
436
489
  const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
437
490
  if (!SETUP_FAILURES.has(record.failure))
@@ -440,8 +493,10 @@ export async function session(target, scenario, options = {}) {
440
493
  }
441
494
  }
442
495
  else if (step.kind === 'wait_for') {
443
- const arrived = await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, step.timeoutMs);
444
- if (!arrived) {
496
+ const arrived = await waitFor(() => conn.wire.notificationsSince(waitFrom, LIST_CHANGED) > 0, step.timeoutMs);
497
+ if (arrived)
498
+ waitFrom = conn.wire.notifications.length;
499
+ else {
445
500
  record.status = 'timed-out';
446
501
  record.note = `no list_changed within ${step.timeoutMs} ms`;
447
502
  }
@@ -503,6 +558,8 @@ export async function session(target, scenario, options = {}) {
503
558
  continue;
504
559
  }
505
560
  const changes = compareMenus(current.tools, next.tools);
561
+ stepChanges = changes;
562
+ stepBefore = current.tools;
506
563
  if (changes.length) {
507
564
  record.changed = true;
508
565
  const paths = variedPaths(current.tools, next.tools, changes);
@@ -567,6 +624,7 @@ export async function session(target, scenario, options = {}) {
567
624
  record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
568
625
  record.tools = current.tools.length;
569
626
  record.tokens = current.totalTokens;
627
+ options.onStepMenu?.(index, stepMenu(current.tools, stepChanges, stepBefore));
570
628
  if (firstTry && !record.changed && record.status === 'ok' && !record.failure && record.outcome !== 'needs-input')
571
629
  quiet.add(step.tool);
572
630
  done();
@@ -575,6 +633,7 @@ export async function session(target, scenario, options = {}) {
575
633
  raw.push(...inputFindings(inputs, answer));
576
634
  raw.push(...handleErrorFindings(handleErrors));
577
635
  raw.push(...roundtripFindings(roundtrips));
636
+ raw.push(...outputFindings(outputs));
578
637
  // `ignore` is applied by tool name; this finding sums several tools up, so it's applied here.
579
638
  const ignored = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
580
639
  raw.push(...unflaggedFindings(unflagged.filter((u) => !ignored.some((re) => re.test(u.tool)))));
@@ -608,10 +667,12 @@ export async function session(target, scenario, options = {}) {
608
667
  final: { tools: current.tools.length, tokens: current.totalTokens },
609
668
  connectionCheck,
610
669
  steps,
611
- findings: settle(mergeRepeats(raw), options),
670
+ findings: settle(mergeRepeats(options.auto?.everything ? raw.map(madeUpInput) : raw), options),
612
671
  ...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
613
672
  // Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
614
- union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
673
+ // With the starting list's cache hints (ttlMs, cacheScope), as snapshot's menu file has
674
+ // them: without, review saw spec/cache-hints as gone from a baseline made this way.
675
+ union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server, baseline.listMeta), from: 'session' },
615
676
  };
616
677
  }
617
678
  finally {
@@ -838,9 +899,12 @@ function failureFindings(failures, auto) {
838
899
  const call = `${c.tool} ${JSON.stringify(c.args)}`;
839
900
  return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
840
901
  });
902
+ // --call-everything: made-up values go in, so a readable tool error is the right
903
+ // answer to most of them (the spec asks for exactly that). Said as info.
904
+ const madeUp = auto?.everything === true && f.rule === 'session/tool-error';
841
905
  return {
842
906
  rule: f.rule,
843
- severity: f.severity,
907
+ severity: madeUp ? 'info' : f.severity,
844
908
  step: f.steps[0],
845
909
  ...(many ? { steps: f.steps } : {}),
846
910
  tool: f.tool,
@@ -848,9 +912,11 @@ function failureFindings(failures, auto) {
848
912
  ...(f.text && shown !== f.text ? { serverText: f.text } : {}),
849
913
  ...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
850
914
  message: f.rule === 'session/tool-error'
851
- ? tools.length > 1
852
- ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
853
- : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
915
+ ? madeUp
916
+ ? `${lead}${tools.length > 1 ? `${who} answered` : `${f.tool} answered`} the made-up values with a tool error the model can read: ${said} With --call-everything that's the right answer; it says nothing about a call with real values.`
917
+ : tools.length > 1
918
+ ? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
919
+ : `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
854
920
  : tools.length > 1
855
921
  ? `${lead}Calling ${who} failed the same way: ${said}`
856
922
  : `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
@@ -1049,14 +1115,19 @@ export function changeFindings(before, after, changes, step, origin) {
1049
1115
  // timestamp in a description, rewritten at every call). A change that touches
1050
1116
  // anything more is a change of its own.
1051
1117
  const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
1118
+ // Only in fields the model doesn't read (a per-tool version hash, the @pipeworx
1119
+ // servers): the cached prompt is untouched, so not an error about it.
1120
+ const outside = a && b ? outsideModel([a], [b], changes.filter((c) => c.tool === tool)) : undefined;
1052
1121
  out.push({
1053
1122
  rule: 'session/edit',
1054
- severity: 'error',
1123
+ severity: outside ? 'warn' : 'error',
1055
1124
  step,
1056
1125
  tool,
1057
1126
  cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
1058
1127
  each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
1059
- message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
1128
+ message: outside
1129
+ ? `${tool}: ${outside.join(', ')} changed mid-session, ${outside.length === 1 ? 'a field' : 'fields'} clients keep to themselves: the model doesn't read ${outside.length === 1 ? 'it' : 'them'}, so the cached prompt isn't affected, but a client that compares tool lists sees the tool change.`
1130
+ : `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
1060
1131
  detail: [...seen, ...cost, ...why],
1061
1132
  fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
1062
1133
  });
@@ -1698,3 +1769,38 @@ function inputFindings(inputs, answer) {
1698
1769
  }
1699
1770
  return out;
1700
1771
  }
1772
+ /** session/output-schema: one finding for the run, every tool whose result broke its own outputSchema. */
1773
+ function outputFindings(list) {
1774
+ if (!list.length)
1775
+ return [];
1776
+ const one = list.length === 1;
1777
+ // Only format checks failed (a URI, a date): many validators skip format, so a warning.
1778
+ const formatOnly = list.every((o) => o.formatOnly);
1779
+ return [
1780
+ {
1781
+ rule: 'session/output-schema',
1782
+ severity: formatOnly ? 'warn' : 'error',
1783
+ step: list[0].step,
1784
+ ...(one ? { tool: list[0].tool } : {}),
1785
+ message: `${one ? `${list[0].tool}'s result` : `${list.length} tools' results`} broke ${one ? 'its' : 'their'} own outputSchema. A tool that declares one MUST return structuredContent that conforms (2025-06-18 and later), and the official SDK client checks: it raises an error where the tool answered.${formatOnly ? ' Only format checks failed (a URI, a date), which many validators skip, so a client may let it through.' : ''}`,
1786
+ detail: list.map((o) => `${o.tool} (step ${o.step}): ${clip(o.problem, 200)}`),
1787
+ fix: 'Return structuredContent that matches the outputSchema on every successful call, or drop the outputSchema from tools whose results vary.',
1788
+ },
1789
+ ];
1790
+ }
1791
+ /**
1792
+ * --call-everything sends made-up values into a sandbox with no network, so a call
1793
+ * that fails, or reaches no network, or none that got through, says how the tool met
1794
+ * that input, not that the server is broken: info. What a result breaks whatever the
1795
+ * input (its own outputSchema, resultType, an error not flagged as one) keeps its level.
1796
+ */
1797
+ function madeUpInput(f) {
1798
+ const note = ' (--call-everything: made-up values, no network)';
1799
+ // Offline, a "rate limit" is most often a network call that couldn't be made.
1800
+ if (f.rule === 'session/step-failed' || f.rule === 'session/untested' || f.rule === 'session/rate-limited')
1801
+ return { ...f, severity: 'info', message: f.message + note };
1802
+ // A server that stops answering after a call is worth knowing, but offline it may be waiting on the network.
1803
+ if (f.rule === 'session/server-unresponsive' && f.severity === 'error')
1804
+ return { ...f, severity: 'warn', message: f.message + note };
1805
+ return f;
1806
+ }
package/dist/snapshot.js CHANGED
@@ -65,8 +65,9 @@ export async function snapshot(given, options = {}) {
65
65
  catch (error) {
66
66
  const stderr = connection.stderr().trim();
67
67
  // A ConnectError already shows the stderr lines that matter.
68
+ // Each line marked, as ConnectError does: a server's line can't pass for toolmenu's.
68
69
  if (stderr && error instanceof Error && !(error instanceof ConnectError))
69
- error.message += `\nserver stderr:\n${stderr}`;
70
+ error.message += `\nserver stderr:\n${stderr.split('\n').map((l) => ` │ ${l}`).join('\n')}`;
70
71
  throw error;
71
72
  }
72
73
  finally {
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "toolmenu",
3
- "version": "0.25.1",
3
+ "version": "0.27.0",
4
4
  "mcpName": "io.github.niksa90/toolmenu",
5
- "description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
5
+ "description": "Test your MCP server the way agents and clients meet it: the menu, the calls, the results, the sign-in, and every release after. No LLM involved.",
6
6
  "author": "Nikša Perović (https://niksa.me)",
7
7
  "main": "dist/index.js",
8
8
  "scripts": {
@@ -18,7 +18,9 @@
18
18
  "keywords": [
19
19
  "mcp",
20
20
  "model-context-protocol",
21
- "linter",
21
+ "testing",
22
+ "interoperability",
23
+ "conformance",
22
24
  "agents",
23
25
  "prompt-caching",
24
26
  "cli",