toolmenu 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +150 -30
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +61 -18
- package/dist/auto.d.ts +71 -7
- package/dist/auto.js +267 -13
- package/dist/breakdown.d.ts +35 -6
- package/dist/breakdown.js +179 -23
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +116 -178
- package/dist/compare.js +20 -5
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +35 -1
- package/dist/diff.js +1132 -121
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +55 -0
- package/dist/failures.js +103 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +100 -5
- package/dist/init.js +520 -26
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/probe.d.ts +15 -3
- package/dist/probe.js +35 -11
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +89 -43
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +146 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +15 -5
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.d.ts +2 -0
- package/dist/rules/spec.js +61 -8
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/semver.js +1 -1
- package/dist/session.d.ts +56 -7
- package/dist/session.js +800 -94
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +25 -7
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/session.js
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
|
-
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
|
-
import {
|
|
3
|
+
import { cacheBreak, canonical, compareMenus } from './compare.js';
|
|
4
|
+
import { listTools } from './connect.js';
|
|
5
5
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
|
-
import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
|
+
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
-
import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
|
|
8
|
+
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
9
9
|
import { SEVERITY_RANK } from './types.js';
|
|
10
|
-
import {
|
|
10
|
+
import { leadingJson } from './catalog.js';
|
|
11
|
+
import { describeToolDifference } from './difference.js';
|
|
12
|
+
import { autoNextStep, neededValues, writeSign } from './auto.js';
|
|
13
|
+
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
11
14
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
15
|
+
/** A server saying the session is gone: "Session not found or expired" (toolception). */
|
|
16
|
+
const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
|
|
12
17
|
/** Read and check a scenario file. Mistakes are reported before anything runs. */
|
|
13
18
|
export async function loadScenario(path) {
|
|
14
19
|
let data;
|
|
@@ -28,6 +33,10 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
28
33
|
if (doc.allow_writes !== undefined && typeof doc.allow_writes !== 'boolean') {
|
|
29
34
|
throw new Error(`${where}: allow_writes must be true or false`);
|
|
30
35
|
}
|
|
36
|
+
const assumed = doc.assume_read_only;
|
|
37
|
+
if (assumed !== undefined && (!Array.isArray(assumed) || !assumed.every((x) => typeof x === 'string' && x && !/[*?[\]]/.test(x)))) {
|
|
38
|
+
throw new Error(`${where}: assume_read_only must be a list of exact tool names (no patterns)`);
|
|
39
|
+
}
|
|
31
40
|
const steps = doc.steps.map((raw, i) => {
|
|
32
41
|
const at = `${where}: step ${i + 1}`;
|
|
33
42
|
if (raw === 'list' || (raw && typeof raw === 'object' && 'list' in raw))
|
|
@@ -52,7 +61,7 @@ export function parseScenario(data, where = 'scenario') {
|
|
|
52
61
|
}
|
|
53
62
|
throw new Error(`${at}: expected "list", "call: <tool>" or "wait_for: tools_list_changed"`);
|
|
54
63
|
});
|
|
55
|
-
return { allowWrites: doc.allow_writes === true, steps };
|
|
64
|
+
return { allowWrites: doc.allow_writes === true, steps, ...(Array.isArray(assumed) && assumed.length ? { assumeReadOnly: assumed } : {}) };
|
|
56
65
|
}
|
|
57
66
|
export function stepLabel(step) {
|
|
58
67
|
if (step.kind === 'list')
|
|
@@ -69,7 +78,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
69
78
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
70
79
|
// a probe with a different seed would count ordering variance as a change.
|
|
71
80
|
const mainTarget = seeded(target, MAIN_SEED);
|
|
72
|
-
const
|
|
81
|
+
const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
|
|
82
|
+
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
83
|
+
// out and they're sent again.
|
|
84
|
+
const conn = await connectPatiently(mainTarget, { timeoutMs, waits });
|
|
73
85
|
try {
|
|
74
86
|
const era = conn.era;
|
|
75
87
|
const modern = era === 'modern';
|
|
@@ -77,7 +89,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
77
89
|
let listening = !modern; // 2025-era notifications arrive on the session itself
|
|
78
90
|
if (modern && declared) {
|
|
79
91
|
try {
|
|
80
|
-
await conn.client.listen({ toolsListChanged: true });
|
|
92
|
+
await patiently(() => conn.client.listen({ toolsListChanged: true }), { waits, error: tooMany });
|
|
81
93
|
listening = true;
|
|
82
94
|
}
|
|
83
95
|
catch {
|
|
@@ -85,41 +97,152 @@ export async function session(target, scenario, options = {}) {
|
|
|
85
97
|
}
|
|
86
98
|
}
|
|
87
99
|
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
88
|
-
|
|
100
|
+
const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
|
|
101
|
+
// A refusal that outlasts the waits, or a write's that isn't sent again, ends
|
|
102
|
+
// the run with one finding, not one per step. `tool`: the write not sent again.
|
|
103
|
+
let limitedAt;
|
|
104
|
+
const limited = (index, why, waitedMs, tool) => {
|
|
105
|
+
limitedAt = index;
|
|
106
|
+
const said = serverWords(why);
|
|
107
|
+
const last = scenario.steps.length;
|
|
108
|
+
const unchecked = index < last ? `Steps ${index}–${last} weren't checked.` : `Step ${index} wasn't checked.`;
|
|
109
|
+
raw.push({
|
|
110
|
+
rule: 'session/rate-limited',
|
|
111
|
+
severity: 'error',
|
|
112
|
+
step: index,
|
|
113
|
+
...(tool ? { tool } : {}),
|
|
114
|
+
message: tool
|
|
115
|
+
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked}`
|
|
116
|
+
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} The limit counts every request from this address, toolmenu's included.`,
|
|
117
|
+
fix: tool
|
|
118
|
+
? "If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota."
|
|
119
|
+
: 'Give the run its own server instance, or raise the limit for it.',
|
|
120
|
+
});
|
|
121
|
+
};
|
|
122
|
+
let current = await listPatiently();
|
|
89
123
|
// Map keeps first-insertion order; set() on a known name updates it in place.
|
|
90
124
|
const seen = new Map();
|
|
91
125
|
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
92
126
|
see(current);
|
|
93
127
|
const baseline = current;
|
|
128
|
+
// Values that differ from one tools/list to the next on their own (a default
|
|
129
|
+
// computed from the clock: PayPal's list_transactions.end_date), by tool and
|
|
130
|
+
// path, with the finding that reported them first. A later change only there
|
|
131
|
+
// has the same cause and points to it (SPEC §25: one root cause, one finding).
|
|
132
|
+
const varying = new Map();
|
|
133
|
+
const learn = (before, after, changes, source) => {
|
|
134
|
+
for (const [key] of variedPaths(before, after, changes))
|
|
135
|
+
if (!varying.has(key))
|
|
136
|
+
varying.set(key, source);
|
|
137
|
+
};
|
|
94
138
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
95
139
|
let connectionCheck;
|
|
96
|
-
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
|
|
140
|
+
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
|
|
97
141
|
if (probe.error) {
|
|
98
|
-
raw.push({
|
|
142
|
+
raw.push({
|
|
143
|
+
rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance',
|
|
144
|
+
severity: 'info',
|
|
145
|
+
step: 0,
|
|
146
|
+
message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}`,
|
|
147
|
+
fix: `If the server can only run ${target.kind === 'stdio' ? 'once at a time' : 'one connection at a time'}, rerun with --processes 1: the check is then skipped on purpose.`,
|
|
148
|
+
});
|
|
99
149
|
continue;
|
|
100
150
|
}
|
|
101
151
|
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
102
152
|
connectionCheck = f ? 'different' : connectionCheck ?? 'same';
|
|
103
153
|
if (f) {
|
|
104
|
-
|
|
154
|
+
const changes = compareMenus(current.tools, probe.tools ?? []);
|
|
155
|
+
raw.push({ ...f, step: 0, ...(f.fix ? {} : varianceFix(current.tools, probe.tools ?? [], changes)) });
|
|
156
|
+
learn(current.tools, probe.tools ?? [], changes, { rule: f.rule, step: 0 });
|
|
105
157
|
break;
|
|
106
158
|
}
|
|
107
159
|
}
|
|
108
160
|
const steps = [];
|
|
161
|
+
const scopeUnchecked = [];
|
|
162
|
+
// processes: 1 turns off every second process or connection, the scope probe too.
|
|
163
|
+
const probes = (options.processes ?? 2) > 1;
|
|
164
|
+
// The step after which toolmenu last opened a second process or connection (0:
|
|
165
|
+
// before the first step). A server that keeps one session per client can end
|
|
166
|
+
// this one when that happens.
|
|
167
|
+
let probedAfter = probes ? 0 : undefined;
|
|
168
|
+
const lost = (why, index) => {
|
|
169
|
+
if (probedAfter === undefined || !SESSION_LOST.test(why))
|
|
170
|
+
return false;
|
|
171
|
+
// The server's own words, not the transport's wrapping of its JSON-RPC error.
|
|
172
|
+
const said = serverWords(why);
|
|
173
|
+
raw.push({
|
|
174
|
+
rule: 'session/session-lost',
|
|
175
|
+
severity: 'error',
|
|
176
|
+
step: index,
|
|
177
|
+
confidence: 'unsure',
|
|
178
|
+
message: `The server ended this session at step ${index} (“${said}”), after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that; the rest of the scenario didn't run.`,
|
|
179
|
+
fix: 'Rerun with --processes 1: toolmenu then opens no second one.',
|
|
180
|
+
});
|
|
181
|
+
return true;
|
|
182
|
+
};
|
|
183
|
+
const callWaited = { ms: 0 };
|
|
184
|
+
const listWaited = { ms: 0 };
|
|
185
|
+
const assumed = new Set(scenario.assumeReadOnly ?? []);
|
|
186
|
+
// Called on the user's word (assume_read_only), by step.
|
|
187
|
+
const onWord = [];
|
|
188
|
+
// The same failure from the same tool at several steps is one finding that lists them.
|
|
189
|
+
const failures = new Map();
|
|
190
|
+
const fail = (rule, severity, index, step, text, failure, timedOutMs) => {
|
|
191
|
+
const key = `${rule}\0${step.tool}\0${text}`;
|
|
192
|
+
const before = failures.get(key);
|
|
193
|
+
if (before)
|
|
194
|
+
before.steps.push(index);
|
|
195
|
+
else {
|
|
196
|
+
const params = Object.keys(current.tools.find((t) => t.name === step.tool)?.inputSchema?.properties ?? {});
|
|
197
|
+
failures.set(key, { rule, severity, tool: step.tool, text, steps: [index], args: step.args, failure, takesParams: params.length > 0, ...(timedOutMs ? { timedOutMs } : {}) });
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
// Steps whose change touched only values already known to vary on their own.
|
|
201
|
+
const explained = [];
|
|
202
|
+
// --auto's guessed unlocks: tried once, and those whose first call changed nothing.
|
|
203
|
+
const tried = new Set();
|
|
204
|
+
const quiet = new Set();
|
|
205
|
+
const stopped = new Map();
|
|
109
206
|
for (const [i, step] of scenario.steps.entries()) {
|
|
110
207
|
const index = i + 1;
|
|
208
|
+
if (step.kind === 'call' && step.tentative) {
|
|
209
|
+
if (quiet.has(step.tool)) {
|
|
210
|
+
stopped.set(step.tool, (stopped.get(step.tool) ?? 0) + 1);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
const firstTry = step.kind === 'call' && step.tentative === true && !tried.has(step.tool);
|
|
215
|
+
if (firstTry)
|
|
216
|
+
tried.add(step.tool);
|
|
111
217
|
const label = stepLabel(step);
|
|
112
218
|
const record = { index, label, status: 'ok', changed: false, listChanged: 0, tools: current.tools.length, tokens: current.totalTokens };
|
|
113
219
|
const mark = conn.wire.notifications.length;
|
|
220
|
+
let word = false;
|
|
221
|
+
const done = () => {
|
|
222
|
+
if (word)
|
|
223
|
+
record.note = record.note ? `on your word (not marked readOnlyHint); ${record.note}` : 'on your word (not marked readOnlyHint)';
|
|
224
|
+
steps.push(record);
|
|
225
|
+
};
|
|
114
226
|
if (step.kind === 'call') {
|
|
115
227
|
const tool = current.tools.find((t) => t.name === step.tool);
|
|
228
|
+
const write = tool ? writeSign(tool) : undefined;
|
|
229
|
+
word = !!tool && tool.annotations?.readOnlyHint !== true && assumed.has(step.tool) && !write && !scenario.allowWrites;
|
|
116
230
|
if (!tool) {
|
|
117
231
|
record.status = 'failed';
|
|
118
232
|
record.reason = 'missing';
|
|
119
233
|
record.note = `${step.tool} isn't in the menu at this point`;
|
|
120
|
-
|
|
234
|
+
const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
|
|
235
|
+
const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
|
|
236
|
+
raw.push({
|
|
237
|
+
rule: 'session/step-failed',
|
|
238
|
+
severity: 'error',
|
|
239
|
+
step: index,
|
|
240
|
+
tool: step.tool,
|
|
241
|
+
message: `${step.tool} isn't in the menu at step ${index}, so it wasn't called.`,
|
|
242
|
+
fix: close ? `Did you mean ${close.name}? Fix the name in the scenario.` : `Check the name, or move the call after the step that adds ${step.tool}.`,
|
|
243
|
+
});
|
|
121
244
|
}
|
|
122
|
-
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true) {
|
|
245
|
+
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
|
|
123
246
|
record.status = 'refused';
|
|
124
247
|
record.reason = 'refused';
|
|
125
248
|
record.note = 'not marked readOnlyHint';
|
|
@@ -128,39 +251,66 @@ export async function session(target, scenario, options = {}) {
|
|
|
128
251
|
severity: 'error',
|
|
129
252
|
step: index,
|
|
130
253
|
tool: step.tool,
|
|
131
|
-
message:
|
|
254
|
+
message: assumed.has(step.tool) && write
|
|
255
|
+
? `Refused to call ${step.tool}: ${write}, and assume_read_only doesn't override the server's own marking. Session calls tools for real.`
|
|
256
|
+
: `Refused to call ${step.tool}: it isn't marked readOnlyHint, and session calls tools for real.`,
|
|
257
|
+
fix: write || tool.annotations?.readOnlyHint === false
|
|
258
|
+
? 'Set "allow_writes: true" in the scenario if calling it is intended.'
|
|
259
|
+
: `If it only reads, list it under assume_read_only in the scenario; if it writes and that's intended, set "allow_writes: true".`,
|
|
132
260
|
});
|
|
133
261
|
}
|
|
262
|
+
if (word)
|
|
263
|
+
onWord.push({ tool: step.tool, step: index });
|
|
134
264
|
if (record.status === 'ok')
|
|
135
265
|
try {
|
|
136
|
-
|
|
266
|
+
// Sent again after a refusal only when that repeats nothing: the transport's
|
|
267
|
+
// 429 means the server never ran it; a tool that says it was rate-limited,
|
|
268
|
+
// thrown or as its result, may have done part of the work first, so only a
|
|
269
|
+
// read-only one is asked again. On the server's word, not the user's.
|
|
270
|
+
const readOnly = tool?.annotations?.readOnlyHint === true;
|
|
271
|
+
callWaited.ms = 0;
|
|
272
|
+
const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
|
|
273
|
+
waits,
|
|
274
|
+
waited: callWaited,
|
|
275
|
+
error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
|
|
276
|
+
result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
|
|
277
|
+
});
|
|
137
278
|
if (result.isError) {
|
|
138
|
-
const text = (
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
.
|
|
142
|
-
.
|
|
279
|
+
const text = errorText(result);
|
|
280
|
+
if (RATE_LIMITED.test(text)) {
|
|
281
|
+
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
282
|
+
record.status = 'failed';
|
|
283
|
+
record.note = 'rate-limited';
|
|
284
|
+
done();
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
143
287
|
const short = clip(text, 120);
|
|
144
288
|
record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
145
289
|
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
146
290
|
// A setup failure is reported once, for the whole run (session/untested).
|
|
147
291
|
if (!SETUP_FAILURES.has(record.failure))
|
|
148
|
-
|
|
149
|
-
rule: 'session/tool-error',
|
|
150
|
-
severity: 'warn',
|
|
151
|
-
step: index,
|
|
152
|
-
tool: step.tool,
|
|
153
|
-
message: `Step ${index} (${label}) returned an error${text ? ` (“${short}”)` : ''}. A step that fails tests less than it looks, so a clean run can hide a broken setup: check credentials and arguments.`,
|
|
154
|
-
});
|
|
292
|
+
fail('session/tool-error', 'warn', index, step, text, record.failure);
|
|
155
293
|
}
|
|
156
294
|
}
|
|
157
295
|
catch (error) {
|
|
158
296
|
record.status = 'failed';
|
|
159
|
-
record.note = error
|
|
297
|
+
record.note = serverWords(error, 200);
|
|
298
|
+
if (lost(record.note, index)) {
|
|
299
|
+
done();
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
if (tooMany(error)) {
|
|
303
|
+
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
304
|
+
record.note = 'rate-limited';
|
|
305
|
+
done();
|
|
306
|
+
break;
|
|
307
|
+
}
|
|
160
308
|
const code = error.code;
|
|
161
309
|
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
310
|
+
// The SDK's own request timeout: toolmenu stopped waiting, the server didn't refuse anything.
|
|
311
|
+
const timedOutMs = code === 'REQUEST_TIMEOUT' ? timeoutMs : undefined;
|
|
162
312
|
if (!SETUP_FAILURES.has(record.failure))
|
|
163
|
-
|
|
313
|
+
fail('session/step-failed', 'error', index, step, record.note, record.failure, timedOutMs);
|
|
164
314
|
}
|
|
165
315
|
}
|
|
166
316
|
else if (step.kind === 'wait_for') {
|
|
@@ -174,44 +324,87 @@ export async function session(target, scenario, options = {}) {
|
|
|
174
324
|
// have changed the menu, and the change belongs to this step, not the next.
|
|
175
325
|
let next;
|
|
176
326
|
try {
|
|
177
|
-
|
|
327
|
+
listWaited.ms = 0;
|
|
328
|
+
next = await listPatiently(listWaited);
|
|
178
329
|
}
|
|
179
330
|
catch (error) {
|
|
180
|
-
const why = error
|
|
331
|
+
const why = serverWords(error, 200);
|
|
181
332
|
record.status = 'failed';
|
|
182
333
|
record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
|
|
183
|
-
|
|
184
|
-
|
|
334
|
+
if (lost(why, index)) {
|
|
335
|
+
done();
|
|
336
|
+
break;
|
|
337
|
+
}
|
|
338
|
+
if (tooMany(error)) {
|
|
339
|
+
limited(index, why, listWaited.ms);
|
|
340
|
+
record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
|
|
341
|
+
done();
|
|
342
|
+
break;
|
|
343
|
+
}
|
|
344
|
+
raw.push({
|
|
345
|
+
rule: 'session/step-failed',
|
|
346
|
+
severity: 'error',
|
|
347
|
+
step: index,
|
|
348
|
+
message: `After step ${index} (${label}), listing the menu failed: ${why}. The menu after this step wasn't checked.`,
|
|
349
|
+
fix: "Check whether the step crashed or disconnected the server (its stderr), then rerun.",
|
|
350
|
+
});
|
|
351
|
+
done();
|
|
185
352
|
continue;
|
|
186
353
|
}
|
|
187
354
|
const changes = compareMenus(current.tools, next.tools);
|
|
188
355
|
if (changes.length) {
|
|
189
356
|
record.changed = true;
|
|
190
|
-
|
|
191
|
-
|
|
357
|
+
const paths = variedPaths(current.tools, next.tools, changes);
|
|
358
|
+
const onItsOwn = paths.size > 0 && changes.every((c) => EDIT_KINDS.has(c.kind)) && [...paths.keys()].every((k) => varying.has(k));
|
|
359
|
+
if (onItsOwn) {
|
|
360
|
+
// Same cause as a finding already made: said once, after the steps.
|
|
361
|
+
explained.push({ step: index, source: varying.get([...paths.keys()][0]), seen: [...paths].map(([k, d]) => `${k.replace('\0', ': ')}: ${show(d.before)} vs ${show(d.after)}`) });
|
|
362
|
+
const cause = `only where it varies on its own, see ${explained[explained.length - 1].source.rule}`;
|
|
363
|
+
record.note = record.note ? `${record.note}; ${cause}` : cause;
|
|
192
364
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
365
|
+
else {
|
|
366
|
+
if (conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
367
|
+
await waitFor(() => conn.wire.notificationsSince(mark, LIST_CHANGED) > 0, grace);
|
|
368
|
+
}
|
|
369
|
+
const origin = step.kind === 'list'
|
|
370
|
+
? 'no tool call in between'
|
|
371
|
+
: record.reason === 'missing'
|
|
372
|
+
? 'no tool call made (the tool wasn\'t in the menu)'
|
|
373
|
+
: record.reason === 'refused'
|
|
374
|
+
? 'no tool call made (refused)'
|
|
375
|
+
: record.status === 'failed'
|
|
376
|
+
? 'the call failed, but the menu changed: the server may have applied it anyway'
|
|
377
|
+
: undefined;
|
|
378
|
+
raw.push(...changeFindings(current.tools, next.tools, changes, index, origin));
|
|
379
|
+
// With no call in between, whatever changed varies on its own: a later
|
|
380
|
+
// change only there has the same cause.
|
|
381
|
+
if (step.kind === 'list' || record.reason !== undefined)
|
|
382
|
+
learn(current.tools, next.tools, changes, { rule: 'session/edit', step: index });
|
|
383
|
+
if (declared && listening && conn.wire.notificationsSince(mark, LIST_CHANGED) === 0) {
|
|
384
|
+
raw.push({
|
|
385
|
+
rule: 'session/unannounced',
|
|
386
|
+
severity: 'warn',
|
|
387
|
+
step: index,
|
|
388
|
+
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to fetch it again; the spec says servers SHOULD send it.`,
|
|
389
|
+
fix: 'Send notifications/tools/list_changed whenever the tool list changes.',
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
if (step.kind !== 'list' && probes) {
|
|
393
|
+
// The main process is still running here: a server that holds a file or
|
|
394
|
+
// a port can't start a second copy. That leaves the scope unknown, not
|
|
395
|
+
// the run failed.
|
|
396
|
+
const probeWaited = { ms: 0 };
|
|
397
|
+
try {
|
|
398
|
+
probedAfter = index;
|
|
399
|
+
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
400
|
+
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
401
|
+
raw.push(...scopeFindings(record.scope, modern, index));
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
record.scope = 'unclear';
|
|
405
|
+
scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
|
|
406
|
+
}
|
|
407
|
+
}
|
|
215
408
|
}
|
|
216
409
|
current = next;
|
|
217
410
|
see(next);
|
|
@@ -219,9 +412,32 @@ export async function session(target, scenario, options = {}) {
|
|
|
219
412
|
record.listChanged = conn.wire.notificationsSince(mark, LIST_CHANGED);
|
|
220
413
|
record.tools = current.tools.length;
|
|
221
414
|
record.tokens = current.totalTokens;
|
|
222
|
-
|
|
415
|
+
if (firstTry && !record.changed && record.status === 'ok' && !record.failure)
|
|
416
|
+
quiet.add(step.tool);
|
|
417
|
+
done();
|
|
223
418
|
}
|
|
419
|
+
raw.push(...failureFindings(sameError([...failures.values()]), options.auto));
|
|
420
|
+
if (explained.length)
|
|
421
|
+
raw.push(knownVariance(explained));
|
|
422
|
+
if (onWord.length)
|
|
423
|
+
raw.push(onTheirWord(onWord, options.auto !== undefined));
|
|
224
424
|
raw.push(...untested(steps, target));
|
|
425
|
+
// --auto means "find what you can": an unlock it skipped is a gap too.
|
|
426
|
+
const leftOut = [...stopped].map(([tool, calls]) => ({ tool, calls }));
|
|
427
|
+
// An unlock --auto stopped trying isn't charged for the values it then left out.
|
|
428
|
+
const covered = baseline.tools.filter((t) => !stopped.has(t.name));
|
|
429
|
+
raw.push(...unlockCoverage(covered, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
|
|
430
|
+
if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
|
|
431
|
+
raw.push(nothingCalled(options.auto));
|
|
432
|
+
if (scopeUnchecked.length) {
|
|
433
|
+
const which = scopeUnchecked.map((s) => s.step);
|
|
434
|
+
raw.push({
|
|
435
|
+
rule: 'session/scope-unchecked',
|
|
436
|
+
severity: 'info',
|
|
437
|
+
step: which[0],
|
|
438
|
+
message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} while the session ran, so whether the change${which.length === 1 ? ` at step ${which[0]}` : `s at steps ${which.join(', ')}`} reach${which.length === 1 ? 'es' : ''} a fresh ${target.kind === 'stdio' ? 'process' : 'connection'} wasn't checked: ${scopeUnchecked[0].why}`,
|
|
439
|
+
});
|
|
440
|
+
}
|
|
225
441
|
return {
|
|
226
442
|
scenario: options.scenarioName ?? 'scenario',
|
|
227
443
|
server: baseline.server,
|
|
@@ -231,7 +447,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
231
447
|
final: { tools: current.tools.length, tokens: current.totalTokens },
|
|
232
448
|
connectionCheck,
|
|
233
449
|
steps,
|
|
234
|
-
findings: settle(
|
|
450
|
+
findings: settle(mergeRepeats(raw), options),
|
|
451
|
+
...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
|
|
235
452
|
union: buildMenu([...seen.values()].map(toolDefinition), baseline.server),
|
|
236
453
|
};
|
|
237
454
|
}
|
|
@@ -254,19 +471,311 @@ function untested(steps, target) {
|
|
|
254
471
|
for (const s of setup)
|
|
255
472
|
byClass.set(s.failure, [...(byClass.get(s.failure) ?? []), s]);
|
|
256
473
|
const all = setup.length === calls.length;
|
|
257
|
-
|
|
474
|
+
// One step per class seen, not just the first: a run can fail on credentials and a missing browser at once.
|
|
475
|
+
const { fix } = setupFailureAdvice(byClass.keys(), target.kind);
|
|
258
476
|
return [
|
|
259
477
|
{
|
|
260
478
|
rule: 'session/untested',
|
|
261
479
|
severity: all ? 'error' : 'warn',
|
|
262
|
-
|
|
480
|
+
// Classified from the error's words: a guess, if a good one.
|
|
481
|
+
confidence: 'unsure',
|
|
482
|
+
message: `${all ? `All ${calls.length}` : `${setup.length} of ${calls.length}`} tool calls failed before reaching the tool: ${[...byClass].map(([c, list]) => `${list.length} on ${FAILURE_LABELS[c]}`).join(', ')}. Those steps only tested whether a failed call changes the menu.`,
|
|
263
483
|
detail: [...byClass].map(([c, list]) => `${c}: steps ${list.map((s) => s.index).join(', ')} (“${(list[0].note ?? '').replace(/^the tool returned an error: /, '').slice(0, 100)}”)`),
|
|
484
|
+
fix,
|
|
264
485
|
},
|
|
265
486
|
];
|
|
266
487
|
}
|
|
488
|
+
/**
|
|
489
|
+
* Unlocks whose schema lists every value (an enum), and how many of those values
|
|
490
|
+
* the run got through. The session sees, and --union-out keeps, only the tools
|
|
491
|
+
* behind the values it unlocked: GitHub's enable_toolset has 19, the first starter
|
|
492
|
+
* unlocked 2, and 72 of 81 tools never reached the baseline.
|
|
493
|
+
*/
|
|
494
|
+
function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], stoppedAt) {
|
|
495
|
+
const out = [];
|
|
496
|
+
// A run a rate limit stopped (session/rate-limited says so) is charged only for
|
|
497
|
+
// values its scenario leaves out, not for ones in steps it never got to.
|
|
498
|
+
const unrun = stoppedAt === undefined ? [] : scenario.steps.map((step, i) => ({ step, index: i + 1 })).filter((s) => s.index >= stoppedAt);
|
|
499
|
+
for (const u of unlockers(menu)) {
|
|
500
|
+
if (!u.param || u.values.length === 0)
|
|
501
|
+
continue;
|
|
502
|
+
const reached = new Set();
|
|
503
|
+
let last;
|
|
504
|
+
const add = (arg) => {
|
|
505
|
+
for (const v of Array.isArray(arg) ? arg : [arg])
|
|
506
|
+
if (v !== undefined)
|
|
507
|
+
reached.add(canonical(v));
|
|
508
|
+
};
|
|
509
|
+
for (const record of steps) {
|
|
510
|
+
const step = scenario.steps[record.index - 1];
|
|
511
|
+
if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure)
|
|
512
|
+
continue;
|
|
513
|
+
last = record.index;
|
|
514
|
+
add(step.args[u.param]);
|
|
515
|
+
}
|
|
516
|
+
for (const { step } of unrun)
|
|
517
|
+
if (step.kind === 'call' && step.tool === u.tool.name)
|
|
518
|
+
add(step.args[u.param]);
|
|
519
|
+
const missing = u.values.filter((v) => !reached.has(canonical(v)));
|
|
520
|
+
// Never called: a scenario about something else, unless it's building the baseline.
|
|
521
|
+
if (missing.length === 0 || (reached.size === 0 && !baselineWanted))
|
|
522
|
+
continue;
|
|
523
|
+
const got = u.values.length - missing.length;
|
|
524
|
+
const skip = skipped.find((s) => s.tool === u.tool.name)?.reason;
|
|
525
|
+
const why = skip === 'open world'
|
|
526
|
+
? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
|
|
527
|
+
: skip === 'not read-only'
|
|
528
|
+
? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
|
|
529
|
+
: skip === 'unmarked'
|
|
530
|
+
? ` --auto skipped it: the server doesn't mark it readOnlyHint.`
|
|
531
|
+
: '';
|
|
532
|
+
out.push({
|
|
533
|
+
rule: 'session/unlock-coverage',
|
|
534
|
+
severity: 'warn',
|
|
535
|
+
tool: u.tool.name,
|
|
536
|
+
// "Looks like it unlocks": from its name, parameter and description.
|
|
537
|
+
confidence: 'unsure',
|
|
538
|
+
...(last !== undefined ? { step: last } : {}),
|
|
539
|
+
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}`,
|
|
540
|
+
detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
|
|
541
|
+
fix: skip === 'open world'
|
|
542
|
+
? 'Rerun with --open-world.'
|
|
543
|
+
: skip === 'unmarked'
|
|
544
|
+
? `If it only changes which tools are listed, rerun with --assume-read-only ${u.tool.name}.`
|
|
545
|
+
: skip === 'not read-only'
|
|
546
|
+
? `Write a scenario that unlocks every value (session --init puts each one in the starter) and set allow_writes: true.`
|
|
547
|
+
: 'Unlock every value: session --init puts each one in the starter.',
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
return out;
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* --auto that called nothing: the run only listed the menu, so a clean result
|
|
554
|
+
* says nothing about what the tools do to it. Why each tool was left out, and the
|
|
555
|
+
* flag that reaches the most of them.
|
|
556
|
+
*/
|
|
557
|
+
function nothingCalled(auto) {
|
|
558
|
+
const by = (reason) => auto.skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
559
|
+
const parts = [];
|
|
560
|
+
const openWorld = by('open world');
|
|
561
|
+
const needs = by('needs values');
|
|
562
|
+
const unmarked = by('unmarked');
|
|
563
|
+
const writes = by('not read-only');
|
|
564
|
+
if (needs.length)
|
|
565
|
+
parts.push(`${needs.length} need values the schema doesn't give`);
|
|
566
|
+
if (unmarked.length)
|
|
567
|
+
parts.push(`${unmarked.length} aren't marked readOnlyHint`);
|
|
568
|
+
if (openWorld.length)
|
|
569
|
+
parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
|
|
570
|
+
if (writes.length)
|
|
571
|
+
parts.push(`${writes.length} marked or named as writes`);
|
|
572
|
+
const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
|
|
573
|
+
const params = neededValues(auto.skipped);
|
|
574
|
+
const fix = autoNextStep(auto);
|
|
575
|
+
return {
|
|
576
|
+
rule: 'session/nothing-called',
|
|
577
|
+
severity: 'warn',
|
|
578
|
+
message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
|
|
579
|
+
detail: [
|
|
580
|
+
...(needs.length ? [`need values: ${params.slice(0, 6).map((p) => `${p.param} (${list(p.tools)})`).join('; ')}${params.length > 6 ? `; and ${params.length - 6} more` : ''}`] : []),
|
|
581
|
+
...(unmarked.length ? [`not marked readOnlyHint: ${list(unmarked)}`] : []),
|
|
582
|
+
...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
|
|
583
|
+
],
|
|
584
|
+
...(fix ? { fix } : {}),
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* The same error text from different tools is one finding (two tools of a feature
|
|
589
|
+
* that "isn't set up on this deployment"). An empty text says nothing in common.
|
|
590
|
+
*/
|
|
591
|
+
function sameError(failures) {
|
|
592
|
+
const out = [];
|
|
593
|
+
const byText = new Map();
|
|
594
|
+
for (const f of failures) {
|
|
595
|
+
const key = `${f.rule}\0${f.text}`;
|
|
596
|
+
const first = f.text ? byText.get(key) : undefined;
|
|
597
|
+
if (!first) {
|
|
598
|
+
out.push({ ...f, steps: [...f.steps] });
|
|
599
|
+
if (f.text)
|
|
600
|
+
byText.set(key, out[out.length - 1]);
|
|
601
|
+
continue;
|
|
602
|
+
}
|
|
603
|
+
first.steps = [...first.steps, ...f.steps].sort((a, b) => a - b);
|
|
604
|
+
first.others = [...(first.others ?? []), { tool: f.tool, args: f.args, takesParams: f.takesParams }];
|
|
605
|
+
}
|
|
606
|
+
return out;
|
|
607
|
+
}
|
|
608
|
+
// The server says the feature isn't there, or isn't for this account: no argument
|
|
609
|
+
// fixes that. A guess from its words.
|
|
610
|
+
const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\bpermission\b|\bforbidden\b|\bnot allowed\b|\brole\b/i;
|
|
611
|
+
/** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
|
|
612
|
+
function stepsText(steps) {
|
|
613
|
+
if (steps.length === 1)
|
|
614
|
+
return `step ${steps[0]}`;
|
|
615
|
+
const runs = [];
|
|
616
|
+
for (let i = 0; i < steps.length;) {
|
|
617
|
+
let j = i;
|
|
618
|
+
while (j + 1 < steps.length && steps[j + 1] === steps[j] + 1)
|
|
619
|
+
j++;
|
|
620
|
+
if (j - i >= 2)
|
|
621
|
+
runs.push(`${steps[i]}–${steps[j]}`);
|
|
622
|
+
else
|
|
623
|
+
for (let k = i; k <= j; k++)
|
|
624
|
+
runs.push(String(steps[k]));
|
|
625
|
+
i = j + 1;
|
|
626
|
+
}
|
|
627
|
+
return runs.length === 1 ? `steps ${runs[0]}` : `steps ${runs.slice(0, -1).join(', ')} and ${runs[runs.length - 1]}`;
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* The same error from the same tool is one finding that names every step it
|
|
631
|
+
* happened at (Exa: one bad argument, steps 2 and 3). The error in the server's
|
|
632
|
+
* words, the call that got it, and what to try.
|
|
633
|
+
*/
|
|
634
|
+
function failureFindings(failures, auto) {
|
|
635
|
+
return failures.map((f) => {
|
|
636
|
+
const many = f.steps.length > 1;
|
|
637
|
+
const calls = [f, ...(f.others ?? [])];
|
|
638
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
639
|
+
const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
|
|
640
|
+
const said = f.text ? quotedSentence(clip(f.text, 160)) : 'no error text.';
|
|
641
|
+
const firstArg = Object.keys(f.args)[0];
|
|
642
|
+
const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
|
|
643
|
+
// No argument can fix a call to a tool that takes none, or a feature the server
|
|
644
|
+
// says isn't there. From the server's words: a guess.
|
|
645
|
+
const unavailable = argsCan && f.failure !== 'invalid-arguments' && UNAVAILABLE.test(f.text);
|
|
646
|
+
const them = tools.length === 1 ? 'its' : 'their';
|
|
647
|
+
const leaveOut = auto ? `save the steps with --save-scenario, drop ${them} calls and run that with --scenario` : `drop ${them} calls from the scenario`;
|
|
648
|
+
const noParams = calls.every((c) => !c.takesParams && Object.keys(c.args).length === 0);
|
|
649
|
+
const secs = f.timedOutMs ? `${Math.round(f.timedOutMs / 1000)} s` : '';
|
|
650
|
+
const fix = f.timedOutMs
|
|
651
|
+
? `${who} didn't answer within toolmenu's ${secs} request timeout: the call may just be slow (fetching or searching a lot). Rerun with a longer --timeout (e.g. --timeout ${f.timedOutMs * 4}); if it still times out, the server hangs on this call.`
|
|
652
|
+
: !argsCan
|
|
653
|
+
? undefined
|
|
654
|
+
: unavailable
|
|
655
|
+
? `The server says this feature isn't available here: nothing to change in the call. Test ${tools.length === 1 ? 'it' : 'them'} on a deployment that has it, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
656
|
+
: noParams
|
|
657
|
+
? `${tools.length === 1 ? `${f.tool} takes` : 'These tools take'} no arguments, so nothing in the call can fix this: the error is the server's. Check it on the server side, or leave ${tools.length === 1 ? 'it' : 'them'} out: ${leaveOut}.`
|
|
658
|
+
: auto?.withValues?.includes(f.tool)
|
|
659
|
+
? `Check the values you gave ${f.tool} (--value) against what the server says above.`
|
|
660
|
+
: auto
|
|
661
|
+
? `If the arguments are what it refused, give it real ones: --value ${f.tool}.${firstArg ?? '<param>'}=…${firstArg ? ' (--auto made these up from the schema).' : ''}`
|
|
662
|
+
: `Check the arguments of the call at ${stepsText([f.steps[0]])} in the scenario against what ${f.tool} expects.`;
|
|
663
|
+
const lead = many ? `${stepsText(f.steps).replace(/^s/, 'S')}: ` : '';
|
|
664
|
+
const detail = calls.slice(0, 8).map((c) => {
|
|
665
|
+
const call = `${c.tool} ${JSON.stringify(c.args)}`;
|
|
666
|
+
return `call: ${call.length > 200 ? call.slice(0, 199) + '…' : call}`;
|
|
667
|
+
});
|
|
668
|
+
return {
|
|
669
|
+
rule: f.rule,
|
|
670
|
+
severity: f.severity,
|
|
671
|
+
step: f.steps[0],
|
|
672
|
+
...(many ? { steps: f.steps } : {}),
|
|
673
|
+
tool: f.tool,
|
|
674
|
+
...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
|
|
675
|
+
message: f.rule === 'session/tool-error'
|
|
676
|
+
? tools.length > 1
|
|
677
|
+
? `${lead}${who} returned the same error: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
678
|
+
: `${lead}${f.tool} returned an error${many ? ', the same one each time' : ''}: ${said} A call that fails tests less than it looks: the menu was checked after it, the tool's own work wasn't.`
|
|
679
|
+
: tools.length > 1
|
|
680
|
+
? `${lead}Calling ${who} failed the same way: ${said}`
|
|
681
|
+
: `${lead}Calling ${f.tool} failed${many ? ', the same way each time' : ''}: ${said}`,
|
|
682
|
+
detail: [...new Set(detail)],
|
|
683
|
+
...(fix ? { fix } : {}),
|
|
684
|
+
};
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Changes that touched only values already seen to vary on their own: one info
|
|
689
|
+
* finding for the run, pointing to the one that reported the cause.
|
|
690
|
+
*/
|
|
691
|
+
function knownVariance(explained) {
|
|
692
|
+
const steps = explained.map((e) => e.step);
|
|
693
|
+
const src = explained[0].source;
|
|
694
|
+
const where = src.step === 0 ? 'before step 1' : `at step ${src.step}`;
|
|
695
|
+
// The values at the first such step; the rest only repeat the pattern.
|
|
696
|
+
const seen = explained[0].seen;
|
|
697
|
+
return {
|
|
698
|
+
rule: 'session/known-variance',
|
|
699
|
+
severity: 'info',
|
|
700
|
+
step: steps[0],
|
|
701
|
+
message: `${steps.length > 1 ? `${stepsText(steps).replace(/^s/, 'S')}: the` : 'The'} menu changed again, only in values that already differ from one tools/list to the next. Same cause as ${src.rule} (${where}), so it isn't counted again.`,
|
|
702
|
+
detail: [...seen.slice(0, 4), ...(seen.length > 4 ? [`…and ${seen.length - 4} more`] : [])],
|
|
703
|
+
fix: `Fix the ${src.rule} finding (${where}); these changes go away with it.`,
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
/** Tools called because the user said they only read: said once, loudly. */
|
|
707
|
+
function onTheirWord(calls, auto) {
|
|
708
|
+
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
709
|
+
return {
|
|
710
|
+
rule: 'session/assumed-read-only',
|
|
711
|
+
severity: 'info',
|
|
712
|
+
step: calls[0].step,
|
|
713
|
+
message: `Called ${tools.length === 1 ? `${tools[0]}, which the server doesn't mark readOnlyHint,` : `${tools.length} tools the server doesn't mark readOnlyHint`} on your word (${auto ? '--assume-read-only' : 'assume_read_only'}). If ${tools.length === 1 ? 'it writes, it wrote' : 'one of them writes, it wrote'}.`,
|
|
714
|
+
detail: [`${stepsText([...new Set(calls.map((c) => c.step))])}: ${tools.join(', ')}`],
|
|
715
|
+
fix: `Ask the server's authors to mark ${tools.length === 1 ? 'it' : 'them'} readOnlyHint: true; then ${auto ? `--auto calls ${tools.length === 1 ? 'it' : 'them'} without --assume-read-only` : 'drop assume_read_only'}.`,
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Every leaf value that differs between the old and new definition of each edited
|
|
720
|
+
* tool, keyed `tool\0path` (`list_transactions\0inputSchema.properties.end_date.default`).
|
|
721
|
+
*/
|
|
722
|
+
function variedPaths(before, after, changes) {
|
|
723
|
+
const out = new Map();
|
|
724
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
725
|
+
for (const name of new Set(changes.filter((c) => EDIT_KINDS.has(c.kind)).map((c) => c.tool))) {
|
|
726
|
+
const a = old.get(name);
|
|
727
|
+
const b = after.find((t) => t.name === name);
|
|
728
|
+
if (!a || !b)
|
|
729
|
+
continue;
|
|
730
|
+
for (const [path, d] of leafDiffs(toolDefinition(a), toolDefinition(b)))
|
|
731
|
+
out.set(`${name}\0${path}`, d);
|
|
732
|
+
}
|
|
733
|
+
return out;
|
|
734
|
+
}
|
|
735
|
+
function leafDiffs(a, b, path = '', out = new Map()) {
|
|
736
|
+
if (canonical(a) === canonical(b))
|
|
737
|
+
return out;
|
|
738
|
+
const obj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
739
|
+
if (obj(a) && obj(b)) {
|
|
740
|
+
for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
|
|
741
|
+
leafDiffs(a[key], b[key], path ? `${path}.${key}` : key, out);
|
|
742
|
+
}
|
|
743
|
+
else if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
|
|
744
|
+
a.forEach((x, i) => leafDiffs(x, b[i], `${path}[${i}]`, out));
|
|
745
|
+
}
|
|
746
|
+
else
|
|
747
|
+
out.set(path, { before: a, after: b });
|
|
748
|
+
return out;
|
|
749
|
+
}
|
|
750
|
+
function show(value) {
|
|
751
|
+
if (value === undefined)
|
|
752
|
+
return '(absent)';
|
|
753
|
+
const text = JSON.stringify(value);
|
|
754
|
+
return text.length > 60 ? text.slice(0, 57) + '…' : text;
|
|
755
|
+
}
|
|
756
|
+
const TIMESTAMP = /^\d{4}-\d{2}-\d{2}(T|\s)\d{2}:\d{2}/;
|
|
757
|
+
/** The next step for a menu that differs between processes, from what differs. */
|
|
758
|
+
function varianceFix(main, other, changes) {
|
|
759
|
+
const diffs = [...variedPaths(main, other, changes)];
|
|
760
|
+
if (diffs.length === 0)
|
|
761
|
+
return changes.every((c) => c.kind === 'moved') ? { fix: 'Return the tools in a fixed order (sorted, or as registered).' } : {};
|
|
762
|
+
const where = diffs.slice(0, 2).map(([k]) => k.replace('\0', '.')).join(', ');
|
|
763
|
+
if (diffs.every(([, d]) => typeof d.before === 'string' && typeof d.after === 'string' && TIMESTAMP.test(d.before) && TIMESTAMP.test(d.after))) {
|
|
764
|
+
return { fix: `Build ${where}${diffs.length > 2 ? ' and the rest' : ''} from a fixed value, not the current time (or leave the default out and say it in the description).` };
|
|
765
|
+
}
|
|
766
|
+
return { fix: `Make ${where}${diffs.length > 2 ? ' and the rest' : ''} the same on every tools/list: sort what comes from a set or a map, and build nothing from the clock or a random value.` };
|
|
767
|
+
}
|
|
267
768
|
function serverOf(c) {
|
|
268
769
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
269
770
|
}
|
|
771
|
+
/** A tool result's text, whitespace collapsed. */
|
|
772
|
+
function errorText(result) {
|
|
773
|
+
return (Array.isArray(result.content) ? result.content : [])
|
|
774
|
+
.map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
|
|
775
|
+
.join(' ')
|
|
776
|
+
.replace(/\s+/g, ' ')
|
|
777
|
+
.trim();
|
|
778
|
+
}
|
|
270
779
|
function waitFor(done, ms) {
|
|
271
780
|
return new Promise((resolve) => {
|
|
272
781
|
if (done())
|
|
@@ -284,6 +793,7 @@ function waitFor(done, ms) {
|
|
|
284
793
|
}, 20);
|
|
285
794
|
});
|
|
286
795
|
}
|
|
796
|
+
const APPEND_WHY = " The end of the tool list isn't the end of the prompt: most clients send tools first (Claude's Messages API does), so any change to them, an append too, invalidates the cached conversation after them. Appends are cache-safe only if your client adds new tools after the cached content, as tool search (deferred loading) does.";
|
|
287
797
|
const EDIT_KINDS = new Set(['description', 'inputSchema', 'outputSchema', 'annotations', 'other', 'serialization']);
|
|
288
798
|
/** Turn a menu change into findings, with the cost of each. */
|
|
289
799
|
export function changeFindings(before, after, changes, step, origin) {
|
|
@@ -311,8 +821,10 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
311
821
|
rule: 'session/append',
|
|
312
822
|
severity: 'warn',
|
|
313
823
|
step,
|
|
314
|
-
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')})
|
|
824
|
+
message: `+${names.length} tool${names.length === 1 ? '' : 's'} appended at the end of the list (${names.join(', ')}).${APPEND_WHY}`,
|
|
315
825
|
detail: [`~${tokens.toLocaleString('en-US')} new tokens (estimate)`, ...why],
|
|
826
|
+
each: `step ${step}: +${names.length} tool${names.length === 1 ? '' : 's'} (~${tokens.toLocaleString('en-US')} tokens): ${names.slice(0, 8).join(', ')}${names.length > 8 ? ` and ${names.length - 8} more` : ''}`,
|
|
827
|
+
fix: 'If clients should keep their cache, list these tools from the start, or have clients load them through tool search.',
|
|
316
828
|
});
|
|
317
829
|
}
|
|
318
830
|
else {
|
|
@@ -323,31 +835,63 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
323
835
|
step,
|
|
324
836
|
message: `+${names.length} tool${names.length === 1 ? '' : 's'} inserted at position ${first} (${names.join(', ')}). Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
325
837
|
detail: [...cost, ...why],
|
|
838
|
+
fix: 'Add new tools at the end of the list, not in the middle; better, list them from the start.',
|
|
326
839
|
});
|
|
327
840
|
}
|
|
328
841
|
}
|
|
329
842
|
const moved = changes.filter((c) => c.kind === 'moved');
|
|
330
843
|
if (moved.length) {
|
|
331
|
-
out.push({
|
|
844
|
+
out.push({
|
|
845
|
+
rule: 'session/reorder',
|
|
846
|
+
severity: 'error',
|
|
847
|
+
step,
|
|
848
|
+
message: `Tool order changed mid-session (${moved.map((m) => m.tool).join(', ')}). Invalidates the cached prompt, although no tool changed.`,
|
|
849
|
+
detail: [...cost, ...why],
|
|
850
|
+
fix: 'Return the tools in one fixed order (sorted, or as registered) on every tools/list.',
|
|
851
|
+
});
|
|
332
852
|
}
|
|
333
853
|
const removed = changes.filter((c) => c.kind === 'removed');
|
|
334
854
|
if (removed.length) {
|
|
335
|
-
out.push({
|
|
855
|
+
out.push({
|
|
856
|
+
rule: 'session/remove',
|
|
857
|
+
severity: 'error',
|
|
858
|
+
step,
|
|
859
|
+
message: `${removed.map((r) => r.tool).join(', ')} disappeared from the menu mid-session. Invalidates the cached prompt, and a model that already saw ${removed.length === 1 ? 'it' : 'them'} may still call ${removed.length === 1 ? 'it' : 'them'}.`,
|
|
860
|
+
detail: [...cost, ...why],
|
|
861
|
+
fix: `Keep ${removed.length === 1 ? 'the tool' : 'the tools'} listed for the whole session; refuse the call with an error that says why instead.`,
|
|
862
|
+
});
|
|
336
863
|
}
|
|
337
864
|
const edits = new Map();
|
|
338
865
|
for (const c of changes.filter((c) => EDIT_KINDS.has(c.kind)))
|
|
339
866
|
edits.set(c.tool, [...(edits.get(c.tool) ?? []), c.kind === 'other' ? 'definition' : c.kind === 'serialization' ? 'key order (same content, different bytes)' : c.kind]);
|
|
867
|
+
const old = new Map(before.map((t) => [t.name, t]));
|
|
340
868
|
for (const [tool, fields] of edits) {
|
|
341
|
-
|
|
869
|
+
const a = old.get(tool);
|
|
870
|
+
const b = after.find((t) => t.name === tool);
|
|
871
|
+
// What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
|
|
872
|
+
const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
|
|
873
|
+
out.push({
|
|
874
|
+
rule: 'session/edit',
|
|
875
|
+
severity: 'error',
|
|
876
|
+
step,
|
|
877
|
+
tool,
|
|
878
|
+
message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
879
|
+
detail: [...seen, ...cost, ...why],
|
|
880
|
+
fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
|
|
881
|
+
});
|
|
342
882
|
}
|
|
343
883
|
return out;
|
|
344
884
|
}
|
|
345
885
|
/**
|
|
346
886
|
* Does a fresh connection see this step's changes? All of them: the server's tool
|
|
347
887
|
* set changed. None: the change belongs to this connection (or, on stdio, this
|
|
348
|
-
* process). Reorders alone can't tell, and a mix is unclear.
|
|
888
|
+
* process). Reorders alone can't tell, and a mix is unclear. So can't a step that
|
|
889
|
+
* takes the menu back to the session's `baseline` while fresh ones still serve it:
|
|
890
|
+
* a fresh process starts there anyway (an unlock undone reads as "seen" by it).
|
|
349
891
|
*/
|
|
350
|
-
export function scopeOf(changes, next, probe, transport) {
|
|
892
|
+
export function scopeOf(changes, next, probe, transport, baseline) {
|
|
893
|
+
if (baseline && compareMenus(baseline, next).length === 0 && compareMenus(baseline, probe).length === 0)
|
|
894
|
+
return 'unclear';
|
|
351
895
|
const probeByName = new Map(probe.map((t) => [t.name, t]));
|
|
352
896
|
const nextByName = new Map(next.map((t) => [t.name, t]));
|
|
353
897
|
const verdicts = new Set();
|
|
@@ -379,6 +923,7 @@ function scopeFindings(scope, modern, step) {
|
|
|
379
923
|
message: modern
|
|
380
924
|
? `The change is local to this connection: a fresh connection with the same credentials still sees the old menu. On 2026-07-28 the tool set MUST NOT vary per-connection or as a side effect of other requests on the connection.`
|
|
381
925
|
: `The change is local to this connection: a fresh connection still sees the old menu. Allowed before 2026-07-28; ruled out from 2026-07-28 on.`,
|
|
926
|
+
fix: 'Serve every connection the same tool set; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
382
927
|
},
|
|
383
928
|
];
|
|
384
929
|
}
|
|
@@ -389,20 +934,41 @@ function scopeFindings(scope, modern, step) {
|
|
|
389
934
|
severity: 'warn',
|
|
390
935
|
step,
|
|
391
936
|
message: `A freshly started server doesn't show this change. On stdio every connection is its own process, so toolmenu can't tell a per-connection change from a global one. On 2026-07-28 the tool set MUST NOT change as a side effect of requests on the connection.`,
|
|
937
|
+
fix: 'List the same tools whatever the calls before; to let a conversation reach more tools, list them all and let clients load them through tool search.',
|
|
392
938
|
},
|
|
393
939
|
];
|
|
394
940
|
}
|
|
395
941
|
return [];
|
|
396
942
|
}
|
|
397
|
-
/**
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
943
|
+
/**
|
|
944
|
+
* Findings that say the same thing at every step are said once, naming the steps:
|
|
945
|
+
* stdio's side-effect warning, and an unlock per domain, where each value appends
|
|
946
|
+
* tools for this connection only (9 steps made 18 near-identical blocks). What
|
|
947
|
+
* differs per step, the tools and their cost, goes in the detail. Mid-inserts,
|
|
948
|
+
* removals and edits stay per step: each is its own change.
|
|
949
|
+
*/
|
|
950
|
+
const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
|
|
951
|
+
function mergeRepeats(raw) {
|
|
952
|
+
let out = raw;
|
|
953
|
+
for (const rule of REPEATS) {
|
|
954
|
+
const same = out.filter((f) => f.rule === rule);
|
|
955
|
+
if (same.length < 2)
|
|
956
|
+
continue;
|
|
957
|
+
const steps = same.map((f) => f.step);
|
|
958
|
+
const lead = rule === 'session/side-effect' ? `Steps ${steps.join(', ')}` : stepsText(steps).replace(/^s/, 'S');
|
|
959
|
+
let message = `${lead}: ${same[0].message}`;
|
|
960
|
+
let detail = same[0].detail;
|
|
961
|
+
if (rule === 'session/append') {
|
|
962
|
+
const added = same.reduce((n, f) => n + Number(/^\+(\d+)/.exec(f.message)?.[1] ?? 0), 0);
|
|
963
|
+
message = `${lead}: +${added} tools appended at the end of the list, over ${same.length} steps.${APPEND_WHY}`;
|
|
964
|
+
const each = same.map((f) => f.each).filter(Boolean);
|
|
965
|
+
const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.endsWith('new tokens (estimate)'))))];
|
|
966
|
+
detail = [...each, ...why];
|
|
967
|
+
}
|
|
968
|
+
const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
|
|
969
|
+
out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
970
|
+
}
|
|
971
|
+
return out;
|
|
406
972
|
}
|
|
407
973
|
function describe(changes) {
|
|
408
974
|
return changes.slice(0, 8).map((c) => `${c.kind}: ${c.tool} (position ${c.position})`).concat(changes.length > 8 ? [`…and ${changes.length - 8} more`] : []);
|
|
@@ -411,7 +977,7 @@ function settle(raw, options) {
|
|
|
411
977
|
const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
412
978
|
return raw
|
|
413
979
|
.filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
|
|
414
|
-
.map((f) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
980
|
+
.map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
415
981
|
.sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
416
982
|
}
|
|
417
983
|
/*
|
|
@@ -424,10 +990,35 @@ function settle(raw, options) {
|
|
|
424
990
|
// A parameter that names what to unlock. Strong names say so alone; weak ones
|
|
425
991
|
// (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
|
|
426
992
|
// description to back them up.
|
|
427
|
-
|
|
428
|
-
const
|
|
993
|
+
// `namespace` is weak too: Kubernetes and Pinecone take one on every read.
|
|
994
|
+
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
|
|
995
|
+
const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
|
|
429
996
|
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
430
|
-
|
|
997
|
+
// A description about loading tools: "Adds the audit tools", "enables more
|
|
998
|
+
// capabilities", "unlocks…". Whole words: "download" isn't "load", and a device's
|
|
999
|
+
// capabilities alone aren't tools.
|
|
1000
|
+
const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
|
|
1001
|
+
// Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
|
|
1002
|
+
// "exposes"/"adds" aren't: a lookup's text uses them for what it lists.
|
|
1003
|
+
const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
|
|
1004
|
+
/**
|
|
1005
|
+
* Does the tool say that `param` unlocks tools? Its name does, or a sentence of
|
|
1006
|
+
* its description that says it unlocks tools and doesn't pin that on another
|
|
1007
|
+
* parameter ("A query match also enables the tools…" is about `query`, not the
|
|
1008
|
+
* `domain` filter next to it).
|
|
1009
|
+
*/
|
|
1010
|
+
function claimsUnlock(tool, param) {
|
|
1011
|
+
if (UNLOCK_NAME.test(tool.name))
|
|
1012
|
+
return true;
|
|
1013
|
+
const stem = (p) => p.toLowerCase().replace(/_?ids?$/, '').replace(/(?:ies)$/, 'y').replace(/s$/, '');
|
|
1014
|
+
const mentions = (sentence, p) => stem(p).length > 1 && new RegExp(`\\b${stem(p).replace(/[^a-z0-9]/g, '.?')}`, 'i').test(sentence);
|
|
1015
|
+
const others = Object.keys(tool.inputSchema?.properties ?? {}).filter((p) => p !== param);
|
|
1016
|
+
return (tool.description ?? '')
|
|
1017
|
+
.split(/(?<=[.!?])\s+/)
|
|
1018
|
+
.filter((s) => UNLOCK_SAYS.test(s))
|
|
1019
|
+
.some((s) => (param && mentions(s, param)) || !others.some((p) => mentions(s, p)));
|
|
1020
|
+
}
|
|
1021
|
+
const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
431
1022
|
/**
|
|
432
1023
|
* At most `max` characters, cut by code point at a word boundary, never inside a
|
|
433
1024
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
@@ -441,10 +1032,15 @@ export function clip(text, max) {
|
|
|
441
1032
|
return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
|
|
442
1033
|
}
|
|
443
1034
|
function isWriteLike(t) {
|
|
1035
|
+
// Named as an unlock: changing the menu is its job, whatever its hints say
|
|
1036
|
+
// (toolception marks enable_toolset destructive). It's still only suggested,
|
|
1037
|
+
// never called without allow_writes.
|
|
1038
|
+
if (UNLOCK_NAME.test(t.name))
|
|
1039
|
+
return false;
|
|
444
1040
|
if (t.annotations?.destructiveHint === true)
|
|
445
1041
|
return true;
|
|
446
1042
|
const verb = verbOf(t.name);
|
|
447
|
-
return verb !== undefined && WRITE_VERBS.has(verb)
|
|
1043
|
+
return verb !== undefined && WRITE_VERBS.has(verb);
|
|
448
1044
|
}
|
|
449
1045
|
function enumOf(schema) {
|
|
450
1046
|
if (!schema)
|
|
@@ -459,8 +1055,24 @@ export function unlockers(tools) {
|
|
|
459
1055
|
const found = [];
|
|
460
1056
|
for (const tool of tools) {
|
|
461
1057
|
const props = tool.inputSchema?.properties ?? {};
|
|
462
|
-
const
|
|
463
|
-
|
|
1058
|
+
const description = tool.description ?? '';
|
|
1059
|
+
// A lookup names toolsets without changing them (GitHub's get_toolset_tools,
|
|
1060
|
+
// toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
|
|
1061
|
+
const verb = verbOf(tool.name);
|
|
1062
|
+
const lookup = verb !== undefined && LOOKUPS.has(verb);
|
|
1063
|
+
if (lookup && !UNLOCK_SAYS.test(description))
|
|
1064
|
+
continue;
|
|
1065
|
+
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
|
|
1066
|
+
const required = tool.inputSchema?.required ?? [];
|
|
1067
|
+
const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ??
|
|
1068
|
+
(backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined) ??
|
|
1069
|
+
// Named like an unlock, one required parameter: that's what it takes
|
|
1070
|
+
// (toolception's enable_toolset { name }).
|
|
1071
|
+
(UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
|
|
1072
|
+
const claims = claimsUnlock(tool, param);
|
|
1073
|
+
// A lookup that says a *different* parameter unlocks: its enum is a filter.
|
|
1074
|
+
if (lookup && !claims)
|
|
1075
|
+
continue;
|
|
464
1076
|
let score = 0;
|
|
465
1077
|
if (param)
|
|
466
1078
|
score += enumOf(props[param]).length ? 3 : 2;
|
|
@@ -469,25 +1081,109 @@ export function unlockers(tools) {
|
|
|
469
1081
|
if (UNLOCK_DESC.test(tool.description ?? ''))
|
|
470
1082
|
score += 1;
|
|
471
1083
|
if (score >= 2)
|
|
472
|
-
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [] });
|
|
1084
|
+
found.push({ tool, score, param, values: param ? enumOf(props[param]) : [], claims });
|
|
473
1085
|
}
|
|
474
1086
|
return found.sort((a, b) => b.score - a.score);
|
|
475
1087
|
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Unlocks whose values the schema doesn't give (no enum), each with a read-only
|
|
1090
|
+
* lister that should: no required arguments, a lookup verb, and a noun in common
|
|
1091
|
+
* with the unlock (toolception: enable_toolset { name } ← list_toolsets).
|
|
1092
|
+
*/
|
|
1093
|
+
export function unlockListers(tools) {
|
|
1094
|
+
const out = [];
|
|
1095
|
+
const nounSet = (name) => new Set(words(name).map(singular).filter((w) => !VERBS.has(w)));
|
|
1096
|
+
for (const u of unlockers(tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)))) {
|
|
1097
|
+
if (!u.param || u.values.length)
|
|
1098
|
+
continue;
|
|
1099
|
+
const mine = new Set([...nounSet(u.tool.name), ...nounSet(u.param)]);
|
|
1100
|
+
const lister = tools.find((t) => {
|
|
1101
|
+
if (t === u.tool || t.annotations?.readOnlyHint !== true || (t.inputSchema?.required ?? []).length)
|
|
1102
|
+
return false;
|
|
1103
|
+
const verb = verbOf(t.name);
|
|
1104
|
+
return !!verb && LOOKUPS.has(verb) && [...nounSet(t.name)].some((n) => mine.has(n));
|
|
1105
|
+
});
|
|
1106
|
+
if (lister)
|
|
1107
|
+
out.push({ unlock: u.tool.name, lister: lister.name });
|
|
1108
|
+
}
|
|
1109
|
+
return out;
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* The values a listing offers, from its JSON (structuredContent, or text that
|
|
1113
|
+
* starts with JSON): the first list of strings, or of objects with a key, id,
|
|
1114
|
+
* slug or name, each item's own field first (toolception's `key: "quotes"`, not
|
|
1115
|
+
* the display name "Quotes" nested under it).
|
|
1116
|
+
*/
|
|
1117
|
+
export function valuesFromListing(result) {
|
|
1118
|
+
const roots = [];
|
|
1119
|
+
if (result.structuredContent)
|
|
1120
|
+
roots.push(result.structuredContent);
|
|
1121
|
+
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
1122
|
+
const text = part.text;
|
|
1123
|
+
if (typeof text === 'string') {
|
|
1124
|
+
const value = leadingJson(text);
|
|
1125
|
+
if (value !== undefined)
|
|
1126
|
+
roots.push(value);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
const idOf = (item) => {
|
|
1130
|
+
if (typeof item === 'string')
|
|
1131
|
+
return item;
|
|
1132
|
+
if (!item || typeof item !== 'object')
|
|
1133
|
+
return undefined;
|
|
1134
|
+
const o = item;
|
|
1135
|
+
for (const k of ['key', 'id', 'slug', 'name'])
|
|
1136
|
+
if (typeof o[k] === 'string' && o[k])
|
|
1137
|
+
return o[k];
|
|
1138
|
+
return undefined;
|
|
1139
|
+
};
|
|
1140
|
+
const walk = (v, depth) => {
|
|
1141
|
+
if (depth > 5 || !v || typeof v !== 'object')
|
|
1142
|
+
return undefined;
|
|
1143
|
+
if (Array.isArray(v)) {
|
|
1144
|
+
const ids = v.map(idOf).filter((x) => !!x);
|
|
1145
|
+
if (ids.length && ids.length === v.length)
|
|
1146
|
+
return [...new Set(ids)];
|
|
1147
|
+
for (const x of v) {
|
|
1148
|
+
const found = walk(x, depth + 1);
|
|
1149
|
+
if (found)
|
|
1150
|
+
return found;
|
|
1151
|
+
}
|
|
1152
|
+
return undefined;
|
|
1153
|
+
}
|
|
1154
|
+
for (const x of Object.values(v)) {
|
|
1155
|
+
const found = walk(x, depth + 1);
|
|
1156
|
+
if (found)
|
|
1157
|
+
return found;
|
|
1158
|
+
}
|
|
1159
|
+
return undefined;
|
|
1160
|
+
};
|
|
1161
|
+
for (const r of roots) {
|
|
1162
|
+
const found = walk(r, 0);
|
|
1163
|
+
if (found)
|
|
1164
|
+
return found;
|
|
1165
|
+
}
|
|
1166
|
+
return [];
|
|
1167
|
+
}
|
|
476
1168
|
/** A YAML value for an unlock parameter: an array if the schema takes one. */
|
|
477
1169
|
function unlockValue(u, value) {
|
|
478
1170
|
const schema = u.tool.inputSchema?.properties?.[u.param];
|
|
479
1171
|
return schema?.type === 'array' || schema?.items ? `[${JSON.stringify(value)}]` : JSON.stringify(value);
|
|
480
1172
|
}
|
|
481
|
-
|
|
1173
|
+
/** At most this many values of one unlock in a scenario; the rest are named in a comment. */
|
|
1174
|
+
export const MAX_UNLOCKS = 50;
|
|
1175
|
+
export function starterScenario(menu, options = {}) {
|
|
482
1176
|
const readOnly = menu.tools.filter((t) => t.annotations?.readOnlyHint === true);
|
|
483
1177
|
const required = (t) => t.inputSchema?.required ?? [];
|
|
484
|
-
const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)));
|
|
1178
|
+
const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t))).map((u) => u.values.length === 0 && u.param && options.values?.[u.tool.name]?.length ? { ...u, values: options.values[u.tool.name] } : u);
|
|
485
1179
|
const unlocking = new Set(found.filter((u) => u.param).map((u) => u.tool));
|
|
486
1180
|
const ready = readOnly.filter((t) => required(t).length === 0 && !unlocking.has(t)).slice(0, 12);
|
|
487
1181
|
const needsArgs = readOnly.filter((t) => required(t).length > 0);
|
|
488
|
-
// The best candidate with
|
|
1182
|
+
// The best candidate with real values runs for real: every value, so the tools
|
|
1183
|
+
// behind each are seen (a session baseline holds only what was unlocked), then a repeat.
|
|
489
1184
|
const live = found.find((u) => u.values.length > 0 && u.tool.annotations?.readOnlyHint === true && required(u.tool).every((p) => p === u.param));
|
|
490
|
-
const
|
|
1185
|
+
const others = found.filter((u) => u !== live);
|
|
1186
|
+
const changers = others.map((u) => u.tool);
|
|
491
1187
|
const example = (t, p) => {
|
|
492
1188
|
const schema = t.inputSchema?.properties?.[p];
|
|
493
1189
|
const u = found.find((x) => x.tool === t && x.param === p);
|
|
@@ -520,18 +1216,28 @@ export function starterScenario(menu) {
|
|
|
520
1216
|
for (const t of ready)
|
|
521
1217
|
lines.push(` - call: ${t.name}`);
|
|
522
1218
|
if (live) {
|
|
523
|
-
const
|
|
524
|
-
lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock,
|
|
525
|
-
for (const v of
|
|
1219
|
+
const shown = live.values.slice(0, MAX_UNLOCKS);
|
|
1220
|
+
lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock every ${live.param} (${live.values.length}), listing after each, then the first again:`);
|
|
1221
|
+
for (const v of shown)
|
|
526
1222
|
lines.push(` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, v)} }`, ' - list');
|
|
1223
|
+
if (live.values.length > shown.length) {
|
|
1224
|
+
lines.push(` # …and ${live.values.length - shown.length} more: ${live.values.slice(shown.length).map((v) => JSON.stringify(v)).join(', ')}`);
|
|
527
1225
|
}
|
|
528
|
-
lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live,
|
|
1226
|
+
lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, shown[0])} }`, ' - list');
|
|
529
1227
|
}
|
|
530
1228
|
if (changers.length) {
|
|
531
1229
|
lines.push('', ` # ${live ? 'These also' : 'These'} look like they could change the menu. Try them:`);
|
|
532
|
-
for (const
|
|
1230
|
+
for (const u of others.slice(0, 8)) {
|
|
1231
|
+
const t = u.tool;
|
|
533
1232
|
if (t.annotations?.readOnlyHint !== true)
|
|
534
1233
|
lines.push(` # (not marked readOnlyHint: needs allow_writes: true)`);
|
|
1234
|
+
// Every known value, as with a live unlock: each may bring different tools.
|
|
1235
|
+
const values = u.param && required(t).every((p) => p === u.param) ? u.values.slice(0, MAX_UNLOCKS) : [];
|
|
1236
|
+
if (values.length) {
|
|
1237
|
+
for (const v of values)
|
|
1238
|
+
lines.push(` # - call: ${t.name}`, ` # args: { ${u.param}: ${unlockValue(u, v)} }`, ' # - list');
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
535
1241
|
lines.push(` # - call: ${t.name}`);
|
|
536
1242
|
if (argNames(t).length)
|
|
537
1243
|
lines.push(` # args: ${placeholder(t)}`);
|