toolmenu 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -20
- package/dist/auth.js +8 -1
- package/dist/auto.d.ts +1 -1
- package/dist/auto.js +24 -1
- package/dist/breakdown.d.ts +33 -6
- package/dist/breakdown.js +129 -16
- package/dist/catalog.d.ts +20 -0
- package/dist/catalog.js +100 -17
- package/dist/cli.js +39 -11
- package/dist/compare.js +20 -5
- package/dist/config.d.ts +3 -0
- package/dist/diff.d.ts +13 -1
- package/dist/diff.js +671 -72
- package/dist/failures.d.ts +46 -0
- package/dist/failures.js +86 -0
- package/dist/init.d.ts +8 -3
- package/dist/init.js +14 -7
- package/dist/probe.d.ts +15 -3
- package/dist/probe.js +35 -11
- package/dist/rules/description.js +4 -1
- package/dist/rules/determinism.js +2 -1
- package/dist/rules/index.js +3 -1
- package/dist/rules/spec.d.ts +2 -0
- package/dist/rules/spec.js +29 -0
- package/dist/semver.js +1 -1
- package/dist/session.d.ts +42 -3
- package/dist/session.js +354 -40
- package/dist/snapshot.js +8 -5
- package/package.json +1 -1
package/dist/session.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { parse } from 'yaml';
|
|
3
|
-
import { cacheBreak, compareMenus } from './compare.js';
|
|
4
|
-
import {
|
|
3
|
+
import { cacheBreak, canonical, compareMenus } from './compare.js';
|
|
4
|
+
import { listTools } from './connect.js';
|
|
5
5
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
|
-
import { isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
6
|
+
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
-
import { classifyFailure, FAILURE_LABELS, SETUP_FAILURES } from './failures.js';
|
|
8
|
+
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_ADVICE, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, tooMany, waitedFor, whyNot } from './failures.js';
|
|
9
9
|
import { SEVERITY_RANK } from './types.js';
|
|
10
|
-
import {
|
|
10
|
+
import { leadingJson } from './catalog.js';
|
|
11
|
+
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
11
12
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
13
|
+
/** A server saying the session is gone: "Session not found or expired" (toolception). */
|
|
14
|
+
const SESSION_LOST = /session (?:not found|(?:has )?expired|is no longer valid)|(?:unknown|invalid|no valid) session/i;
|
|
12
15
|
/** Read and check a scenario file. Mistakes are reported before anything runs. */
|
|
13
16
|
export async function loadScenario(path) {
|
|
14
17
|
let data;
|
|
@@ -69,7 +72,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
69
72
|
// The main process runs with a pinned hash seed, and so does every scope probe:
|
|
70
73
|
// a probe with a different seed would count ordering variance as a change.
|
|
71
74
|
const mainTarget = seeded(target, MAIN_SEED);
|
|
72
|
-
const
|
|
75
|
+
const waits = options.rateLimitWaitsMs ?? RATE_LIMIT_WAITS_MS;
|
|
76
|
+
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
77
|
+
// out and they're sent again.
|
|
78
|
+
const conn = await connectPatiently(mainTarget, { timeoutMs, waits });
|
|
73
79
|
try {
|
|
74
80
|
const era = conn.era;
|
|
75
81
|
const modern = era === 'modern';
|
|
@@ -77,7 +83,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
77
83
|
let listening = !modern; // 2025-era notifications arrive on the session itself
|
|
78
84
|
if (modern && declared) {
|
|
79
85
|
try {
|
|
80
|
-
await conn.client.listen({ toolsListChanged: true });
|
|
86
|
+
await patiently(() => conn.client.listen({ toolsListChanged: true }), { waits, error: tooMany });
|
|
81
87
|
listening = true;
|
|
82
88
|
}
|
|
83
89
|
catch {
|
|
@@ -85,7 +91,25 @@ export async function session(target, scenario, options = {}) {
|
|
|
85
91
|
}
|
|
86
92
|
}
|
|
87
93
|
const menuOf = async (c) => buildMenu((await listTools(c, { timeoutMs })).tools, serverOf(c));
|
|
88
|
-
|
|
94
|
+
const listPatiently = (waited = { ms: 0 }) => patiently(() => menuOf(conn), { waits, error: tooMany, waited });
|
|
95
|
+
// A refusal that outlasts the waits, or a write's that isn't sent again, ends
|
|
96
|
+
// the run with one finding, not one per step. `tool`: the write not sent again.
|
|
97
|
+
let limitedAt;
|
|
98
|
+
const limited = (index, why, waitedMs, tool) => {
|
|
99
|
+
limitedAt = index;
|
|
100
|
+
const said = serverWords(why);
|
|
101
|
+
const last = scenario.steps.length;
|
|
102
|
+
const unchecked = index < last ? `Steps ${index}–${last} weren't checked.` : `Step ${index} wasn't checked.`;
|
|
103
|
+
raw.push({
|
|
104
|
+
rule: 'session/rate-limited',
|
|
105
|
+
severity: 'error',
|
|
106
|
+
step: index,
|
|
107
|
+
message: tool
|
|
108
|
+
? `Rate-limited at step ${index}: ${tool} said ${quotedSentence(said)} Not called again: it isn't marked readOnlyHint, so it may have done part of the work. ${unchecked} If the limit is the server's, give the run its own server instance or raise the limit; if it's an upstream API's, run later or with a higher quota.`
|
|
109
|
+
: `Rate-limited at step ${index}, and still after waiting ${waitedFor(waitedMs)}: ${quotedSentence(said)} ${unchecked} ${RATE_LIMIT_ADVICE}`,
|
|
110
|
+
});
|
|
111
|
+
};
|
|
112
|
+
let current = await listPatiently();
|
|
89
113
|
// Map keeps first-insertion order; set() on a known name updates it in place.
|
|
90
114
|
const seen = new Map();
|
|
91
115
|
const see = (menu) => menu.tools.forEach((t) => seen.set(t.name, t));
|
|
@@ -93,9 +117,9 @@ export async function session(target, scenario, options = {}) {
|
|
|
93
117
|
const baseline = current;
|
|
94
118
|
// Fresh processes or connections, same credentials, must see the same menu.
|
|
95
119
|
let connectionCheck;
|
|
96
|
-
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs)) {
|
|
120
|
+
for (const probe of await probeVariance(target, current.tools, options.processes ?? 2, timeoutMs, waits)) {
|
|
97
121
|
if (probe.error) {
|
|
98
|
-
raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${probe.error
|
|
122
|
+
raw.push({ rule: target.kind === 'stdio' ? 'menu/process-variance' : 'menu/connection-variance', severity: 'info', step: 0, message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} to compare menus, so this wasn't checked: ${serverWords(probe.error, 200)}` });
|
|
99
123
|
continue;
|
|
100
124
|
}
|
|
101
125
|
const f = varianceFinding(current.tools, probe.tools ?? [], { transport: target.kind, modern, wrapper: isContainerWrapper(target) });
|
|
@@ -106,6 +130,28 @@ export async function session(target, scenario, options = {}) {
|
|
|
106
130
|
}
|
|
107
131
|
}
|
|
108
132
|
const steps = [];
|
|
133
|
+
const scopeUnchecked = [];
|
|
134
|
+
// processes: 1 turns off every second process or connection, the scope probe too.
|
|
135
|
+
const probes = (options.processes ?? 2) > 1;
|
|
136
|
+
// The step after which toolmenu last opened a second process or connection (0:
|
|
137
|
+
// before the first step). A server that keeps one session per client can end
|
|
138
|
+
// this one when that happens.
|
|
139
|
+
let probedAfter = probes ? 0 : undefined;
|
|
140
|
+
const lost = (why, index) => {
|
|
141
|
+
if (probedAfter === undefined || !SESSION_LOST.test(why))
|
|
142
|
+
return false;
|
|
143
|
+
// The server's own words, not the transport's wrapping of its JSON-RPC error.
|
|
144
|
+
const said = serverWords(why);
|
|
145
|
+
raw.push({
|
|
146
|
+
rule: 'session/session-lost',
|
|
147
|
+
severity: 'error',
|
|
148
|
+
step: index,
|
|
149
|
+
message: `The server ended this session (“${said}”) after toolmenu opened a second ${target.kind === 'stdio' ? 'process' : 'connection'} with the same credentials${probedAfter ? ` to check step ${probedAfter}'s scope` : ' to compare menus'}. A server that keeps one session per client does that, so the rest of the scenario didn't run. Run with --processes 1: toolmenu then opens no second one.`,
|
|
150
|
+
});
|
|
151
|
+
return true;
|
|
152
|
+
};
|
|
153
|
+
const callWaited = { ms: 0 };
|
|
154
|
+
const listWaited = { ms: 0 };
|
|
109
155
|
for (const [i, step] of scenario.steps.entries()) {
|
|
110
156
|
const index = i + 1;
|
|
111
157
|
const label = stepLabel(step);
|
|
@@ -133,13 +179,27 @@ export async function session(target, scenario, options = {}) {
|
|
|
133
179
|
}
|
|
134
180
|
if (record.status === 'ok')
|
|
135
181
|
try {
|
|
136
|
-
|
|
182
|
+
// Sent again after a refusal only when that repeats nothing: the transport's
|
|
183
|
+
// 429 means the server never ran it; a tool that says it was rate-limited,
|
|
184
|
+
// thrown or as its result, may have done part of the work first, so only a
|
|
185
|
+
// read-only one is asked again.
|
|
186
|
+
const readOnly = tool?.annotations?.readOnlyHint === true;
|
|
187
|
+
callWaited.ms = 0;
|
|
188
|
+
const result = await patiently(() => conn.client.callTool({ name: step.tool, arguments: step.args }, { timeout: timeoutMs }), {
|
|
189
|
+
waits,
|
|
190
|
+
waited: callWaited,
|
|
191
|
+
error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
|
|
192
|
+
result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
|
|
193
|
+
});
|
|
137
194
|
if (result.isError) {
|
|
138
|
-
const text = (
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
.
|
|
142
|
-
.
|
|
195
|
+
const text = errorText(result);
|
|
196
|
+
if (RATE_LIMITED.test(text)) {
|
|
197
|
+
limited(index, text, callWaited.ms, readOnly ? undefined : step.tool);
|
|
198
|
+
record.status = 'failed';
|
|
199
|
+
record.note = 'rate-limited';
|
|
200
|
+
steps.push(record);
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
143
203
|
const short = clip(text, 120);
|
|
144
204
|
record.note = text ? `the tool returned an error: ${short}` : 'the tool returned an error';
|
|
145
205
|
record.failure = classifyFailure(text, { hadArguments: Object.keys(step.args).length > 0 });
|
|
@@ -156,7 +216,17 @@ export async function session(target, scenario, options = {}) {
|
|
|
156
216
|
}
|
|
157
217
|
catch (error) {
|
|
158
218
|
record.status = 'failed';
|
|
159
|
-
record.note = error
|
|
219
|
+
record.note = serverWords(error, 200);
|
|
220
|
+
if (lost(record.note, index)) {
|
|
221
|
+
steps.push(record);
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
if (tooMany(error)) {
|
|
225
|
+
limited(index, record.note, callWaited.ms, httpStatus(error) === 429 || tool?.annotations?.readOnlyHint === true ? undefined : step.tool);
|
|
226
|
+
record.note = 'rate-limited';
|
|
227
|
+
steps.push(record);
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
160
230
|
const code = error.code;
|
|
161
231
|
record.failure = classifyFailure(record.note, { code: typeof code === 'number' ? code : undefined, hadArguments: Object.keys(step.args).length > 0 });
|
|
162
232
|
if (!SETUP_FAILURES.has(record.failure))
|
|
@@ -174,12 +244,23 @@ export async function session(target, scenario, options = {}) {
|
|
|
174
244
|
// have changed the menu, and the change belongs to this step, not the next.
|
|
175
245
|
let next;
|
|
176
246
|
try {
|
|
177
|
-
|
|
247
|
+
listWaited.ms = 0;
|
|
248
|
+
next = await listPatiently(listWaited);
|
|
178
249
|
}
|
|
179
250
|
catch (error) {
|
|
180
|
-
const why = error
|
|
251
|
+
const why = serverWords(error, 200);
|
|
181
252
|
record.status = 'failed';
|
|
182
253
|
record.note = record.note ? `${record.note}; then listing the menu failed: ${why}` : `listing the menu failed: ${why}`;
|
|
254
|
+
if (lost(why, index)) {
|
|
255
|
+
steps.push(record);
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
258
|
+
if (tooMany(error)) {
|
|
259
|
+
limited(index, why, listWaited.ms);
|
|
260
|
+
record.note = record.note?.startsWith('listing the menu failed') ? 'rate-limited listing the menu' : 'rate-limited listing the menu after the call';
|
|
261
|
+
steps.push(record);
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
183
264
|
raw.push({ rule: 'session/step-failed', severity: 'error', step: index, message: `After step ${index} (${label}), listing the menu failed: ${why}` });
|
|
184
265
|
steps.push(record);
|
|
185
266
|
continue;
|
|
@@ -208,10 +289,21 @@ export async function session(target, scenario, options = {}) {
|
|
|
208
289
|
message: `The menu changed without a notifications/tools/list_changed, although the server declared listChanged. Clients that cache the list won't know to re-fetch it. The spec says servers SHOULD send it.`,
|
|
209
290
|
});
|
|
210
291
|
}
|
|
211
|
-
if (step.kind !== 'list') {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
292
|
+
if (step.kind !== 'list' && probes) {
|
|
293
|
+
// The main process is still running here: a server that holds a file or
|
|
294
|
+
// a port can't start a second copy. That leaves the scope unknown, not
|
|
295
|
+
// the run failed.
|
|
296
|
+
const probeWaited = { ms: 0 };
|
|
297
|
+
try {
|
|
298
|
+
probedAfter = index;
|
|
299
|
+
const probe = await probeMenu(mainTarget, timeoutMs, waits, probeWaited);
|
|
300
|
+
record.scope = scopeOf(changes, next.tools, probe.tools, target.kind, baseline.tools);
|
|
301
|
+
raw.push(...scopeFindings(record.scope, modern, index));
|
|
302
|
+
}
|
|
303
|
+
catch (error) {
|
|
304
|
+
record.scope = 'unclear';
|
|
305
|
+
scopeUnchecked.push({ step: index, why: whyNot(error, probeWaited.ms) });
|
|
306
|
+
}
|
|
215
307
|
}
|
|
216
308
|
current = next;
|
|
217
309
|
see(next);
|
|
@@ -222,6 +314,19 @@ export async function session(target, scenario, options = {}) {
|
|
|
222
314
|
steps.push(record);
|
|
223
315
|
}
|
|
224
316
|
raw.push(...untested(steps, target));
|
|
317
|
+
// --auto means "find what you can": an unlock it skipped is a gap too.
|
|
318
|
+
raw.push(...unlockCoverage(baseline.tools, scenario, steps, options.unionOut === true || options.auto !== undefined, options.auto?.skipped, limitedAt));
|
|
319
|
+
if (options.auto && !scenario.steps.some((s) => s.kind === 'call'))
|
|
320
|
+
raw.push(nothingCalled(options.auto.skipped));
|
|
321
|
+
if (scopeUnchecked.length) {
|
|
322
|
+
const which = scopeUnchecked.map((s) => s.step);
|
|
323
|
+
raw.push({
|
|
324
|
+
rule: 'session/scope-unchecked',
|
|
325
|
+
severity: 'info',
|
|
326
|
+
step: which[0],
|
|
327
|
+
message: `Couldn't ${target.kind === 'stdio' ? 'start a second server process' : 'open a second connection'} while the session ran, so whether the change${which.length === 1 ? ` at step ${which[0]}` : `s at steps ${which.join(', ')}`} reach${which.length === 1 ? 'es' : ''} a fresh ${target.kind === 'stdio' ? 'process' : 'connection'} wasn't checked: ${scopeUnchecked[0].why}`,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
225
330
|
return {
|
|
226
331
|
scenario: options.scenarioName ?? 'scenario',
|
|
227
332
|
server: baseline.server,
|
|
@@ -264,9 +369,97 @@ function untested(steps, target) {
|
|
|
264
369
|
},
|
|
265
370
|
];
|
|
266
371
|
}
|
|
372
|
+
/**
|
|
373
|
+
* Unlocks whose schema lists every value (an enum), and how many of those values
|
|
374
|
+
* the run got through. The session sees, and --union-out keeps, only the tools
|
|
375
|
+
* behind the values it unlocked: GitHub's enable_toolset has 19, the first starter
|
|
376
|
+
* unlocked 2, and 72 of 81 tools never reached the baseline.
|
|
377
|
+
*/
|
|
378
|
+
function unlockCoverage(menu, scenario, steps, baselineWanted, skipped = [], stoppedAt) {
|
|
379
|
+
const out = [];
|
|
380
|
+
// A run a rate limit stopped (session/rate-limited says so) is charged only for
|
|
381
|
+
// values its scenario leaves out, not for ones in steps it never got to.
|
|
382
|
+
const unrun = stoppedAt === undefined ? [] : scenario.steps.map((step, i) => ({ step, index: i + 1 })).filter((s) => s.index >= stoppedAt);
|
|
383
|
+
for (const u of unlockers(menu)) {
|
|
384
|
+
if (!u.param || u.values.length === 0)
|
|
385
|
+
continue;
|
|
386
|
+
const reached = new Set();
|
|
387
|
+
let last;
|
|
388
|
+
const add = (arg) => {
|
|
389
|
+
for (const v of Array.isArray(arg) ? arg : [arg])
|
|
390
|
+
if (v !== undefined)
|
|
391
|
+
reached.add(canonical(v));
|
|
392
|
+
};
|
|
393
|
+
for (const record of steps) {
|
|
394
|
+
const step = scenario.steps[record.index - 1];
|
|
395
|
+
if (step?.kind !== 'call' || step.tool !== u.tool.name || record.status !== 'ok' || record.failure)
|
|
396
|
+
continue;
|
|
397
|
+
last = record.index;
|
|
398
|
+
add(step.args[u.param]);
|
|
399
|
+
}
|
|
400
|
+
for (const { step } of unrun)
|
|
401
|
+
if (step.kind === 'call' && step.tool === u.tool.name)
|
|
402
|
+
add(step.args[u.param]);
|
|
403
|
+
const missing = u.values.filter((v) => !reached.has(canonical(v)));
|
|
404
|
+
// Never called: a scenario about something else, unless it's building the baseline.
|
|
405
|
+
if (missing.length === 0 || (reached.size === 0 && !baselineWanted))
|
|
406
|
+
continue;
|
|
407
|
+
const got = u.values.length - missing.length;
|
|
408
|
+
const skip = skipped.find((s) => s.tool === u.tool.name)?.reason;
|
|
409
|
+
const why = skip === 'open world'
|
|
410
|
+
? ` --auto skipped it: it's marked openWorldHint, so it's called only with --open-world.`
|
|
411
|
+
: skip === 'not read-only'
|
|
412
|
+
? ` --auto skipped it: it isn't marked readOnlyHint, and --auto calls only tools that are.`
|
|
413
|
+
: '';
|
|
414
|
+
out.push({
|
|
415
|
+
rule: 'session/unlock-coverage',
|
|
416
|
+
severity: 'warn',
|
|
417
|
+
tool: u.tool.name,
|
|
418
|
+
...(last !== undefined ? { step: last } : {}),
|
|
419
|
+
message: `${u.tool.name} looks like it unlocks tools, and the run got through ${got} of its ${u.values.length} ${u.param} values. The tools behind the other ${missing.length} were never seen: this session didn't check them, and a baseline from it (--union-out, baseline-from: session) misses them, so diff can't either.${why}${why ? '' : ' Unlock every value (session --init puts every one in the starter).'}`,
|
|
420
|
+
detail: [`not unlocked: ${missing.slice(0, 12).map((v) => JSON.stringify(v)).join(', ')}${missing.length > 12 ? `, and ${missing.length - 12} more` : ''}`],
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
return out;
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* --auto that called nothing: the run only listed the menu, so a clean result
|
|
427
|
+
* says nothing about what the tools do to it. Why each tool was left out.
|
|
428
|
+
*/
|
|
429
|
+
function nothingCalled(skipped) {
|
|
430
|
+
const by = (reason) => skipped.filter((s) => s.reason === reason).map((s) => s.tool);
|
|
431
|
+
const parts = [];
|
|
432
|
+
const openWorld = by('open world');
|
|
433
|
+
const needs = by('needs values');
|
|
434
|
+
const writes = by('not read-only');
|
|
435
|
+
if (openWorld.length)
|
|
436
|
+
parts.push(`${openWorld.length} marked openWorldHint (call them with --open-world: they may cost API credits)`);
|
|
437
|
+
if (needs.length)
|
|
438
|
+
parts.push(`${needs.length} need values the schema doesn't give (--save-scenario puts them in a scenario to fill in)`);
|
|
439
|
+
if (writes.length)
|
|
440
|
+
parts.push(`${writes.length} not marked readOnlyHint`);
|
|
441
|
+
const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
|
|
442
|
+
return {
|
|
443
|
+
rule: 'session/nothing-called',
|
|
444
|
+
severity: 'warn',
|
|
445
|
+
message: `--auto called no tools${parts.length ? `: ${parts.join('; ')}` : ''}. The run only listed the menu, so a clean result says nothing about what calls do to it.`,
|
|
446
|
+
detail: [
|
|
447
|
+
...(openWorld.length ? [`open world: ${list(openWorld)}`] : []),
|
|
448
|
+
...(needs.length ? [`need values: ${list(needs)}`] : []),
|
|
449
|
+
],
|
|
450
|
+
};
|
|
451
|
+
}
|
|
267
452
|
function serverOf(c) {
|
|
268
453
|
return { name: c.server.name, version: c.server.version, protocolVersion: c.protocolVersion, era: c.era };
|
|
269
454
|
}
|
|
455
|
+
/** A tool result's text, whitespace collapsed. */
|
|
456
|
+
function errorText(result) {
|
|
457
|
+
return (Array.isArray(result.content) ? result.content : [])
|
|
458
|
+
.map((c) => (c && typeof c === 'object' && 'text' in c && typeof c.text === 'string' ? c.text : ''))
|
|
459
|
+
.join(' ')
|
|
460
|
+
.replace(/\s+/g, ' ')
|
|
461
|
+
.trim();
|
|
462
|
+
}
|
|
270
463
|
function waitFor(done, ms) {
|
|
271
464
|
return new Promise((resolve) => {
|
|
272
465
|
if (done())
|
|
@@ -345,9 +538,13 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
345
538
|
/**
|
|
346
539
|
* Does a fresh connection see this step's changes? All of them: the server's tool
|
|
347
540
|
* set changed. None: the change belongs to this connection (or, on stdio, this
|
|
348
|
-
* process). Reorders alone can't tell, and a mix is unclear.
|
|
541
|
+
* process). Reorders alone can't tell, and a mix is unclear. So can't a step that
|
|
542
|
+
* takes the menu back to the session's `baseline` while fresh ones still serve it:
|
|
543
|
+
* a fresh process starts there anyway (an unlock undone reads as "seen" by it).
|
|
349
544
|
*/
|
|
350
|
-
export function scopeOf(changes, next, probe, transport) {
|
|
545
|
+
export function scopeOf(changes, next, probe, transport, baseline) {
|
|
546
|
+
if (baseline && compareMenus(baseline, next).length === 0 && compareMenus(baseline, probe).length === 0)
|
|
547
|
+
return 'unclear';
|
|
351
548
|
const probeByName = new Map(probe.map((t) => [t.name, t]));
|
|
352
549
|
const nextByName = new Map(next.map((t) => [t.name, t]));
|
|
353
550
|
const verdicts = new Set();
|
|
@@ -424,10 +621,17 @@ function settle(raw, options) {
|
|
|
424
621
|
// A parameter that names what to unlock. Strong names say so alone; weak ones
|
|
425
622
|
// (`category` is also a search filter: GitHub, Firecrawl) need the tool's name or
|
|
426
623
|
// description to back them up.
|
|
427
|
-
|
|
428
|
-
const
|
|
624
|
+
// `namespace` is weak too: Kubernetes and Pinecone take one on every read.
|
|
625
|
+
const UNLOCK_PARAM = /^(?:domains?|toolsets?|tool_?sets?|capabilit(?:y|ies)|packs?|bundles?)$/i;
|
|
626
|
+
const WEAK_UNLOCK_PARAM = /^(?:categor(?:y|ies)|modules?|features?|groups?|namespaces?)$/i;
|
|
429
627
|
const UNLOCK_NAME = /unlock|enable|activate|capabilit|toolset|load_?tools|expand/i;
|
|
430
|
-
|
|
628
|
+
// A description about loading tools: "Adds the audit tools", "enables more
|
|
629
|
+
// capabilities", "unlocks…". Whole words: "download" isn't "load", and a device's
|
|
630
|
+
// capabilities alone aren't tools.
|
|
631
|
+
const UNLOCK_DESC = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|capabilit(?:y|ies))\b|\bmore tools\b|\btoolsets?\b|\bunlock/i;
|
|
632
|
+
// Said outright: a verb that unlocks, then what it gives. Enough to keep a lookup.
|
|
633
|
+
const UNLOCK_SAYS = /\b(?:unlock|enable|activate|load|expose|add)s?\b[^.]{0,60}\b(?:tools?|toolsets?|capabilit(?:y|ies))\b/i;
|
|
634
|
+
const LOOKUPS = new Set([...LOOKUP_VERBS, 'describe']);
|
|
431
635
|
/**
|
|
432
636
|
* At most `max` characters, cut by code point at a word boundary, never inside a
|
|
433
637
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
@@ -441,10 +645,15 @@ export function clip(text, max) {
|
|
|
441
645
|
return (space > max * 0.6 ? cut.slice(0, space) : cut).replace(/[\s.,;:]+$/, '') + '…';
|
|
442
646
|
}
|
|
443
647
|
function isWriteLike(t) {
|
|
648
|
+
// Named as an unlock: changing the menu is its job, whatever its hints say
|
|
649
|
+
// (toolception marks enable_toolset destructive). It's still only suggested,
|
|
650
|
+
// never called without allow_writes.
|
|
651
|
+
if (UNLOCK_NAME.test(t.name))
|
|
652
|
+
return false;
|
|
444
653
|
if (t.annotations?.destructiveHint === true)
|
|
445
654
|
return true;
|
|
446
655
|
const verb = verbOf(t.name);
|
|
447
|
-
return verb !== undefined && WRITE_VERBS.has(verb)
|
|
656
|
+
return verb !== undefined && WRITE_VERBS.has(verb);
|
|
448
657
|
}
|
|
449
658
|
function enumOf(schema) {
|
|
450
659
|
if (!schema)
|
|
@@ -459,8 +668,19 @@ export function unlockers(tools) {
|
|
|
459
668
|
const found = [];
|
|
460
669
|
for (const tool of tools) {
|
|
461
670
|
const props = tool.inputSchema?.properties ?? {};
|
|
462
|
-
const
|
|
463
|
-
|
|
671
|
+
const description = tool.description ?? '';
|
|
672
|
+
// A lookup names toolsets without changing them (GitHub's get_toolset_tools,
|
|
673
|
+
// toolception's list_toolsets, Firecrawl's find_tools), unless it says it does.
|
|
674
|
+
const verb = verbOf(tool.name);
|
|
675
|
+
if (verb && LOOKUPS.has(verb) && !UNLOCK_SAYS.test(description))
|
|
676
|
+
continue;
|
|
677
|
+
const backed = UNLOCK_NAME.test(tool.name) || UNLOCK_DESC.test(description);
|
|
678
|
+
const required = tool.inputSchema?.required ?? [];
|
|
679
|
+
const param = Object.keys(props).find((p) => UNLOCK_PARAM.test(p)) ??
|
|
680
|
+
(backed ? Object.keys(props).find((p) => WEAK_UNLOCK_PARAM.test(p)) : undefined) ??
|
|
681
|
+
// Named like an unlock, one required parameter: that's what it takes
|
|
682
|
+
// (toolception's enable_toolset { name }).
|
|
683
|
+
(UNLOCK_NAME.test(tool.name) && required.length === 1 ? required[0] : undefined);
|
|
464
684
|
let score = 0;
|
|
465
685
|
if (param)
|
|
466
686
|
score += enumOf(props[param]).length ? 3 : 2;
|
|
@@ -473,21 +693,105 @@ export function unlockers(tools) {
|
|
|
473
693
|
}
|
|
474
694
|
return found.sort((a, b) => b.score - a.score);
|
|
475
695
|
}
|
|
696
|
+
/**
|
|
697
|
+
* Unlocks whose values the schema doesn't give (no enum), each with a read-only
|
|
698
|
+
* lister that should: no required arguments, a lookup verb, and a noun in common
|
|
699
|
+
* with the unlock (toolception: enable_toolset { name } ← list_toolsets).
|
|
700
|
+
*/
|
|
701
|
+
export function unlockListers(tools) {
|
|
702
|
+
const out = [];
|
|
703
|
+
const nounSet = (name) => new Set(words(name).map(singular).filter((w) => !VERBS.has(w)));
|
|
704
|
+
for (const u of unlockers(tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)))) {
|
|
705
|
+
if (!u.param || u.values.length)
|
|
706
|
+
continue;
|
|
707
|
+
const mine = new Set([...nounSet(u.tool.name), ...nounSet(u.param)]);
|
|
708
|
+
const lister = tools.find((t) => {
|
|
709
|
+
if (t === u.tool || t.annotations?.readOnlyHint !== true || (t.inputSchema?.required ?? []).length)
|
|
710
|
+
return false;
|
|
711
|
+
const verb = verbOf(t.name);
|
|
712
|
+
return !!verb && LOOKUPS.has(verb) && [...nounSet(t.name)].some((n) => mine.has(n));
|
|
713
|
+
});
|
|
714
|
+
if (lister)
|
|
715
|
+
out.push({ unlock: u.tool.name, lister: lister.name });
|
|
716
|
+
}
|
|
717
|
+
return out;
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* The values a listing offers, from its JSON (structuredContent, or text that
|
|
721
|
+
* starts with JSON): the first list of strings, or of objects with a key, id,
|
|
722
|
+
* slug or name, each item's own field first (toolception's `key: "quotes"`, not
|
|
723
|
+
* the display name "Quotes" nested under it).
|
|
724
|
+
*/
|
|
725
|
+
export function valuesFromListing(result) {
|
|
726
|
+
const roots = [];
|
|
727
|
+
if (result.structuredContent)
|
|
728
|
+
roots.push(result.structuredContent);
|
|
729
|
+
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
730
|
+
const text = part.text;
|
|
731
|
+
if (typeof text === 'string') {
|
|
732
|
+
const value = leadingJson(text);
|
|
733
|
+
if (value !== undefined)
|
|
734
|
+
roots.push(value);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
const idOf = (item) => {
|
|
738
|
+
if (typeof item === 'string')
|
|
739
|
+
return item;
|
|
740
|
+
if (!item || typeof item !== 'object')
|
|
741
|
+
return undefined;
|
|
742
|
+
const o = item;
|
|
743
|
+
for (const k of ['key', 'id', 'slug', 'name'])
|
|
744
|
+
if (typeof o[k] === 'string' && o[k])
|
|
745
|
+
return o[k];
|
|
746
|
+
return undefined;
|
|
747
|
+
};
|
|
748
|
+
const walk = (v, depth) => {
|
|
749
|
+
if (depth > 5 || !v || typeof v !== 'object')
|
|
750
|
+
return undefined;
|
|
751
|
+
if (Array.isArray(v)) {
|
|
752
|
+
const ids = v.map(idOf).filter((x) => !!x);
|
|
753
|
+
if (ids.length && ids.length === v.length)
|
|
754
|
+
return [...new Set(ids)];
|
|
755
|
+
for (const x of v) {
|
|
756
|
+
const found = walk(x, depth + 1);
|
|
757
|
+
if (found)
|
|
758
|
+
return found;
|
|
759
|
+
}
|
|
760
|
+
return undefined;
|
|
761
|
+
}
|
|
762
|
+
for (const x of Object.values(v)) {
|
|
763
|
+
const found = walk(x, depth + 1);
|
|
764
|
+
if (found)
|
|
765
|
+
return found;
|
|
766
|
+
}
|
|
767
|
+
return undefined;
|
|
768
|
+
};
|
|
769
|
+
for (const r of roots) {
|
|
770
|
+
const found = walk(r, 0);
|
|
771
|
+
if (found)
|
|
772
|
+
return found;
|
|
773
|
+
}
|
|
774
|
+
return [];
|
|
775
|
+
}
|
|
476
776
|
/** A YAML value for an unlock parameter: an array if the schema takes one. */
|
|
477
777
|
function unlockValue(u, value) {
|
|
478
778
|
const schema = u.tool.inputSchema?.properties?.[u.param];
|
|
479
779
|
return schema?.type === 'array' || schema?.items ? `[${JSON.stringify(value)}]` : JSON.stringify(value);
|
|
480
780
|
}
|
|
481
|
-
|
|
781
|
+
/** At most this many values of one unlock in a scenario; the rest are named in a comment. */
|
|
782
|
+
export const MAX_UNLOCKS = 50;
|
|
783
|
+
export function starterScenario(menu, options = {}) {
|
|
482
784
|
const readOnly = menu.tools.filter((t) => t.annotations?.readOnlyHint === true);
|
|
483
785
|
const required = (t) => t.inputSchema?.required ?? [];
|
|
484
|
-
const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t)));
|
|
786
|
+
const found = unlockers(menu.tools.filter((t) => t.annotations?.readOnlyHint === true || !isWriteLike(t))).map((u) => u.values.length === 0 && u.param && options.values?.[u.tool.name]?.length ? { ...u, values: options.values[u.tool.name] } : u);
|
|
485
787
|
const unlocking = new Set(found.filter((u) => u.param).map((u) => u.tool));
|
|
486
788
|
const ready = readOnly.filter((t) => required(t).length === 0 && !unlocking.has(t)).slice(0, 12);
|
|
487
789
|
const needsArgs = readOnly.filter((t) => required(t).length > 0);
|
|
488
|
-
// The best candidate with
|
|
790
|
+
// The best candidate with real values runs for real: every value, so the tools
|
|
791
|
+
// behind each are seen (a session baseline holds only what was unlocked), then a repeat.
|
|
489
792
|
const live = found.find((u) => u.values.length > 0 && u.tool.annotations?.readOnlyHint === true && required(u.tool).every((p) => p === u.param));
|
|
490
|
-
const
|
|
793
|
+
const others = found.filter((u) => u !== live);
|
|
794
|
+
const changers = others.map((u) => u.tool);
|
|
491
795
|
const example = (t, p) => {
|
|
492
796
|
const schema = t.inputSchema?.properties?.[p];
|
|
493
797
|
const u = found.find((x) => x.tool === t && x.param === p);
|
|
@@ -520,18 +824,28 @@ export function starterScenario(menu) {
|
|
|
520
824
|
for (const t of ready)
|
|
521
825
|
lines.push(` - call: ${t.name}`);
|
|
522
826
|
if (live) {
|
|
523
|
-
const
|
|
524
|
-
lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock,
|
|
525
|
-
for (const v of
|
|
827
|
+
const shown = live.values.slice(0, MAX_UNLOCKS);
|
|
828
|
+
lines.push('', ` # ${live.tool.name} looks like it unlocks tools. Unlock every ${live.param} (${live.values.length}), listing after each, then the first again:`);
|
|
829
|
+
for (const v of shown)
|
|
526
830
|
lines.push(` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, v)} }`, ' - list');
|
|
831
|
+
if (live.values.length > shown.length) {
|
|
832
|
+
lines.push(` # …and ${live.values.length - shown.length} more: ${live.values.slice(shown.length).map((v) => JSON.stringify(v)).join(', ')}`);
|
|
527
833
|
}
|
|
528
|
-
lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live,
|
|
834
|
+
lines.push(' # The same unlock twice should change nothing.', ` - call: ${live.tool.name}`, ` args: { ${live.param}: ${unlockValue(live, shown[0])} }`, ' - list');
|
|
529
835
|
}
|
|
530
836
|
if (changers.length) {
|
|
531
837
|
lines.push('', ` # ${live ? 'These also' : 'These'} look like they could change the menu. Try them:`);
|
|
532
|
-
for (const
|
|
838
|
+
for (const u of others.slice(0, 8)) {
|
|
839
|
+
const t = u.tool;
|
|
533
840
|
if (t.annotations?.readOnlyHint !== true)
|
|
534
841
|
lines.push(` # (not marked readOnlyHint: needs allow_writes: true)`);
|
|
842
|
+
// Every known value, as with a live unlock: each may bring different tools.
|
|
843
|
+
const values = u.param && required(t).every((p) => p === u.param) ? u.values.slice(0, MAX_UNLOCKS) : [];
|
|
844
|
+
if (values.length) {
|
|
845
|
+
for (const v of values)
|
|
846
|
+
lines.push(` # - call: ${t.name}`, ` # args: { ${u.param}: ${unlockValue(u, v)} }`, ' # - list');
|
|
847
|
+
continue;
|
|
848
|
+
}
|
|
535
849
|
lines.push(` # - call: ${t.name}`);
|
|
536
850
|
if (argNames(t).length)
|
|
537
851
|
lines.push(` # args: ${placeholder(t)}`);
|
package/dist/snapshot.js
CHANGED
|
@@ -1,17 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { listTools } from './connect.js';
|
|
2
|
+
import { patiently, tooMany } from './failures.js';
|
|
2
3
|
import { buildMenu } from './menu.js';
|
|
3
|
-
import { isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
4
|
+
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
4
5
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
5
6
|
import { readCatalog } from './catalog.js';
|
|
6
7
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
7
8
|
export async function snapshot(target, options = {}) {
|
|
8
|
-
|
|
9
|
+
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
10
|
+
// out and they're sent again (CI behind a shared rate limit).
|
|
11
|
+
const connection = await connectPatiently(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
9
12
|
let menu;
|
|
10
13
|
let secondList;
|
|
11
14
|
let first;
|
|
12
15
|
try {
|
|
13
|
-
first = await listTools(connection, options);
|
|
14
|
-
const second = await listTools(connection, options);
|
|
16
|
+
first = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
17
|
+
const second = await patiently(() => listTools(connection, options), { error: tooMany });
|
|
15
18
|
const server = {
|
|
16
19
|
name: connection.server.name,
|
|
17
20
|
version: connection.server.version,
|