toolmenu 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +150 -30
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +61 -18
- package/dist/auto.d.ts +71 -7
- package/dist/auto.js +267 -13
- package/dist/breakdown.d.ts +35 -6
- package/dist/breakdown.js +179 -23
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +116 -178
- package/dist/compare.js +20 -5
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +35 -1
- package/dist/diff.js +1132 -121
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +55 -0
- package/dist/failures.js +103 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +100 -5
- package/dist/init.js +520 -26
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/probe.d.ts +15 -3
- package/dist/probe.js +35 -11
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +89 -43
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +146 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +15 -5
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.d.ts +2 -0
- package/dist/rules/spec.js +61 -8
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/semver.js +1 -1
- package/dist/session.d.ts +56 -7
- package/dist/session.js +800 -94
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +25 -7
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/auth.js
CHANGED
|
@@ -37,7 +37,7 @@ async function save(data) {
|
|
|
37
37
|
await writeFile(path, JSON.stringify({ ...data, savedAt: new Date().toISOString() }, null, 2) + '\n', { mode: 0o600 });
|
|
38
38
|
await chmod(path, 0o600);
|
|
39
39
|
}
|
|
40
|
-
/** Is there a stored
|
|
40
|
+
/** Is there a stored file for this server (a login, or what's left of one)? Why a login is needed reads its contents. */
|
|
41
41
|
export function hasLogin(serverUrl) {
|
|
42
42
|
return existsSync(storePath(serverUrl));
|
|
43
43
|
}
|
|
@@ -51,6 +51,8 @@ export class StoredOAuthProvider {
|
|
|
51
51
|
options;
|
|
52
52
|
data;
|
|
53
53
|
expectedState;
|
|
54
|
+
/** invalidateCredentials('all') deleted a file that held tokens, in this run. */
|
|
55
|
+
removedLogin = false;
|
|
54
56
|
constructor(serverUrl, options = {}) {
|
|
55
57
|
this.serverUrl = serverUrl;
|
|
56
58
|
this.options = options;
|
|
@@ -124,11 +126,11 @@ export class StoredOAuthProvider {
|
|
|
124
126
|
return (await this.stored()).tokens;
|
|
125
127
|
}
|
|
126
128
|
async saveTokens(tokens, ctx) {
|
|
127
|
-
await this.update({ tokens, ...(ctx?.issuer ? { issuer: ctx.issuer } : {}) });
|
|
129
|
+
await this.update({ tokens, rejected: undefined, ...(ctx?.issuer ? { issuer: ctx.issuer } : {}) });
|
|
128
130
|
}
|
|
129
131
|
async redirectToAuthorization(authorizationUrl) {
|
|
130
132
|
if (!this.options.interactive) {
|
|
131
|
-
throw new LoginNeededError(this.serverUrl);
|
|
133
|
+
throw new LoginNeededError(this.serverUrl, whyLoginNeeded(await this.stored(), this.removedLogin));
|
|
132
134
|
}
|
|
133
135
|
await this.options.onRedirect?.(authorizationUrl);
|
|
134
136
|
}
|
|
@@ -138,7 +140,7 @@ export class StoredOAuthProvider {
|
|
|
138
140
|
async codeVerifier() {
|
|
139
141
|
const v = (await this.stored()).codeVerifier;
|
|
140
142
|
if (!v)
|
|
141
|
-
throw new Error(
|
|
143
|
+
throw new Error(`No PKCE code verifier is saved for ${this.serverUrl}: the stored login changed while this one ran (another login at the same time?).\n→ Next: Run toolmenu auth login ${this.serverUrl} again.`);
|
|
142
144
|
return v;
|
|
143
145
|
}
|
|
144
146
|
async saveDiscoveryState(discoveryState) {
|
|
@@ -150,15 +152,22 @@ export class StoredOAuthProvider {
|
|
|
150
152
|
async invalidateCredentials(scope) {
|
|
151
153
|
const s = await this.stored();
|
|
152
154
|
if (scope === 'all') {
|
|
155
|
+
this.removedLogin ||= Boolean(s.tokens) || Boolean(s.rejected);
|
|
153
156
|
this.data = { serverUrl: s.serverUrl };
|
|
154
157
|
await rm(storePath(this.serverUrl), { force: true });
|
|
155
158
|
return;
|
|
156
159
|
}
|
|
157
160
|
const patch = {};
|
|
158
|
-
if (scope === 'client')
|
|
161
|
+
if (scope === 'client') {
|
|
159
162
|
patch.clientInformation = undefined;
|
|
160
|
-
|
|
163
|
+
if (s.tokens)
|
|
164
|
+
patch.rejected = 'client';
|
|
165
|
+
}
|
|
166
|
+
if (scope === 'tokens') {
|
|
161
167
|
patch.tokens = undefined;
|
|
168
|
+
if (s.tokens)
|
|
169
|
+
patch.rejected ??= s.rejected ?? 'refresh';
|
|
170
|
+
}
|
|
162
171
|
if (scope === 'verifier')
|
|
163
172
|
patch.codeVerifier = undefined;
|
|
164
173
|
if (scope === 'discovery')
|
|
@@ -166,9 +175,28 @@ export class StoredOAuthProvider {
|
|
|
166
175
|
await this.update(patch);
|
|
167
176
|
}
|
|
168
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Why snapshot or session needs a login, from what's stored: nothing, a registration
|
|
180
|
+
* from a login that never finished, tokens the server refused, or a login the server
|
|
181
|
+
* rejected and toolmenu removed (removedLogin: in this run).
|
|
182
|
+
*/
|
|
183
|
+
function whyLoginNeeded(s, removedLogin) {
|
|
184
|
+
if (removedLogin)
|
|
185
|
+
return 'the server rejected the stored login for it, so toolmenu removed it';
|
|
186
|
+
if (s.rejected === 'client')
|
|
187
|
+
return "the stored login no longer works: the server doesn't accept the client toolmenu registered (it may have been deleted), so its tokens were dropped";
|
|
188
|
+
if (s.rejected === 'refresh')
|
|
189
|
+
return 'the stored login no longer works: its token expired or was revoked and the server refused to refresh it';
|
|
190
|
+
if (s.tokens)
|
|
191
|
+
return 'the stored login no longer works: the server refused its token';
|
|
192
|
+
if (s.clientInformation)
|
|
193
|
+
return "there's no stored login for it: an earlier toolmenu auth login registered a client but didn't finish";
|
|
194
|
+
return `there's no stored login for it (stored in ${authDir()})`;
|
|
195
|
+
}
|
|
169
196
|
export class LoginNeededError extends Error {
|
|
170
|
-
constructor(serverUrl) {
|
|
171
|
-
super(`${serverUrl}
|
|
197
|
+
constructor(serverUrl, why = `there's no stored login for it (stored in ${authDir()})`) {
|
|
198
|
+
super(`${serverUrl} asks for an OAuth login, and ${why}. snapshot and session don't open a browser.\n` +
|
|
199
|
+
`→ Next: Log in once in a terminal with a browser: toolmenu auth login ${serverUrl}. Or pass a token: --header "Authorization: Bearer <token>".`);
|
|
172
200
|
}
|
|
173
201
|
}
|
|
174
202
|
/**
|
|
@@ -196,10 +224,14 @@ export async function login(serverUrl, options = {}) {
|
|
|
196
224
|
if (!(error instanceof UnauthorizedError))
|
|
197
225
|
throw error;
|
|
198
226
|
const params = await callback.params;
|
|
199
|
-
if (params.get('error'))
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
227
|
+
if (params.get('error')) {
|
|
228
|
+
const described = params.get('error_description');
|
|
229
|
+
throw new Error(`The authorization server refused the login for ${serverUrl}: ${params.get('error')}${described ? ` (“${described}”)` : ''}.\n` +
|
|
230
|
+
`→ Next: Try again. If it repeats, ask for fewer scopes with --scope, or check the account may use this server.`);
|
|
231
|
+
}
|
|
232
|
+
if (!provider.lastState || params.get('state') !== provider.lastState) {
|
|
233
|
+
throw new Error(`The login callback for ${serverUrl} carried the wrong state (not the one this login sent), so it was ignored (an old browser tab, or a second login at once).\n→ Next: Run toolmenu auth login ${serverUrl} again and use the tab it opens.`);
|
|
234
|
+
}
|
|
203
235
|
await transport.finishAuth(params);
|
|
204
236
|
await client.close().catch(() => { });
|
|
205
237
|
}
|
|
@@ -210,10 +242,10 @@ export async function login(serverUrl, options = {}) {
|
|
|
210
242
|
// register itself. Say what works instead.
|
|
211
243
|
if (error instanceof Error && /does not support dynamic client registration/i.test(error.message) && !options.clientId) {
|
|
212
244
|
error.message =
|
|
213
|
-
`${serverUrl} doesn't let clients register themselves (no dynamic client registration)
|
|
214
|
-
|
|
245
|
+
`${serverUrl} doesn't let clients register themselves (its authorization server has no dynamic client registration), so toolmenu can't log in without a client of its own.\n` +
|
|
246
|
+
`→ Next: Register an OAuth app with the provider, with the callback URL http://127.0.0.1:${port}/callback, then:\n` +
|
|
215
247
|
` toolmenu auth login ${serverUrl} --client-id <id> --client-secret <secret>\n` +
|
|
216
|
-
`
|
|
248
|
+
` Or skip OAuth and pass a token the server accepts: toolmenu snapshot ${serverUrl} --header "Authorization: Bearer <token>"`;
|
|
217
249
|
}
|
|
218
250
|
throw error;
|
|
219
251
|
}
|
|
@@ -274,8 +306,12 @@ function listenForCallback(port, timeoutMs) {
|
|
|
274
306
|
res.end('<!doctype html><title>toolmenu</title><p>Logged in. You can close this tab and go back to the terminal.</p>');
|
|
275
307
|
resolveParams(url.searchParams);
|
|
276
308
|
});
|
|
277
|
-
const timer = setTimeout(() => rejectParams(new Error(`
|
|
278
|
-
|
|
309
|
+
const timer = setTimeout(() => rejectParams(new Error(`The browser login didn't come back within ${Math.round(timeoutMs / 1000)} s: nothing reached http://127.0.0.1:${port}/callback.\n` +
|
|
310
|
+
`→ Next: Run toolmenu auth login again and finish it in the browser. The browser has to run on this machine (127.0.0.1); over SSH, log in on your own machine and copy ${authDir()}, or pass a token with --header.`)), timeoutMs);
|
|
311
|
+
server.on('error', (error) => fail(error.code === 'EADDRINUSE'
|
|
312
|
+
? new Error(`Couldn't wait for the login's redirect: port ${port} on 127.0.0.1 is in use (another toolmenu login, or another program).\n` +
|
|
313
|
+
`→ Next: Close it, or pick another port with --port <n>. The login registers http://127.0.0.1:<n>/callback, so use the same port the next time too.`)
|
|
314
|
+
: error));
|
|
279
315
|
server.listen(port, '127.0.0.1', () => ready({
|
|
280
316
|
params,
|
|
281
317
|
close: () => {
|
|
@@ -294,7 +330,14 @@ function openInBrowser(url) {
|
|
|
294
330
|
if (scope)
|
|
295
331
|
process.stderr.write(`Asking for the server's default scopes: ${scope} (narrower: --scope; toolmenu never calls a write tool on its own)\n`);
|
|
296
332
|
process.stderr.write(`Opening your browser to log in. If it doesn't open, visit:\n ${url.href}\n`);
|
|
297
|
-
|
|
333
|
+
// Not `cmd /c start` on Windows: cmd reads the URL's `&` as a command separator,
|
|
334
|
+
// cutting the URL, and the URL comes from the server's metadata. rundll32 opens
|
|
335
|
+
// it in the default browser without a shell.
|
|
336
|
+
const [cmd, args] = process.platform === 'darwin'
|
|
337
|
+
? ['open', [url.href]]
|
|
338
|
+
: process.platform === 'win32'
|
|
339
|
+
? ['rundll32', ['url.dll,FileProtocolHandler', url.href]]
|
|
340
|
+
: ['xdg-open', [url.href]];
|
|
298
341
|
try {
|
|
299
342
|
spawn(cmd, args, { detached: true, stdio: 'ignore' }).on('error', () => { }).unref();
|
|
300
343
|
}
|
package/dist/auto.d.ts
CHANGED
|
@@ -1,28 +1,92 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import type { Menu } from './types.js';
|
|
1
|
+
import { type Scenario } from './session.js';
|
|
2
|
+
import type { Menu, MenuTool } from './types.js';
|
|
3
|
+
/** A value the user gave: `raw` is the text as typed on the command line, when it was typed. */
|
|
4
|
+
export interface GivenValue {
|
|
5
|
+
/** The text as typed, for fitValue: `42` for a string parameter is "42". */
|
|
6
|
+
raw?: string;
|
|
7
|
+
value: unknown;
|
|
8
|
+
/** A values file's map grouped by tool: each field with its own text. */
|
|
9
|
+
fields?: Record<string, GivenValue>;
|
|
10
|
+
}
|
|
3
11
|
export interface AutoOptions {
|
|
4
12
|
/** Also call read-only tools marked openWorldHint: true (web search, fetch, scraping). */
|
|
5
13
|
openWorld?: boolean;
|
|
6
14
|
/** At most this many calls (default 20). */
|
|
7
15
|
maxCalls?: number;
|
|
16
|
+
/**
|
|
17
|
+
* Values for required parameters, by name (`repo_path`) or for one tool
|
|
18
|
+
* (`get_issue.issue_key`). Used before anything is synthesized.
|
|
19
|
+
*/
|
|
20
|
+
values?: Record<string, GivenValue>;
|
|
21
|
+
/**
|
|
22
|
+
* Tools the user says only read, although the server doesn't mark them
|
|
23
|
+
* readOnlyHint. Exact names: a pattern would also cover tools a later release
|
|
24
|
+
* adds, which nobody has looked at. A tool marked or named as a write is still
|
|
25
|
+
* never called.
|
|
26
|
+
*/
|
|
27
|
+
assumeReadOnly?: string[];
|
|
8
28
|
}
|
|
9
|
-
export type SkipReason = 'not read-only' | 'open world' | 'needs values' | 'over the call budget';
|
|
29
|
+
export type SkipReason = 'not read-only' | 'unmarked' | 'open world' | 'needs values' | 'over the call budget';
|
|
10
30
|
export interface AutoPlan {
|
|
11
31
|
scenario: Scenario;
|
|
12
32
|
called: string[];
|
|
33
|
+
/** Called on the user's word (--assume-read-only): the server doesn't mark them readOnlyHint. */
|
|
34
|
+
assumed: string[];
|
|
35
|
+
/** Called with at least one value the user gave. */
|
|
36
|
+
withValues: string[];
|
|
13
37
|
skipped: {
|
|
14
38
|
tool: string;
|
|
15
39
|
reason: SkipReason;
|
|
16
40
|
missing?: string[];
|
|
41
|
+
why?: string;
|
|
42
|
+
}[];
|
|
43
|
+
/** --value and --assume-read-only entries that did nothing, and why. */
|
|
44
|
+
ignored: {
|
|
45
|
+
input: string;
|
|
46
|
+
why: string;
|
|
17
47
|
}[];
|
|
18
48
|
}
|
|
49
|
+
/** What a session report shows about an --auto plan. */
|
|
50
|
+
export type AutoSummary = Pick<AutoPlan, 'called' | 'skipped'> & Partial<Pick<AutoPlan, 'assumed' | 'withValues' | 'ignored'>> & {
|
|
51
|
+
maxCalls?: number;
|
|
52
|
+
/** Guessed unlocks whose first value changed nothing, and how many calls to them were left out. */
|
|
53
|
+
stopped?: {
|
|
54
|
+
tool: string;
|
|
55
|
+
calls: number;
|
|
56
|
+
}[];
|
|
57
|
+
};
|
|
58
|
+
/** Marked or named as a write: never called on the user's word. Says why, or undefined. */
|
|
59
|
+
export declare function writeSign(tool: MenuTool): string | undefined;
|
|
19
60
|
/**
|
|
20
61
|
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
21
|
-
* whose required arguments the schema can fill, cheapest
|
|
22
|
-
* then lists and searches, then single lookups, then the
|
|
23
|
-
* call again, which should change nothing. Never a tool that
|
|
24
|
-
* readOnlyHint, whatever allow_writes says
|
|
62
|
+
* whose required arguments the user's values and the schema can fill, cheapest
|
|
63
|
+
* first (no arguments, then lists and searches, then single lookups, then the
|
|
64
|
+
* rest), then the first call again, which should change nothing. Never a tool that
|
|
65
|
+
* isn't marked readOnlyHint, whatever allow_writes says, unless the user named it
|
|
66
|
+
* in assumeReadOnly and nothing marks or names it as a write.
|
|
25
67
|
*/
|
|
26
68
|
export declare function autoScenario(menu: Menu, options?: AutoOptions): AutoPlan;
|
|
69
|
+
/** `name=value` from --value: JSON when it parses (numbers, true, lists), the text otherwise. */
|
|
70
|
+
export declare function parseValueFlag(text: string): [string, GivenValue];
|
|
71
|
+
/** A values file: a YAML or JSON map, keys as in --value (`repo_path`, `get_issue.issue_key`). */
|
|
72
|
+
export declare function loadValuesFile(path: string): Promise<Record<string, GivenValue>>;
|
|
73
|
+
/** --assume-read-only a,b --assume-read-only c: exact names, no patterns. */
|
|
74
|
+
export declare function parseAssumeReadOnly(flags: string[]): string[];
|
|
75
|
+
/** Parameters the skipped tools need, most needed first, with how many tools each fills. */
|
|
76
|
+
export declare function neededValues(skipped: AutoPlan['skipped']): {
|
|
77
|
+
param: string;
|
|
78
|
+
tools: string[];
|
|
79
|
+
}[];
|
|
80
|
+
/**
|
|
81
|
+
* The one flag or two that would let --auto call more, as a next step: values for
|
|
82
|
+
* the parameters most tools need, the unmarked tools to vouch for, --open-world.
|
|
83
|
+
* Undefined when nothing was left out that a flag can reach.
|
|
84
|
+
*/
|
|
85
|
+
export declare function autoNextStep(auto: AutoSummary): string | undefined;
|
|
86
|
+
/**
|
|
87
|
+
* What --auto called and what it left out, each with the flag that reaches it: the
|
|
88
|
+
* header line of a session report, then one line per reason.
|
|
89
|
+
*/
|
|
90
|
+
export declare function autoSummary(auto: AutoSummary): string[];
|
|
27
91
|
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
28
92
|
export declare function scenarioYaml(plan: AutoPlan, serverName?: string): string;
|
package/dist/auto.js
CHANGED
|
@@ -1,20 +1,68 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { isMap, isNode, isScalar, parseDocument } from 'yaml';
|
|
1
3
|
import { synthesizeArgs } from './args.js';
|
|
2
|
-
import {
|
|
4
|
+
import { MAX_UNLOCKS, unlockers } from './session.js';
|
|
5
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf, WRITE_VERBS } from './words.js';
|
|
6
|
+
/** Marked or named as a write: never called on the user's word. Says why, or undefined. */
|
|
7
|
+
export function writeSign(tool) {
|
|
8
|
+
const a = tool.annotations ?? {};
|
|
9
|
+
if (a.readOnlyHint === false)
|
|
10
|
+
return 'the server marks it readOnlyHint: false';
|
|
11
|
+
if (a.destructiveHint === true)
|
|
12
|
+
return 'the server marks it destructiveHint: true';
|
|
13
|
+
const verb = verbOf(tool.name);
|
|
14
|
+
if (verb && WRITE_VERBS.has(verb))
|
|
15
|
+
return `its name starts with "${verb}", a write`;
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
3
18
|
/**
|
|
4
19
|
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
5
|
-
* whose required arguments the schema can fill, cheapest
|
|
6
|
-
* then lists and searches, then single lookups, then the
|
|
7
|
-
* call again, which should change nothing. Never a tool that
|
|
8
|
-
* readOnlyHint, whatever allow_writes says
|
|
20
|
+
* whose required arguments the user's values and the schema can fill, cheapest
|
|
21
|
+
* first (no arguments, then lists and searches, then single lookups, then the
|
|
22
|
+
* rest), then the first call again, which should change nothing. Never a tool that
|
|
23
|
+
* isn't marked readOnlyHint, whatever allow_writes says, unless the user named it
|
|
24
|
+
* in assumeReadOnly and nothing marks or names it as a write.
|
|
9
25
|
*/
|
|
10
26
|
export function autoScenario(menu, options = {}) {
|
|
11
27
|
const maxCalls = options.maxCalls ?? 20;
|
|
12
28
|
const skipped = [];
|
|
29
|
+
const ignored = [];
|
|
30
|
+
const byName = new Map(menu.tools.map((t) => [t.name, t]));
|
|
31
|
+
// --assume-read-only: exact names, and never a write.
|
|
32
|
+
const assumed = new Set();
|
|
33
|
+
for (const name of options.assumeReadOnly ?? []) {
|
|
34
|
+
const tool = byName.get(name);
|
|
35
|
+
if (!tool)
|
|
36
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: 'no tool by that name in the menu' });
|
|
37
|
+
else if (tool.annotations?.readOnlyHint === true)
|
|
38
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: 'the server already marks it readOnlyHint: true' });
|
|
39
|
+
else if (writeSign(tool))
|
|
40
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: `not called: ${writeSign(tool)}` });
|
|
41
|
+
else
|
|
42
|
+
assumed.add(name);
|
|
43
|
+
}
|
|
44
|
+
const readOnly = (t) => t.annotations?.readOnlyHint === true || assumed.has(t.name);
|
|
45
|
+
const { perTool, ignored: unusedValues } = resolveValues(menu, options.values ?? {});
|
|
46
|
+
ignored.push(...unusedValues);
|
|
13
47
|
const candidates = [];
|
|
48
|
+
// Read-only unlocks with every value in an enum: each value, outside the call
|
|
49
|
+
// budget, so the tools behind every toolset are seen (GitHub: 19 toolsets).
|
|
50
|
+
// Called even when marked openWorldHint, without --open-world: an unlock changes
|
|
51
|
+
// the menu, which is what a session watches, and spends no search credits. A
|
|
52
|
+
// server whose every tool calls an outside service marks its unlock that way too,
|
|
53
|
+
// and --auto then tested nothing.
|
|
54
|
+
const unlocks = unlockers(menu.tools).filter((u) => {
|
|
55
|
+
const required = u.tool.inputSchema?.required ?? [];
|
|
56
|
+
return u.param && u.values.length && readOnly(u.tool) && required.every((p) => p === u.param);
|
|
57
|
+
});
|
|
58
|
+
const unlocking = new Set(unlocks.map((u) => u.tool.name));
|
|
14
59
|
for (const tool of menu.tools) {
|
|
60
|
+
if (unlocking.has(tool.name))
|
|
61
|
+
continue;
|
|
15
62
|
const a = tool.annotations ?? {};
|
|
16
|
-
if (
|
|
17
|
-
|
|
63
|
+
if (!readOnly(tool)) {
|
|
64
|
+
const write = writeSign(tool);
|
|
65
|
+
skipped.push(write || a.readOnlyHint !== undefined ? { tool: tool.name, reason: 'not read-only', ...(write ? { why: write } : {}) } : { tool: tool.name, reason: 'unmarked' });
|
|
18
66
|
continue;
|
|
19
67
|
}
|
|
20
68
|
// readOnlyHint is the server's own claim, and a read-only search still spends
|
|
@@ -26,12 +74,12 @@ export function autoScenario(menu, options = {}) {
|
|
|
26
74
|
skipped.push({ tool: tool.name, reason: 'open world' });
|
|
27
75
|
continue;
|
|
28
76
|
}
|
|
29
|
-
const filled = synthesizeArgs(tool.inputSchema);
|
|
77
|
+
const filled = synthesizeArgs(tool.inputSchema, perTool.get(tool.name));
|
|
30
78
|
if (!filled.ok) {
|
|
31
79
|
skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
|
|
32
80
|
continue;
|
|
33
81
|
}
|
|
34
|
-
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args) });
|
|
82
|
+
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args), valued: Object.values(filled.sources).includes('value') });
|
|
35
83
|
}
|
|
36
84
|
candidates.sort((x, y) => x.rank - y.rank);
|
|
37
85
|
const chosen = candidates.slice(0, maxCalls);
|
|
@@ -40,9 +88,139 @@ export function autoScenario(menu, options = {}) {
|
|
|
40
88
|
const steps = [{ kind: 'list' }];
|
|
41
89
|
for (const c of chosen)
|
|
42
90
|
steps.push({ kind: 'call', tool: c.tool.name, args: c.args });
|
|
91
|
+
for (const u of unlocks) {
|
|
92
|
+
const array = u.tool.inputSchema?.properties?.[u.param]?.type === 'array';
|
|
93
|
+
const values = u.values.slice(0, MAX_UNLOCKS);
|
|
94
|
+
// Guessed from its schema alone (nothing says it unlocks tools): if the first
|
|
95
|
+
// value changes nothing, the session doesn't spend a call on every other one.
|
|
96
|
+
const tentative = u.claims ? {} : { tentative: true };
|
|
97
|
+
for (const v of values)
|
|
98
|
+
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [v] : v }, ...tentative });
|
|
99
|
+
// The same unlock again should change nothing, as in the --init starter.
|
|
100
|
+
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [values[0]] : values[0] }, ...tentative });
|
|
101
|
+
}
|
|
43
102
|
if (chosen.length)
|
|
44
103
|
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
45
|
-
|
|
104
|
+
const called = [...chosen.map((c) => c.tool.name), ...unlocks.map((u) => u.tool.name)];
|
|
105
|
+
return {
|
|
106
|
+
scenario: { allowWrites: false, steps, ...(assumed.size ? { assumeReadOnly: [...assumed] } : {}) },
|
|
107
|
+
called,
|
|
108
|
+
assumed: called.filter((n) => assumed.has(n)),
|
|
109
|
+
withValues: chosen.filter((c) => c.valued).map((c) => c.tool.name),
|
|
110
|
+
skipped,
|
|
111
|
+
ignored,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The user's values, per tool. `param` fills that required parameter wherever a
|
|
116
|
+
* tool has it; `tool.param` fills it, required or not, on that tool only (and
|
|
117
|
+
* wins over a by-name value). A value no tool takes is reported, not dropped.
|
|
118
|
+
*/
|
|
119
|
+
function resolveValues(menu, values) {
|
|
120
|
+
const perTool = new Map();
|
|
121
|
+
const ignored = [];
|
|
122
|
+
const set = (tool, param, v) => perTool.set(tool, { ...perTool.get(tool), [param]: v });
|
|
123
|
+
const byName = new Map(menu.tools.map((t) => [t.name, t]));
|
|
124
|
+
const targeted = [];
|
|
125
|
+
for (const [key, v] of Object.entries(values)) {
|
|
126
|
+
// A values file can group by tool: `get_issue: { issue_key: ABC-1 }`.
|
|
127
|
+
if (byName.has(key) && v.raw === undefined && v.value && typeof v.value === 'object' && !Array.isArray(v.value)) {
|
|
128
|
+
for (const [param, pv] of Object.entries(v.value)) {
|
|
129
|
+
if (!byName.get(key).inputSchema?.properties?.[param])
|
|
130
|
+
ignored.push({ input: `${key}.${param}`, why: `${key} has no parameter ${param}` });
|
|
131
|
+
else
|
|
132
|
+
targeted.push([key, param, v.fields?.[param] ?? { value: pv, ...(typeof pv === 'string' ? { raw: pv } : {}) }]);
|
|
133
|
+
}
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const dot = key.lastIndexOf('.');
|
|
137
|
+
const toolName = dot > 0 ? key.slice(0, dot) : '';
|
|
138
|
+
if (toolName && byName.has(toolName)) {
|
|
139
|
+
const param = key.slice(dot + 1);
|
|
140
|
+
if (!byName.get(toolName).inputSchema?.properties?.[param])
|
|
141
|
+
ignored.push({ input: `--value ${key}`, why: `${toolName} has no parameter ${param}` });
|
|
142
|
+
else
|
|
143
|
+
targeted.push([toolName, param, v]);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const takers = menu.tools.filter((t) => (t.inputSchema?.required ?? []).includes(key));
|
|
147
|
+
if (takers.length) {
|
|
148
|
+
for (const t of takers)
|
|
149
|
+
set(t.name, key, v);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const optional = menu.tools.filter((t) => t.inputSchema?.properties?.[key]).map((t) => t.name);
|
|
153
|
+
ignored.push({
|
|
154
|
+
input: `--value ${key}`,
|
|
155
|
+
why: optional.length
|
|
156
|
+
? `no tool requires ${key}; to pass it where it's optional, name the tool (--value ${optional[0]}.${key}=…)`
|
|
157
|
+
: toolName
|
|
158
|
+
? `no tool named ${toolName}, and no parameter named ${key}`
|
|
159
|
+
: `no tool takes a parameter named ${key}`,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
for (const [tool, param, v] of targeted)
|
|
163
|
+
set(tool, param, v);
|
|
164
|
+
return { perTool, ignored };
|
|
165
|
+
}
|
|
166
|
+
/** `name=value` from --value: JSON when it parses (numbers, true, lists), the text otherwise. */
|
|
167
|
+
export function parseValueFlag(text) {
|
|
168
|
+
const eq = text.indexOf('=');
|
|
169
|
+
if (eq <= 0)
|
|
170
|
+
throw new Error(`--value ${text}: expected name=value or tool.name=value (e.g. --value repo_path=/src/app)`);
|
|
171
|
+
const key = text.slice(0, eq).trim();
|
|
172
|
+
const raw = text.slice(eq + 1);
|
|
173
|
+
let value = raw;
|
|
174
|
+
try {
|
|
175
|
+
value = JSON.parse(raw);
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// plain text
|
|
179
|
+
}
|
|
180
|
+
return [key, { raw, value }];
|
|
181
|
+
}
|
|
182
|
+
/** A values file: a YAML or JSON map, keys as in --value (`repo_path`, `get_issue.issue_key`). */
|
|
183
|
+
export async function loadValuesFile(path) {
|
|
184
|
+
let doc;
|
|
185
|
+
try {
|
|
186
|
+
doc = parseDocument(await readFile(path, 'utf8'));
|
|
187
|
+
if (doc.errors.length)
|
|
188
|
+
throw doc.errors[0];
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
192
|
+
}
|
|
193
|
+
if (!isMap(doc.contents))
|
|
194
|
+
throw new Error(`${path}: expected a map of parameter names to values (repo_path: /src/app)`);
|
|
195
|
+
const data = doc.toJS();
|
|
196
|
+
const out = {};
|
|
197
|
+
for (const pair of doc.contents.items) {
|
|
198
|
+
const key = String(isScalar(pair.key) ? pair.key.value : pair.key);
|
|
199
|
+
// Grouped per tool (`get_issue: { issue_key: ABC-1 }`) is sorted out against the menu.
|
|
200
|
+
const fields = isMap(pair.value)
|
|
201
|
+
? Object.fromEntries(pair.value.items.map((p) => [String(isScalar(p.key) ? p.key.value : p.key), given(p.value)]))
|
|
202
|
+
: undefined;
|
|
203
|
+
out[key] = { ...given(pair.value, data[key]), ...(fields ? { fields } : {}) };
|
|
204
|
+
}
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* One value from a values file, with the text a scalar was written as: YAML reads
|
|
209
|
+
* `42` as a number and `02134` as 2134, but for a string parameter the user meant
|
|
210
|
+
* the text, as with --value.
|
|
211
|
+
*/
|
|
212
|
+
function given(node, value = isNode(node) ? node.toJSON() : node) {
|
|
213
|
+
const raw = isScalar(node) ? (node.source ?? (typeof node.value === 'string' ? node.value : undefined)) : undefined;
|
|
214
|
+
return { value, ...(raw !== undefined ? { raw } : {}) };
|
|
215
|
+
}
|
|
216
|
+
/** --assume-read-only a,b --assume-read-only c: exact names, no patterns. */
|
|
217
|
+
export function parseAssumeReadOnly(flags) {
|
|
218
|
+
const names = flags.flatMap((f) => f.split(',')).map((s) => s.trim()).filter(Boolean);
|
|
219
|
+
const pattern = names.find((n) => /[*?[\]]/.test(n));
|
|
220
|
+
if (pattern) {
|
|
221
|
+
throw new Error(`--assume-read-only takes exact tool names, not patterns (${pattern}): a pattern would also cover tools a later release adds, which nobody has checked. List the tools.`);
|
|
222
|
+
}
|
|
223
|
+
return [...new Set(names)];
|
|
46
224
|
}
|
|
47
225
|
function rank(tool, args) {
|
|
48
226
|
if (Object.keys(args).length === 0)
|
|
@@ -54,14 +232,88 @@ function rank(tool, args) {
|
|
|
54
232
|
return 2;
|
|
55
233
|
return 3;
|
|
56
234
|
}
|
|
235
|
+
/** Parameters the skipped tools need, most needed first, with how many tools each fills. */
|
|
236
|
+
export function neededValues(skipped) {
|
|
237
|
+
const by = new Map();
|
|
238
|
+
for (const s of skipped)
|
|
239
|
+
if (s.reason === 'needs values')
|
|
240
|
+
for (const m of s.missing ?? [])
|
|
241
|
+
by.set(m, [...(by.get(m) ?? []), s.tool]);
|
|
242
|
+
return [...by].map(([param, tools]) => ({ param, tools })).sort((a, b) => b.tools.length - a.tools.length || a.param.localeCompare(b.param));
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* The one flag or two that would let --auto call more, as a next step: values for
|
|
246
|
+
* the parameters most tools need, the unmarked tools to vouch for, --open-world.
|
|
247
|
+
* Undefined when nothing was left out that a flag can reach.
|
|
248
|
+
*/
|
|
249
|
+
export function autoNextStep(auto) {
|
|
250
|
+
const needs = neededValues(auto.skipped);
|
|
251
|
+
const unmarked = auto.skipped.filter((s) => s.reason === 'unmarked').map((s) => s.tool);
|
|
252
|
+
const openWorld = auto.skipped.filter((s) => s.reason === 'open world').length;
|
|
253
|
+
const parts = [];
|
|
254
|
+
if (needs.length) {
|
|
255
|
+
const top = needs.slice(0, 3).map((n) => `--value ${n.param}=…`).join(' ');
|
|
256
|
+
parts.push(needs.length > 3 ? `${top} (or --save-scenario auto.yml and fill in the other ${needs.length - 3})` : top);
|
|
257
|
+
}
|
|
258
|
+
if (unmarked.length)
|
|
259
|
+
parts.push(`--assume-read-only ${list(unmarked, 4, ',')} for the ones that only read`);
|
|
260
|
+
if (openWorld)
|
|
261
|
+
parts.push('--open-world (may cost API credits)');
|
|
262
|
+
return parts.length ? `Rerun with ${parts.join('; or ')}.` : undefined;
|
|
263
|
+
}
|
|
264
|
+
function list(names, max, sep = ', ') {
|
|
265
|
+
return names.slice(0, max).join(sep) + (names.length > max ? `${sep === ',' ? ',' : sep}…` : '');
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* What --auto called and what it left out, each with the flag that reaches it: the
|
|
269
|
+
* header line of a session report, then one line per reason.
|
|
270
|
+
*/
|
|
271
|
+
export function autoSummary(auto) {
|
|
272
|
+
const n = auto.called.length;
|
|
273
|
+
const extras = [
|
|
274
|
+
auto.withValues?.length ? `${auto.withValues.length} with your --value` : '',
|
|
275
|
+
auto.assumed?.length ? `${auto.assumed.length} on your word, not marked read-only: ${list(auto.assumed, 6)}` : '',
|
|
276
|
+
].filter(Boolean);
|
|
277
|
+
const lines = [`auto: called ${n} tool${n === 1 ? '' : 's'}${extras.length ? ` (${extras.join('; ')})` : ''}`];
|
|
278
|
+
const skip = (reason) => auto.skipped.filter((s) => s.reason === reason);
|
|
279
|
+
const rows = [];
|
|
280
|
+
const needs = skip('needs values');
|
|
281
|
+
if (needs.length) {
|
|
282
|
+
const params = neededValues(auto.skipped);
|
|
283
|
+
const shown = params.slice(0, 4).map((p) => `--value ${p.param}=… (${p.tools.length})`);
|
|
284
|
+
rows.push(`${needs.length} need values the schema doesn't give → ${shown.join(', ')}${params.length > 4 ? `, and ${params.length - 4} more (--save-scenario lists them)` : ''}`);
|
|
285
|
+
}
|
|
286
|
+
const unmarked = skip('unmarked');
|
|
287
|
+
if (unmarked.length)
|
|
288
|
+
rows.push(`${unmarked.length} not marked read-only → --assume-read-only ${list(unmarked.map((s) => s.tool), 4, ',')} if they only read`);
|
|
289
|
+
const writes = skip('not read-only');
|
|
290
|
+
if (writes.length)
|
|
291
|
+
rows.push(`${writes.length} marked or named as writes (never called)`);
|
|
292
|
+
const openWorld = skip('open world');
|
|
293
|
+
if (openWorld.length)
|
|
294
|
+
rows.push(`${openWorld.length} marked openWorldHint → --open-world (may cost API credits)`);
|
|
295
|
+
const over = skip('over the call budget');
|
|
296
|
+
// Unlocks run outside the budget, so called.length would overshoot.
|
|
297
|
+
if (over.length)
|
|
298
|
+
rows.push(`${over.length} over the call budget → --max-calls ${(auto.maxCalls ?? 20) + over.length}`);
|
|
299
|
+
rows.forEach((r, i) => lines.push(`${i === 0 ? 'not called: ' : ' '}${r}`));
|
|
300
|
+
for (const x of auto.stopped ?? [])
|
|
301
|
+
lines.push(`stopped: ${x.tool}: its first value changed nothing and it doesn't say it unlocks tools, so the other ${x.calls} call${x.calls === 1 ? ' was' : 's were'} left out (a scenario can still make them)`);
|
|
302
|
+
for (const x of auto.ignored ?? [])
|
|
303
|
+
lines.push(`ignored: ${x.input}: ${x.why}`);
|
|
304
|
+
return lines;
|
|
305
|
+
}
|
|
57
306
|
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
58
307
|
export function scenarioYaml(plan, serverName) {
|
|
59
308
|
const lines = [
|
|
60
309
|
`# Written by \`toolmenu session --auto\`${serverName ? ` for ${serverName}` : ''}: every read-only tool whose`,
|
|
61
|
-
'# required arguments the schema could fill. Edit freely.',
|
|
310
|
+
'# required arguments your values and the schema could fill. Edit freely.',
|
|
62
311
|
'allow_writes: false',
|
|
63
|
-
'steps:',
|
|
64
312
|
];
|
|
313
|
+
if (plan.assumed.length) {
|
|
314
|
+
lines.push('# Not marked readOnlyHint by the server; called because you said they only read:', `assume_read_only: [${plan.assumed.map((n) => JSON.stringify(n)).join(', ')}]`);
|
|
315
|
+
}
|
|
316
|
+
lines.push('steps:');
|
|
65
317
|
for (const step of plan.scenario.steps) {
|
|
66
318
|
if (step.kind === 'list')
|
|
67
319
|
lines.push(' - list');
|
|
@@ -73,9 +325,11 @@ export function scenarioYaml(plan, serverName) {
|
|
|
73
325
|
}
|
|
74
326
|
const needs = plan.skipped.filter((s) => s.reason === 'needs values');
|
|
75
327
|
if (needs.length) {
|
|
76
|
-
lines.push('', ' # Read-only tools that need values the schema doesn\'t give:');
|
|
328
|
+
lines.push('', ' # Read-only tools that need values the schema doesn\'t give. Fill in the TODOs and uncomment:');
|
|
77
329
|
for (const s of needs.slice(0, 15))
|
|
78
330
|
lines.push(` # - call: ${s.tool}`, ` # args: { ${s.missing.map((m) => `${m}: TODO`).join(', ')} }`);
|
|
331
|
+
if (needs.length > 15)
|
|
332
|
+
lines.push(` # …and ${needs.length - 15} more: ${needs.slice(15).map((s) => s.tool).join(', ')}`);
|
|
79
333
|
}
|
|
80
334
|
return lines.join('\n') + '\n';
|
|
81
335
|
}
|
package/dist/breakdown.d.ts
CHANGED
|
@@ -16,15 +16,44 @@ export interface Breakdown {
|
|
|
16
16
|
values: number;
|
|
17
17
|
tokens: number;
|
|
18
18
|
}[];
|
|
19
|
-
/**
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
/**
|
|
20
|
+
* A block of schema, byte for byte the same content, at several places: inside
|
|
21
|
+
* one tool (a `$defs` entry could hold it once) or across tools (paid once per
|
|
22
|
+
* tool: a tool's schema can't refer to another's). The largest repeated block
|
|
23
|
+
* only, not the pieces inside it.
|
|
24
|
+
*/
|
|
25
|
+
repeated: Repeated[];
|
|
26
|
+
/**
|
|
27
|
+
* `$defs`/`definitions` entries that nothing in their tool refers to, followed
|
|
28
|
+
* transitively: sent with every conversation, read by no one. Notion's server
|
|
29
|
+
* 2.5.2 ships the same 9 in all 24 tools, 72% of its menu.
|
|
30
|
+
*/
|
|
31
|
+
unusedDefs: {
|
|
23
32
|
tools: number;
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
tokens: number;
|
|
34
|
+
examples: string[];
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export interface Repeated {
|
|
38
|
+
/** The property it sits under (the first place it's found), for a label. */
|
|
39
|
+
param: string;
|
|
40
|
+
tokens: number;
|
|
41
|
+
/** Places it appears, in all. */
|
|
42
|
+
count: number;
|
|
43
|
+
tools: number;
|
|
44
|
+
/** tokens × count. */
|
|
45
|
+
total: number;
|
|
46
|
+
/** The most copies inside one tool, and that tool. */
|
|
47
|
+
within: number;
|
|
48
|
+
withinTool: string;
|
|
49
|
+
/** Tokens a `$defs` entry could save inside the tools that repeat it (estimate). */
|
|
50
|
+
saving: number;
|
|
51
|
+
/** Up to three places, as `tool.path`. */
|
|
52
|
+
where: string[];
|
|
26
53
|
}
|
|
27
54
|
/** Where a menu's tokens go. Estimates, like every count in toolmenu. */
|
|
28
55
|
export declare function breakdown(menu: Menu): Breakdown;
|
|
29
56
|
/** The breakdown as report lines, or none for a menu too small to need one. */
|
|
30
57
|
export declare function breakdownLines(b: Breakdown, tools: number): string[];
|
|
58
|
+
/** The breakdown for a PR comment: a fold, the biggest tools as a table, the waste as a list. */
|
|
59
|
+
export declare function breakdownMarkdown(b: Breakdown, tools: number): string[];
|