toolmenu 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -20
- package/dist/args.d.ts +31 -5
- package/dist/args.js +91 -14
- package/dist/auth.d.ts +4 -2
- package/dist/auth.js +53 -17
- package/dist/auto.d.ts +70 -6
- package/dist/auto.js +248 -17
- package/dist/breakdown.d.ts +2 -0
- package/dist/breakdown.js +62 -19
- package/dist/catalog.d.ts +87 -10
- package/dist/catalog.js +415 -58
- package/dist/cli.js +80 -170
- package/dist/config.d.ts +5 -1
- package/dist/config.js +14 -0
- package/dist/connect.d.ts +4 -0
- package/dist/connect.js +132 -11
- package/dist/diff.d.ts +23 -1
- package/dist/diff.js +526 -114
- package/dist/difference.d.ts +2 -0
- package/dist/difference.js +13 -12
- package/dist/explain.d.ts +103 -0
- package/dist/explain.js +434 -0
- package/dist/failures.d.ts +9 -0
- package/dist/failures.js +17 -0
- package/dist/help.d.ts +9 -0
- package/dist/help.js +198 -0
- package/dist/history.d.ts +13 -0
- package/dist/history.js +86 -6
- package/dist/init.d.ts +92 -2
- package/dist/init.js +507 -20
- package/dist/menu.js +10 -2
- package/dist/options.d.ts +135 -0
- package/dist/options.js +48 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +229 -99
- package/dist/rules/description.d.ts +7 -2
- package/dist/rules/description.js +85 -42
- package/dist/rules/determinism.d.ts +3 -1
- package/dist/rules/determinism.js +145 -11
- package/dist/rules/ids.d.ts +1 -1
- package/dist/rules/ids.js +9 -4
- package/dist/rules/index.d.ts +6 -0
- package/dist/rules/index.js +12 -4
- package/dist/rules/naming.d.ts +1 -1
- package/dist/rules/naming.js +66 -19
- package/dist/rules/rule.d.ts +18 -0
- package/dist/rules/rule.js +25 -1
- package/dist/rules/spec.js +33 -9
- package/dist/rules/write.d.ts +1 -1
- package/dist/rules/write.js +42 -10
- package/dist/session.d.ts +18 -8
- package/dist/session.js +484 -92
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +17 -2
- package/dist/types.d.ts +14 -0
- package/dist/words.d.ts +5 -0
- package/dist/words.js +14 -0
- package/package.json +1 -1
package/dist/auto.d.ts
CHANGED
|
@@ -1,28 +1,92 @@
|
|
|
1
1
|
import { type Scenario } from './session.js';
|
|
2
|
-
import type { Menu } from './types.js';
|
|
2
|
+
import type { Menu, MenuTool } from './types.js';
|
|
3
|
+
/** A value the user gave: `raw` is the text as typed on the command line, when it was typed. */
|
|
4
|
+
export interface GivenValue {
|
|
5
|
+
/** The text as typed, for fitValue: `42` for a string parameter is "42". */
|
|
6
|
+
raw?: string;
|
|
7
|
+
value: unknown;
|
|
8
|
+
/** A values file's map grouped by tool: each field with its own text. */
|
|
9
|
+
fields?: Record<string, GivenValue>;
|
|
10
|
+
}
|
|
3
11
|
export interface AutoOptions {
|
|
4
12
|
/** Also call read-only tools marked openWorldHint: true (web search, fetch, scraping). */
|
|
5
13
|
openWorld?: boolean;
|
|
6
14
|
/** At most this many calls (default 20). */
|
|
7
15
|
maxCalls?: number;
|
|
16
|
+
/**
|
|
17
|
+
* Values for required parameters, by name (`repo_path`) or for one tool
|
|
18
|
+
* (`get_issue.issue_key`). Used before anything is synthesized.
|
|
19
|
+
*/
|
|
20
|
+
values?: Record<string, GivenValue>;
|
|
21
|
+
/**
|
|
22
|
+
* Tools the user says only read, although the server doesn't mark them
|
|
23
|
+
* readOnlyHint. Exact names: a pattern would also cover tools a later release
|
|
24
|
+
* adds, which nobody has looked at. A tool marked or named as a write is still
|
|
25
|
+
* never called.
|
|
26
|
+
*/
|
|
27
|
+
assumeReadOnly?: string[];
|
|
8
28
|
}
|
|
9
|
-
export type SkipReason = 'not read-only' | 'open world' | 'needs values' | 'over the call budget';
|
|
29
|
+
export type SkipReason = 'not read-only' | 'unmarked' | 'open world' | 'needs values' | 'over the call budget';
|
|
10
30
|
export interface AutoPlan {
|
|
11
31
|
scenario: Scenario;
|
|
12
32
|
called: string[];
|
|
33
|
+
/** Called on the user's word (--assume-read-only): the server doesn't mark them readOnlyHint. */
|
|
34
|
+
assumed: string[];
|
|
35
|
+
/** Called with at least one value the user gave. */
|
|
36
|
+
withValues: string[];
|
|
13
37
|
skipped: {
|
|
14
38
|
tool: string;
|
|
15
39
|
reason: SkipReason;
|
|
16
40
|
missing?: string[];
|
|
41
|
+
why?: string;
|
|
42
|
+
}[];
|
|
43
|
+
/** --value and --assume-read-only entries that did nothing, and why. */
|
|
44
|
+
ignored: {
|
|
45
|
+
input: string;
|
|
46
|
+
why: string;
|
|
17
47
|
}[];
|
|
18
48
|
}
|
|
49
|
+
/** What a session report shows about an --auto plan. */
|
|
50
|
+
export type AutoSummary = Pick<AutoPlan, 'called' | 'skipped'> & Partial<Pick<AutoPlan, 'assumed' | 'withValues' | 'ignored'>> & {
|
|
51
|
+
maxCalls?: number;
|
|
52
|
+
/** Guessed unlocks whose first value changed nothing, and how many calls to them were left out. */
|
|
53
|
+
stopped?: {
|
|
54
|
+
tool: string;
|
|
55
|
+
calls: number;
|
|
56
|
+
}[];
|
|
57
|
+
};
|
|
58
|
+
/** Marked or named as a write: never called on the user's word. Says why, or undefined. */
|
|
59
|
+
export declare function writeSign(tool: MenuTool): string | undefined;
|
|
19
60
|
/**
|
|
20
61
|
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
21
|
-
* whose required arguments the schema can fill, cheapest
|
|
22
|
-
* then lists and searches, then single lookups, then the
|
|
23
|
-
* call again, which should change nothing. Never a tool that
|
|
24
|
-
* readOnlyHint, whatever allow_writes says
|
|
62
|
+
* whose required arguments the user's values and the schema can fill, cheapest
|
|
63
|
+
* first (no arguments, then lists and searches, then single lookups, then the
|
|
64
|
+
* rest), then the first call again, which should change nothing. Never a tool that
|
|
65
|
+
* isn't marked readOnlyHint, whatever allow_writes says, unless the user named it
|
|
66
|
+
* in assumeReadOnly and nothing marks or names it as a write.
|
|
25
67
|
*/
|
|
26
68
|
export declare function autoScenario(menu: Menu, options?: AutoOptions): AutoPlan;
|
|
69
|
+
/** `name=value` from --value: JSON when it parses (numbers, true, lists), the text otherwise. */
|
|
70
|
+
export declare function parseValueFlag(text: string): [string, GivenValue];
|
|
71
|
+
/** A values file: a YAML or JSON map, keys as in --value (`repo_path`, `get_issue.issue_key`). */
|
|
72
|
+
export declare function loadValuesFile(path: string): Promise<Record<string, GivenValue>>;
|
|
73
|
+
/** --assume-read-only a,b --assume-read-only c: exact names, no patterns. */
|
|
74
|
+
export declare function parseAssumeReadOnly(flags: string[]): string[];
|
|
75
|
+
/** Parameters the skipped tools need, most needed first, with how many tools each fills. */
|
|
76
|
+
export declare function neededValues(skipped: AutoPlan['skipped']): {
|
|
77
|
+
param: string;
|
|
78
|
+
tools: string[];
|
|
79
|
+
}[];
|
|
80
|
+
/**
|
|
81
|
+
* The one flag or two that would let --auto call more, as a next step: values for
|
|
82
|
+
* the parameters most tools need, the unmarked tools to vouch for, --open-world.
|
|
83
|
+
* Undefined when nothing was left out that a flag can reach.
|
|
84
|
+
*/
|
|
85
|
+
export declare function autoNextStep(auto: AutoSummary): string | undefined;
|
|
86
|
+
/**
|
|
87
|
+
* What --auto called and what it left out, each with the flag that reaches it: the
|
|
88
|
+
* header line of a session report, then one line per reason.
|
|
89
|
+
*/
|
|
90
|
+
export declare function autoSummary(auto: AutoSummary): string[];
|
|
27
91
|
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
28
92
|
export declare function scenarioYaml(plan: AutoPlan, serverName?: string): string;
|
package/dist/auto.js
CHANGED
|
@@ -1,16 +1,49 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { isMap, isNode, isScalar, parseDocument } from 'yaml';
|
|
1
3
|
import { synthesizeArgs } from './args.js';
|
|
2
4
|
import { MAX_UNLOCKS, unlockers } from './session.js';
|
|
3
|
-
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf } from './words.js';
|
|
5
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, verbOf, WRITE_VERBS } from './words.js';
|
|
6
|
+
/** Marked or named as a write: never called on the user's word. Says why, or undefined. */
|
|
7
|
+
export function writeSign(tool) {
|
|
8
|
+
const a = tool.annotations ?? {};
|
|
9
|
+
if (a.readOnlyHint === false)
|
|
10
|
+
return 'the server marks it readOnlyHint: false';
|
|
11
|
+
if (a.destructiveHint === true)
|
|
12
|
+
return 'the server marks it destructiveHint: true';
|
|
13
|
+
const verb = verbOf(tool.name);
|
|
14
|
+
if (verb && WRITE_VERBS.has(verb))
|
|
15
|
+
return `its name starts with "${verb}", a write`;
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
4
18
|
/**
|
|
5
19
|
* A scenario built from the menu, no hand-written steps: every read-only tool
|
|
6
|
-
* whose required arguments the schema can fill, cheapest
|
|
7
|
-
* then lists and searches, then single lookups, then the
|
|
8
|
-
* call again, which should change nothing. Never a tool that
|
|
9
|
-
* readOnlyHint, whatever allow_writes says
|
|
20
|
+
* whose required arguments the user's values and the schema can fill, cheapest
|
|
21
|
+
* first (no arguments, then lists and searches, then single lookups, then the
|
|
22
|
+
* rest), then the first call again, which should change nothing. Never a tool that
|
|
23
|
+
* isn't marked readOnlyHint, whatever allow_writes says, unless the user named it
|
|
24
|
+
* in assumeReadOnly and nothing marks or names it as a write.
|
|
10
25
|
*/
|
|
11
26
|
export function autoScenario(menu, options = {}) {
|
|
12
27
|
const maxCalls = options.maxCalls ?? 20;
|
|
13
28
|
const skipped = [];
|
|
29
|
+
const ignored = [];
|
|
30
|
+
const byName = new Map(menu.tools.map((t) => [t.name, t]));
|
|
31
|
+
// --assume-read-only: exact names, and never a write.
|
|
32
|
+
const assumed = new Set();
|
|
33
|
+
for (const name of options.assumeReadOnly ?? []) {
|
|
34
|
+
const tool = byName.get(name);
|
|
35
|
+
if (!tool)
|
|
36
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: 'no tool by that name in the menu' });
|
|
37
|
+
else if (tool.annotations?.readOnlyHint === true)
|
|
38
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: 'the server already marks it readOnlyHint: true' });
|
|
39
|
+
else if (writeSign(tool))
|
|
40
|
+
ignored.push({ input: `--assume-read-only ${name}`, why: `not called: ${writeSign(tool)}` });
|
|
41
|
+
else
|
|
42
|
+
assumed.add(name);
|
|
43
|
+
}
|
|
44
|
+
const readOnly = (t) => t.annotations?.readOnlyHint === true || assumed.has(t.name);
|
|
45
|
+
const { perTool, ignored: unusedValues } = resolveValues(menu, options.values ?? {});
|
|
46
|
+
ignored.push(...unusedValues);
|
|
14
47
|
const candidates = [];
|
|
15
48
|
// Read-only unlocks with every value in an enum: each value, outside the call
|
|
16
49
|
// budget, so the tools behind every toolset are seen (GitHub: 19 toolsets).
|
|
@@ -19,17 +52,17 @@ export function autoScenario(menu, options = {}) {
|
|
|
19
52
|
// server whose every tool calls an outside service marks its unlock that way too,
|
|
20
53
|
// and --auto then tested nothing.
|
|
21
54
|
const unlocks = unlockers(menu.tools).filter((u) => {
|
|
22
|
-
const a = u.tool.annotations ?? {};
|
|
23
55
|
const required = u.tool.inputSchema?.required ?? [];
|
|
24
|
-
return u.param && u.values.length &&
|
|
56
|
+
return u.param && u.values.length && readOnly(u.tool) && required.every((p) => p === u.param);
|
|
25
57
|
});
|
|
26
58
|
const unlocking = new Set(unlocks.map((u) => u.tool.name));
|
|
27
59
|
for (const tool of menu.tools) {
|
|
28
60
|
if (unlocking.has(tool.name))
|
|
29
61
|
continue;
|
|
30
62
|
const a = tool.annotations ?? {};
|
|
31
|
-
if (
|
|
32
|
-
|
|
63
|
+
if (!readOnly(tool)) {
|
|
64
|
+
const write = writeSign(tool);
|
|
65
|
+
skipped.push(write || a.readOnlyHint !== undefined ? { tool: tool.name, reason: 'not read-only', ...(write ? { why: write } : {}) } : { tool: tool.name, reason: 'unmarked' });
|
|
33
66
|
continue;
|
|
34
67
|
}
|
|
35
68
|
// readOnlyHint is the server's own claim, and a read-only search still spends
|
|
@@ -41,12 +74,12 @@ export function autoScenario(menu, options = {}) {
|
|
|
41
74
|
skipped.push({ tool: tool.name, reason: 'open world' });
|
|
42
75
|
continue;
|
|
43
76
|
}
|
|
44
|
-
const filled = synthesizeArgs(tool.inputSchema);
|
|
77
|
+
const filled = synthesizeArgs(tool.inputSchema, perTool.get(tool.name));
|
|
45
78
|
if (!filled.ok) {
|
|
46
79
|
skipped.push({ tool: tool.name, reason: 'needs values', missing: filled.missing });
|
|
47
80
|
continue;
|
|
48
81
|
}
|
|
49
|
-
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args) });
|
|
82
|
+
candidates.push({ tool, args: filled.args, rank: rank(tool, filled.args), valued: Object.values(filled.sources).includes('value') });
|
|
50
83
|
}
|
|
51
84
|
candidates.sort((x, y) => x.rank - y.rank);
|
|
52
85
|
const chosen = candidates.slice(0, maxCalls);
|
|
@@ -58,14 +91,136 @@ export function autoScenario(menu, options = {}) {
|
|
|
58
91
|
for (const u of unlocks) {
|
|
59
92
|
const array = u.tool.inputSchema?.properties?.[u.param]?.type === 'array';
|
|
60
93
|
const values = u.values.slice(0, MAX_UNLOCKS);
|
|
94
|
+
// Guessed from its schema alone (nothing says it unlocks tools): if the first
|
|
95
|
+
// value changes nothing, the session doesn't spend a call on every other one.
|
|
96
|
+
const tentative = u.claims ? {} : { tentative: true };
|
|
61
97
|
for (const v of values)
|
|
62
|
-
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [v] : v } });
|
|
98
|
+
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [v] : v }, ...tentative });
|
|
63
99
|
// The same unlock again should change nothing, as in the --init starter.
|
|
64
|
-
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [values[0]] : values[0] } });
|
|
100
|
+
steps.push({ kind: 'call', tool: u.tool.name, args: { [u.param]: array ? [values[0]] : values[0] }, ...tentative });
|
|
65
101
|
}
|
|
66
102
|
if (chosen.length)
|
|
67
103
|
steps.push({ kind: 'call', tool: chosen[0].tool.name, args: chosen[0].args });
|
|
68
|
-
|
|
104
|
+
const called = [...chosen.map((c) => c.tool.name), ...unlocks.map((u) => u.tool.name)];
|
|
105
|
+
return {
|
|
106
|
+
scenario: { allowWrites: false, steps, ...(assumed.size ? { assumeReadOnly: [...assumed] } : {}) },
|
|
107
|
+
called,
|
|
108
|
+
assumed: called.filter((n) => assumed.has(n)),
|
|
109
|
+
withValues: chosen.filter((c) => c.valued).map((c) => c.tool.name),
|
|
110
|
+
skipped,
|
|
111
|
+
ignored,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The user's values, per tool. `param` fills that required parameter wherever a
|
|
116
|
+
* tool has it; `tool.param` fills it, required or not, on that tool only (and
|
|
117
|
+
* wins over a by-name value). A value no tool takes is reported, not dropped.
|
|
118
|
+
*/
|
|
119
|
+
function resolveValues(menu, values) {
|
|
120
|
+
const perTool = new Map();
|
|
121
|
+
const ignored = [];
|
|
122
|
+
const set = (tool, param, v) => perTool.set(tool, { ...perTool.get(tool), [param]: v });
|
|
123
|
+
const byName = new Map(menu.tools.map((t) => [t.name, t]));
|
|
124
|
+
const targeted = [];
|
|
125
|
+
for (const [key, v] of Object.entries(values)) {
|
|
126
|
+
// A values file can group by tool: `get_issue: { issue_key: ABC-1 }`.
|
|
127
|
+
if (byName.has(key) && v.raw === undefined && v.value && typeof v.value === 'object' && !Array.isArray(v.value)) {
|
|
128
|
+
for (const [param, pv] of Object.entries(v.value)) {
|
|
129
|
+
if (!byName.get(key).inputSchema?.properties?.[param])
|
|
130
|
+
ignored.push({ input: `${key}.${param}`, why: `${key} has no parameter ${param}` });
|
|
131
|
+
else
|
|
132
|
+
targeted.push([key, param, v.fields?.[param] ?? { value: pv, ...(typeof pv === 'string' ? { raw: pv } : {}) }]);
|
|
133
|
+
}
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const dot = key.lastIndexOf('.');
|
|
137
|
+
const toolName = dot > 0 ? key.slice(0, dot) : '';
|
|
138
|
+
if (toolName && byName.has(toolName)) {
|
|
139
|
+
const param = key.slice(dot + 1);
|
|
140
|
+
if (!byName.get(toolName).inputSchema?.properties?.[param])
|
|
141
|
+
ignored.push({ input: `--value ${key}`, why: `${toolName} has no parameter ${param}` });
|
|
142
|
+
else
|
|
143
|
+
targeted.push([toolName, param, v]);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const takers = menu.tools.filter((t) => (t.inputSchema?.required ?? []).includes(key));
|
|
147
|
+
if (takers.length) {
|
|
148
|
+
for (const t of takers)
|
|
149
|
+
set(t.name, key, v);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const optional = menu.tools.filter((t) => t.inputSchema?.properties?.[key]).map((t) => t.name);
|
|
153
|
+
ignored.push({
|
|
154
|
+
input: `--value ${key}`,
|
|
155
|
+
why: optional.length
|
|
156
|
+
? `no tool requires ${key}; to pass it where it's optional, name the tool (--value ${optional[0]}.${key}=…)`
|
|
157
|
+
: toolName
|
|
158
|
+
? `no tool named ${toolName}, and no parameter named ${key}`
|
|
159
|
+
: `no tool takes a parameter named ${key}`,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
for (const [tool, param, v] of targeted)
|
|
163
|
+
set(tool, param, v);
|
|
164
|
+
return { perTool, ignored };
|
|
165
|
+
}
|
|
166
|
+
/** `name=value` from --value: JSON when it parses (numbers, true, lists), the text otherwise. */
|
|
167
|
+
export function parseValueFlag(text) {
|
|
168
|
+
const eq = text.indexOf('=');
|
|
169
|
+
if (eq <= 0)
|
|
170
|
+
throw new Error(`--value ${text}: expected name=value or tool.name=value (e.g. --value repo_path=/src/app)`);
|
|
171
|
+
const key = text.slice(0, eq).trim();
|
|
172
|
+
const raw = text.slice(eq + 1);
|
|
173
|
+
let value = raw;
|
|
174
|
+
try {
|
|
175
|
+
value = JSON.parse(raw);
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// plain text
|
|
179
|
+
}
|
|
180
|
+
return [key, { raw, value }];
|
|
181
|
+
}
|
|
182
|
+
/** A values file: a YAML or JSON map, keys as in --value (`repo_path`, `get_issue.issue_key`). */
|
|
183
|
+
export async function loadValuesFile(path) {
|
|
184
|
+
let doc;
|
|
185
|
+
try {
|
|
186
|
+
doc = parseDocument(await readFile(path, 'utf8'));
|
|
187
|
+
if (doc.errors.length)
|
|
188
|
+
throw doc.errors[0];
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
192
|
+
}
|
|
193
|
+
if (!isMap(doc.contents))
|
|
194
|
+
throw new Error(`${path}: expected a map of parameter names to values (repo_path: /src/app)`);
|
|
195
|
+
const data = doc.toJS();
|
|
196
|
+
const out = {};
|
|
197
|
+
for (const pair of doc.contents.items) {
|
|
198
|
+
const key = String(isScalar(pair.key) ? pair.key.value : pair.key);
|
|
199
|
+
// Grouped per tool (`get_issue: { issue_key: ABC-1 }`) is sorted out against the menu.
|
|
200
|
+
const fields = isMap(pair.value)
|
|
201
|
+
? Object.fromEntries(pair.value.items.map((p) => [String(isScalar(p.key) ? p.key.value : p.key), given(p.value)]))
|
|
202
|
+
: undefined;
|
|
203
|
+
out[key] = { ...given(pair.value, data[key]), ...(fields ? { fields } : {}) };
|
|
204
|
+
}
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* One value from a values file, with the text a scalar was written as: YAML reads
|
|
209
|
+
* `42` as a number and `02134` as 2134, but for a string parameter the user meant
|
|
210
|
+
* the text, as with --value.
|
|
211
|
+
*/
|
|
212
|
+
function given(node, value = isNode(node) ? node.toJSON() : node) {
|
|
213
|
+
const raw = isScalar(node) ? (node.source ?? (typeof node.value === 'string' ? node.value : undefined)) : undefined;
|
|
214
|
+
return { value, ...(raw !== undefined ? { raw } : {}) };
|
|
215
|
+
}
|
|
216
|
+
/** --assume-read-only a,b --assume-read-only c: exact names, no patterns. */
|
|
217
|
+
export function parseAssumeReadOnly(flags) {
|
|
218
|
+
const names = flags.flatMap((f) => f.split(',')).map((s) => s.trim()).filter(Boolean);
|
|
219
|
+
const pattern = names.find((n) => /[*?[\]]/.test(n));
|
|
220
|
+
if (pattern) {
|
|
221
|
+
throw new Error(`--assume-read-only takes exact tool names, not patterns (${pattern}): a pattern would also cover tools a later release adds, which nobody has checked. List the tools.`);
|
|
222
|
+
}
|
|
223
|
+
return [...new Set(names)];
|
|
69
224
|
}
|
|
70
225
|
function rank(tool, args) {
|
|
71
226
|
if (Object.keys(args).length === 0)
|
|
@@ -77,14 +232,88 @@ function rank(tool, args) {
|
|
|
77
232
|
return 2;
|
|
78
233
|
return 3;
|
|
79
234
|
}
|
|
235
|
+
/** Parameters the skipped tools need, most needed first, with how many tools each fills. */
|
|
236
|
+
export function neededValues(skipped) {
|
|
237
|
+
const by = new Map();
|
|
238
|
+
for (const s of skipped)
|
|
239
|
+
if (s.reason === 'needs values')
|
|
240
|
+
for (const m of s.missing ?? [])
|
|
241
|
+
by.set(m, [...(by.get(m) ?? []), s.tool]);
|
|
242
|
+
return [...by].map(([param, tools]) => ({ param, tools })).sort((a, b) => b.tools.length - a.tools.length || a.param.localeCompare(b.param));
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* The one flag or two that would let --auto call more, as a next step: values for
|
|
246
|
+
* the parameters most tools need, the unmarked tools to vouch for, --open-world.
|
|
247
|
+
* Undefined when nothing was left out that a flag can reach.
|
|
248
|
+
*/
|
|
249
|
+
export function autoNextStep(auto) {
|
|
250
|
+
const needs = neededValues(auto.skipped);
|
|
251
|
+
const unmarked = auto.skipped.filter((s) => s.reason === 'unmarked').map((s) => s.tool);
|
|
252
|
+
const openWorld = auto.skipped.filter((s) => s.reason === 'open world').length;
|
|
253
|
+
const parts = [];
|
|
254
|
+
if (needs.length) {
|
|
255
|
+
const top = needs.slice(0, 3).map((n) => `--value ${n.param}=…`).join(' ');
|
|
256
|
+
parts.push(needs.length > 3 ? `${top} (or --save-scenario auto.yml and fill in the other ${needs.length - 3})` : top);
|
|
257
|
+
}
|
|
258
|
+
if (unmarked.length)
|
|
259
|
+
parts.push(`--assume-read-only ${list(unmarked, 4, ',')} for the ones that only read`);
|
|
260
|
+
if (openWorld)
|
|
261
|
+
parts.push('--open-world (may cost API credits)');
|
|
262
|
+
return parts.length ? `Rerun with ${parts.join('; or ')}.` : undefined;
|
|
263
|
+
}
|
|
264
|
+
function list(names, max, sep = ', ') {
|
|
265
|
+
return names.slice(0, max).join(sep) + (names.length > max ? `${sep === ',' ? ',' : sep}…` : '');
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* What --auto called and what it left out, each with the flag that reaches it: the
|
|
269
|
+
* header line of a session report, then one line per reason.
|
|
270
|
+
*/
|
|
271
|
+
export function autoSummary(auto) {
|
|
272
|
+
const n = auto.called.length;
|
|
273
|
+
const extras = [
|
|
274
|
+
auto.withValues?.length ? `${auto.withValues.length} with your --value` : '',
|
|
275
|
+
auto.assumed?.length ? `${auto.assumed.length} on your word, not marked read-only: ${list(auto.assumed, 6)}` : '',
|
|
276
|
+
].filter(Boolean);
|
|
277
|
+
const lines = [`auto: called ${n} tool${n === 1 ? '' : 's'}${extras.length ? ` (${extras.join('; ')})` : ''}`];
|
|
278
|
+
const skip = (reason) => auto.skipped.filter((s) => s.reason === reason);
|
|
279
|
+
const rows = [];
|
|
280
|
+
const needs = skip('needs values');
|
|
281
|
+
if (needs.length) {
|
|
282
|
+
const params = neededValues(auto.skipped);
|
|
283
|
+
const shown = params.slice(0, 4).map((p) => `--value ${p.param}=… (${p.tools.length})`);
|
|
284
|
+
rows.push(`${needs.length} need values the schema doesn't give → ${shown.join(', ')}${params.length > 4 ? `, and ${params.length - 4} more (--save-scenario lists them)` : ''}`);
|
|
285
|
+
}
|
|
286
|
+
const unmarked = skip('unmarked');
|
|
287
|
+
if (unmarked.length)
|
|
288
|
+
rows.push(`${unmarked.length} not marked read-only → --assume-read-only ${list(unmarked.map((s) => s.tool), 4, ',')} if they only read`);
|
|
289
|
+
const writes = skip('not read-only');
|
|
290
|
+
if (writes.length)
|
|
291
|
+
rows.push(`${writes.length} marked or named as writes (never called)`);
|
|
292
|
+
const openWorld = skip('open world');
|
|
293
|
+
if (openWorld.length)
|
|
294
|
+
rows.push(`${openWorld.length} marked openWorldHint → --open-world (may cost API credits)`);
|
|
295
|
+
const over = skip('over the call budget');
|
|
296
|
+
// Unlocks run outside the budget, so called.length would overshoot.
|
|
297
|
+
if (over.length)
|
|
298
|
+
rows.push(`${over.length} over the call budget → --max-calls ${(auto.maxCalls ?? 20) + over.length}`);
|
|
299
|
+
rows.forEach((r, i) => lines.push(`${i === 0 ? 'not called: ' : ' '}${r}`));
|
|
300
|
+
for (const x of auto.stopped ?? [])
|
|
301
|
+
lines.push(`stopped: ${x.tool}: its first value changed nothing and it doesn't say it unlocks tools, so the other ${x.calls} call${x.calls === 1 ? ' was' : 's were'} left out (a scenario can still make them)`);
|
|
302
|
+
for (const x of auto.ignored ?? [])
|
|
303
|
+
lines.push(`ignored: ${x.input}: ${x.why}`);
|
|
304
|
+
return lines;
|
|
305
|
+
}
|
|
80
306
|
/** The plan as a scenario file, to commit and run again with --scenario. */
|
|
81
307
|
export function scenarioYaml(plan, serverName) {
|
|
82
308
|
const lines = [
|
|
83
309
|
`# Written by \`toolmenu session --auto\`${serverName ? ` for ${serverName}` : ''}: every read-only tool whose`,
|
|
84
|
-
'# required arguments the schema could fill. Edit freely.',
|
|
310
|
+
'# required arguments your values and the schema could fill. Edit freely.',
|
|
85
311
|
'allow_writes: false',
|
|
86
|
-
'steps:',
|
|
87
312
|
];
|
|
313
|
+
if (plan.assumed.length) {
|
|
314
|
+
lines.push('# Not marked readOnlyHint by the server; called because you said they only read:', `assume_read_only: [${plan.assumed.map((n) => JSON.stringify(n)).join(', ')}]`);
|
|
315
|
+
}
|
|
316
|
+
lines.push('steps:');
|
|
88
317
|
for (const step of plan.scenario.steps) {
|
|
89
318
|
if (step.kind === 'list')
|
|
90
319
|
lines.push(' - list');
|
|
@@ -96,9 +325,11 @@ export function scenarioYaml(plan, serverName) {
|
|
|
96
325
|
}
|
|
97
326
|
const needs = plan.skipped.filter((s) => s.reason === 'needs values');
|
|
98
327
|
if (needs.length) {
|
|
99
|
-
lines.push('', ' # Read-only tools that need values the schema doesn\'t give:');
|
|
328
|
+
lines.push('', ' # Read-only tools that need values the schema doesn\'t give. Fill in the TODOs and uncomment:');
|
|
100
329
|
for (const s of needs.slice(0, 15))
|
|
101
330
|
lines.push(` # - call: ${s.tool}`, ` # args: { ${s.missing.map((m) => `${m}: TODO`).join(', ')} }`);
|
|
331
|
+
if (needs.length > 15)
|
|
332
|
+
lines.push(` # …and ${needs.length - 15} more: ${needs.slice(15).map((s) => s.tool).join(', ')}`);
|
|
102
333
|
}
|
|
103
334
|
return lines.join('\n') + '\n';
|
|
104
335
|
}
|
package/dist/breakdown.d.ts
CHANGED
|
@@ -55,3 +55,5 @@ export interface Repeated {
|
|
|
55
55
|
export declare function breakdown(menu: Menu): Breakdown;
|
|
56
56
|
/** The breakdown as report lines, or none for a menu too small to need one. */
|
|
57
57
|
export declare function breakdownLines(b: Breakdown, tools: number): string[];
|
|
58
|
+
/** The breakdown for a PR comment: a fold, the biggest tools as a table, the waste as a list. */
|
|
59
|
+
export declare function breakdownMarkdown(b: Breakdown, tools: number): string[];
|
package/dist/breakdown.js
CHANGED
|
@@ -145,32 +145,75 @@ function enumValues(schema) {
|
|
|
145
145
|
return schema.items.enum;
|
|
146
146
|
return [];
|
|
147
147
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
return ` repeated: one ${n(r.tokens)}-token block ×${r.count} in ${r.withinTool} (${r.where.map((w) => w.slice(r.withinTool.length + 1)).join(', ')}${r.count > r.where.length ? ', …' : ''})${r.saving > 0 ? `: a $defs entry could save ${n(r.saving)}` : ''}`;
|
|
155
|
-
return ` repeated: ${r.param} (${n(r.tokens)}) in ${r.tools} tools, ${n(r.total)} in all${inside}`;
|
|
156
|
-
});
|
|
157
|
-
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
148
|
+
const tok = (x) => `~${x.toLocaleString('en-US')}`;
|
|
149
|
+
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
150
|
+
const list = (names, total) => `${names.join(', ')}${total > names.length ? ', …' : ''}`;
|
|
151
|
+
/** A tool's schema can't refer to another tool's: said once, after the repeated blocks that span tools. */
|
|
152
|
+
const ACROSS_TOOLS = "A tool's schema can't $ref another tool's, so a block shared across tools is paid once per tool: only a smaller block saves tokens there.";
|
|
153
|
+
function items(b, tools) {
|
|
158
154
|
const u = b.unusedDefs;
|
|
159
155
|
const unused = u.tokens
|
|
160
|
-
? [
|
|
156
|
+
? [{
|
|
157
|
+
kind: 'unused',
|
|
158
|
+
text: `unused $defs: ${tok(u.tokens)} tokens${b.total ? ` (${pct(u.tokens / b.total)} of the menu)` : ''} in ${u.tools} tool${u.tools === 1 ? '' : 's'} (${list(u.examples, u.tools)})`,
|
|
159
|
+
why: 'Definitions nothing in their tool refers to: sent with every conversation, read by no one.',
|
|
160
|
+
fix: "Remove the unreferenced $defs entries from each tool's inputSchema.",
|
|
161
|
+
}]
|
|
161
162
|
: [];
|
|
163
|
+
const repeated = b.repeated.map((r) => {
|
|
164
|
+
if (r.tools === 1) {
|
|
165
|
+
const places = list(r.where.map((w) => w.slice(r.withinTool.length + 1)), r.count);
|
|
166
|
+
return { kind: 'repeated', text: `repeated: one ${tok(r.tokens)}-token block ×${r.count} in ${r.withinTool} (${places})${r.saving > 0 ? `: a $defs entry could save ${tok(r.saving)}` : ''}`, ...(r.saving > 0 ? { fix: `Move the block into ${r.withinTool}'s $defs once and $ref it.` } : {}) };
|
|
167
|
+
}
|
|
168
|
+
const inside = r.within > 1 ? `, ×${r.within} inside ${r.withinTool}${r.saving > 0 ? `: a $defs entry there could save ${tok(r.saving)}` : ''}` : '';
|
|
169
|
+
return { kind: 'repeated', text: `repeated: ${r.param} (${tok(r.tokens)} tokens) in ${r.tools} tools, ${tok(r.total)} in all${inside}` };
|
|
170
|
+
});
|
|
162
171
|
// A small menu gets no breakdown, but waste inside one tool is worth a line anyway.
|
|
163
172
|
if (tools < 5) {
|
|
164
173
|
const within = [...unused, ...repeated.filter((_, i) => b.repeated[i].within > 1)];
|
|
165
|
-
return within.length ?
|
|
174
|
+
return within.length ? { title: 'Where the tokens go (estimate):', items: within } : undefined;
|
|
166
175
|
}
|
|
167
176
|
const topShare = b.top.reduce((s, t) => s + t.share, 0);
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
177
|
+
const width = Math.min(40, Math.max(...b.top.map((t) => t.name.length)));
|
|
178
|
+
const top = b.top.map((t) => ({ kind: 'top', text: `${pct(t.share).padStart(4)} ${t.name.padEnd(width)} ${tok(t.tokens).padStart(7)} tokens (description ${tok(t.description)}, schema ${tok(t.schema)})` }));
|
|
179
|
+
const enums = b.enums.map((e) => ({ kind: 'enum', text: `enum: ${e.tool}.${e.param} has ${e.values} values, ${tok(e.tokens)} tokens` }));
|
|
180
|
+
const note = b.repeated.some((r) => r.tools > 1) ? [{ kind: 'note', text: ACROSS_TOOLS }] : [];
|
|
181
|
+
return { title: `Where the tokens go (estimate): the top ${b.top.length} tools are ${pct(topShare)} of the menu`, items: [...top, ...unused, ...enums, ...repeated, ...note] };
|
|
182
|
+
}
|
|
183
|
+
/** The breakdown as report lines, or none for a menu too small to need one. */
|
|
184
|
+
export function breakdownLines(b, tools) {
|
|
185
|
+
const found = items(b, tools);
|
|
186
|
+
if (!found)
|
|
187
|
+
return [];
|
|
188
|
+
const lines = [found.title];
|
|
189
|
+
for (const item of found.items) {
|
|
190
|
+
lines.push(item.kind === 'note' ? ` (${item.text})` : ` ${item.text}`);
|
|
191
|
+
if (item.why)
|
|
192
|
+
lines.push(` ${item.why}`);
|
|
193
|
+
if (item.fix)
|
|
194
|
+
lines.push(` → Next: ${item.fix}`);
|
|
195
|
+
}
|
|
196
|
+
return lines;
|
|
197
|
+
}
|
|
198
|
+
/** The breakdown for a PR comment: a fold, the biggest tools as a table, the waste as a list. */
|
|
199
|
+
export function breakdownMarkdown(b, tools) {
|
|
200
|
+
const found = items(b, tools);
|
|
201
|
+
if (!found)
|
|
202
|
+
return [];
|
|
203
|
+
const cell = (t) => t.replace(/\|/g, '\\|');
|
|
204
|
+
const code = (t) => t.replace(/\$defs/g, '`$defs`');
|
|
205
|
+
const lines = ['', `<details><summary>${found.title}</summary>`, ''];
|
|
206
|
+
if (tools >= 5) {
|
|
207
|
+
lines.push('| Share | Tool | Tokens | Description | Schema |', '|---:|---|---:|---:|---:|');
|
|
208
|
+
for (const t of b.top)
|
|
209
|
+
lines.push(`| ${pct(t.share)} | \`${cell(t.name)}\` | ${tok(t.tokens)} | ${tok(t.description)} | ${tok(t.schema)} |`);
|
|
210
|
+
lines.push('');
|
|
211
|
+
}
|
|
212
|
+
for (const item of found.items.filter((i) => i.kind !== 'top' && i.kind !== 'note')) {
|
|
213
|
+
lines.push(`- ${code(item.text)}${item.why ? `. ${code(item.why)}` : ''}${item.fix ? `<br>**→ Next:** ${code(item.fix)}` : ''}`);
|
|
214
|
+
}
|
|
215
|
+
for (const item of found.items.filter((i) => i.kind === 'note'))
|
|
216
|
+
lines.push('', `_${code(item.text)}_`);
|
|
217
|
+
lines.push('', '</details>');
|
|
175
218
|
return lines;
|
|
176
219
|
}
|