toolmenu 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/auth.js +6 -4
- package/dist/auto.d.ts +11 -1
- package/dist/auto.js +8 -3
- package/dist/breakdown.js +3 -1
- package/dist/cli.js +100 -25
- package/dist/close.d.ts +8 -0
- package/dist/close.js +18 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/connect.d.ts +10 -0
- package/dist/connect.js +20 -7
- package/dist/diff.js +53 -7
- package/dist/explain.d.ts +10 -0
- package/dist/explain.js +29 -2
- package/dist/failures.d.ts +6 -0
- package/dist/failures.js +42 -2
- package/dist/help.d.ts +2 -0
- package/dist/help.js +13 -6
- package/dist/init.d.ts +1 -0
- package/dist/init.js +1 -1
- package/dist/options.d.ts +6 -0
- package/dist/options.js +2 -0
- package/dist/report.d.ts +1 -1
- package/dist/report.js +76 -2
- package/dist/rules/description.js +4 -3
- package/dist/rules/determinism.d.ts +1 -0
- package/dist/rules/determinism.js +30 -0
- package/dist/rules/gated.d.ts +7 -0
- package/dist/rules/gated.js +77 -0
- package/dist/rules/index.js +6 -2
- package/dist/rules/naming.js +26 -3
- package/dist/rules/rule.d.ts +7 -0
- package/dist/rules/spec.d.ts +6 -0
- package/dist/rules/spec.js +23 -0
- package/dist/rules/write.js +25 -7
- package/dist/session.d.ts +16 -1
- package/dist/session.js +54 -15
- package/dist/snapshot.d.ts +6 -0
- package/dist/snapshot.js +16 -0
- package/dist/types.d.ts +7 -0
- package/package.json +2 -2
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { unlockers } from '../session.js';
|
|
2
|
+
import { singular, VERBS, words } from '../words.js';
|
|
3
|
+
import { names } from './rule.js';
|
|
4
|
+
/**
|
|
5
|
+
* A server that unlocks tools and says its list can change may have served only
|
|
6
|
+
* its starting set. A baseline made from that misses every tool behind the unlock,
|
|
7
|
+
* so a diff can't see their breaking changes (a real server: 18 of 115 tools).
|
|
8
|
+
*/
|
|
9
|
+
export const gated = {
|
|
10
|
+
id: 'menu/gated',
|
|
11
|
+
severity: 'warn',
|
|
12
|
+
summary: 'The menu may be only the starting set of a server that unlocks more tools',
|
|
13
|
+
run(ctx) {
|
|
14
|
+
const listChanged = ctx.capabilities.tools?.listChanged === true;
|
|
15
|
+
if (!listChanged)
|
|
16
|
+
return [];
|
|
17
|
+
// Only an unlock whose parameter lists what it unlocks (an enum of toolsets or
|
|
18
|
+
// domains), the kind session --auto walks: "Enables Cross-Region Restore on a
|
|
19
|
+
// vault" turns on a feature, not tools (Azure, 418 tools).
|
|
20
|
+
const found = unlockers(ctx.menu.tools).filter((u) => u.param && u.values.length);
|
|
21
|
+
if (!found.length)
|
|
22
|
+
return [];
|
|
23
|
+
// A value whose tools are already listed adds nothing: the same server started
|
|
24
|
+
// with every toolset still has its unlock. Where the tools are named
|
|
25
|
+
// namespace-first (content_get, group_list, contentGetEntry), a value matches a
|
|
26
|
+
// tool whose name starts with the value's words (pull_requests → pull_requests_list):
|
|
27
|
+
// an always-on home_getGroups or search_groups isn't the groups domain's. Where
|
|
28
|
+
// they're named verb-first (list_team_audits), every word of the value in the
|
|
29
|
+
// tool's name will do.
|
|
30
|
+
const unlocking = new Set(found.map((u) => u.tool.name));
|
|
31
|
+
const tools = ctx.menu.tools.filter((t) => !unlocking.has(t.name));
|
|
32
|
+
const namespaced = tools.filter((t) => !VERBS.has(singular(words(t.name)[0] ?? ''))).length * 2 > tools.length;
|
|
33
|
+
const startsWith = (name, want) => {
|
|
34
|
+
const have = words(name).map(singular);
|
|
35
|
+
return want.every((w, i) => have[i] === w);
|
|
36
|
+
};
|
|
37
|
+
const missing = (value) => {
|
|
38
|
+
const want = words(String(value)).map(singular);
|
|
39
|
+
if (!want.length)
|
|
40
|
+
return false;
|
|
41
|
+
return namespaced
|
|
42
|
+
? !tools.some((t) => startsWith(t.name, want))
|
|
43
|
+
: !tools.some((t) => {
|
|
44
|
+
const have = new Set(words(t.name).map(singular));
|
|
45
|
+
return want.every((w) => have.has(w));
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
const gaps = found.map((u) => ({ u, missing: u.values.filter(missing).map(String) })).filter((g) => g.missing.length);
|
|
49
|
+
if (!gaps.length)
|
|
50
|
+
return [];
|
|
51
|
+
const { u: first, missing: absent } = gaps[0];
|
|
52
|
+
const more = gaps.slice(1).map((g) => g.u.tool.name);
|
|
53
|
+
// Most values matched: the menu is likely complete, with a few toolsets named for
|
|
54
|
+
// something else (insights → report_get). A hint, not a warning.
|
|
55
|
+
if (absent.length * 2 <= first.values.length) {
|
|
56
|
+
return [
|
|
57
|
+
{
|
|
58
|
+
tool: first.tool.name,
|
|
59
|
+
severity: 'info',
|
|
60
|
+
confidence: 'unsure',
|
|
61
|
+
message: `${first.tool.name} unlocks tools and the server says its tool list can change (listChanged); ${absent.length} of ${first.values.length} values (${names(absent, 5)}) match no tool by name. If their tools are named differently, the menu is complete; if not, it lacks them.`,
|
|
62
|
+
detail: [`unlock: ${first.tool.name} (${first.param}: ${names(first.values.map(String), 12)}); no tool named for: ${names(absent, 8)}${more.length ? `; also ${names(more, 4)}` : ''}`],
|
|
63
|
+
fix: `If ${names(absent, 3)} ${absent.length === 1 ? 'has' : 'have'} tools under other names, nothing to do. Otherwise commit every tool a session sees as the baseline: toolmenu session --auto --union-out menu.json ${ctx.server ?? '-- <the command that starts the server>'}.`,
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
}
|
|
67
|
+
return [
|
|
68
|
+
{
|
|
69
|
+
tool: first.tool.name,
|
|
70
|
+
confidence: 'unsure',
|
|
71
|
+
message: `${first.tool.name} looks like it unlocks more tools, and the server says its tool list can change (listChanged), but no tool in this menu matches ${names(absent, 5)}, so this menu of ${ctx.menu.tools.length} tools may be only the starting set. A baseline made from it misses the tools behind the unlock, and a diff against it can't see their breaking changes.`,
|
|
72
|
+
detail: [`unlock: ${first.tool.name} (${first.param}: ${names(first.values.map(String), 8)}); no tool named for: ${names(absent, 8)}${more.length ? `; also ${names(more, 4)}` : ''}`],
|
|
73
|
+
fix: `Commit every tool a session sees as the baseline instead: toolmenu session --auto --union-out menu.json ${ctx.server ?? '-- <the command that starts the server>'} (it calls each value of ${first.tool.name}). If the server already lists every tool under other names, turn this off with rules: { "menu/gated": "off" }.`,
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
},
|
|
77
|
+
};
|
package/dist/rules/index.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { SEVERITY_RANK } from '../types.js';
|
|
2
2
|
import { buried, cut, lateInstruction } from './description.js';
|
|
3
|
-
import { connectionVariance, nondeterministic, processVariance } from './determinism.js';
|
|
3
|
+
import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js';
|
|
4
4
|
import { authoredIds } from './ids.js';
|
|
5
5
|
import { route, sharedWord, vagueId } from './naming.js';
|
|
6
|
-
import { cacheHints, deprecated, discover, duplicateName, schema, toolsCapability } from './spec.js';
|
|
6
|
+
import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
|
|
7
7
|
import { noDryRun, unannotated } from './write.js';
|
|
8
|
+
import { gated } from './gated.js';
|
|
8
9
|
export const MENU_RULES = [
|
|
9
10
|
nondeterministic,
|
|
10
11
|
duplicateName,
|
|
12
|
+
gated,
|
|
11
13
|
processVariance,
|
|
12
14
|
connectionVariance,
|
|
15
|
+
hostSpecific,
|
|
13
16
|
vagueId,
|
|
14
17
|
sharedWord,
|
|
15
18
|
route,
|
|
@@ -24,6 +27,7 @@ export const MENU_RULES = [
|
|
|
24
27
|
discover,
|
|
25
28
|
deprecated,
|
|
26
29
|
cacheHints,
|
|
30
|
+
serverInfo,
|
|
27
31
|
];
|
|
28
32
|
/**
|
|
29
33
|
* Rules split out of an older one. A setting for the old id still applies to the
|
package/dist/rules/naming.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { COLLECTION_VERBS, LOOKUP_VERBS, WRITE_VERBS, commonWords, nouns, singular, verbOf, words } from '../words.js';
|
|
1
|
+
import { COLLECTION_VERBS, LOOKUP_VERBS, VERBS, WRITE_VERBS, commonWords, nouns, singular, verbOf, words } from '../words.js';
|
|
2
2
|
import { missingWords, negatedMention, routeScores } from '../routes.js';
|
|
3
3
|
import { kept, names } from './rule.js';
|
|
4
4
|
// Not `ref`: in real menus it's a git ref (GitHub ×4), a name with a meaning of its own.
|
|
@@ -57,10 +57,27 @@ export const sharedWord = {
|
|
|
57
57
|
summary: 'The same noun used for different things across tools',
|
|
58
58
|
run(ctx) {
|
|
59
59
|
const byNoun = new Map();
|
|
60
|
-
|
|
61
|
-
const
|
|
60
|
+
// Compared in the singular: "maps" in every name is as common as "map".
|
|
61
|
+
const common = new Set([...commonWords(ctx.menu.tools.map((t) => t.name))].map(singular));
|
|
62
|
+
const subject = (name) => nouns(name).filter((n) => !common.has(singular(n)));
|
|
63
|
+
// A word that opens two or more tool names is a namespace (content_*, maps_*):
|
|
64
|
+
// where it opens the name it isn't the tool's subject, so it isn't compared as
|
|
65
|
+
// a shared word there. It still tells qualifiers apart (content_get_entry vs
|
|
66
|
+
// context_get_entry both say "entry").
|
|
67
|
+
const opens = new Map();
|
|
68
|
+
for (const tool of ctx.menu.tools) {
|
|
69
|
+
const first = singular(words(tool.name)[0] ?? '');
|
|
70
|
+
if (first)
|
|
71
|
+
opens.set(first, (opens.get(first) ?? 0) + 1);
|
|
72
|
+
}
|
|
73
|
+
const namespace = (name) => {
|
|
74
|
+
const first = singular(words(name)[0] ?? '');
|
|
75
|
+
return (opens.get(first) ?? 0) >= 2 && !VERBS.has(first) ? first : undefined;
|
|
76
|
+
};
|
|
62
77
|
for (const tool of ctx.menu.tools) {
|
|
63
78
|
for (const noun of new Set(subject(tool.name))) {
|
|
79
|
+
if (noun === namespace(tool.name))
|
|
80
|
+
continue;
|
|
64
81
|
byNoun.set(noun, [...(byNoun.get(noun) ?? []), tool]);
|
|
65
82
|
}
|
|
66
83
|
}
|
|
@@ -71,6 +88,12 @@ export const sharedWord = {
|
|
|
71
88
|
for (const [noun, tools] of byNoun) {
|
|
72
89
|
if (tools.length > 1 && tools.every((t) => singular(firstWord(t.name) ?? '') === noun))
|
|
73
90
|
continue;
|
|
91
|
+
// In every tool's name of a menu of three or more, it's the server's one subject
|
|
92
|
+
// (DeepWiki: ask_wiki_question, read_wiki_contents, read_wiki_structure), not two
|
|
93
|
+
// things sharing a word. Two tools are too few to tell: list_team_audits and
|
|
94
|
+
// get_audit_trail are the clash this rule exists for.
|
|
95
|
+
if (tools.length >= 3 && tools.length === ctx.menu.tools.length)
|
|
96
|
+
continue;
|
|
74
97
|
// Tools where the noun is qualified differently and neither qualifier set
|
|
75
98
|
// contains the other: list_team_audits vs get_audit_trail.
|
|
76
99
|
const qualifiers = tools.map((t) => ({ tool: t.name, rest: new Set(subject(t.name).filter((n) => n !== noun)) }));
|
package/dist/rules/rule.d.ts
CHANGED
|
@@ -19,6 +19,13 @@ export interface RuleContext {
|
|
|
19
19
|
probes?: import('./determinism.js').Probe[];
|
|
20
20
|
/** The stdio command is a container wrapper (docker run …): env vars may not reach the server. */
|
|
21
21
|
wrapper?: boolean;
|
|
22
|
+
/** serverInfo as the handshake sent it, when there was one (snapshot): spec/server-info. */
|
|
23
|
+
serverInfo?: {
|
|
24
|
+
name?: string;
|
|
25
|
+
version?: string;
|
|
26
|
+
};
|
|
27
|
+
/** How this run reached the server, as the command line ends: `-- node server.js` or the URL. For next steps. */
|
|
28
|
+
server?: string;
|
|
22
29
|
/** Where clients are assumed to cut tool descriptions (description/buried). */
|
|
23
30
|
descriptionLimit?: number | string;
|
|
24
31
|
/** Tools (names or globs) the client sends uncut, so description/buried skips them. */
|
package/dist/rules/spec.d.ts
CHANGED
|
@@ -4,4 +4,10 @@ export declare const toolsCapability: Rule;
|
|
|
4
4
|
export declare const duplicateName: Rule;
|
|
5
5
|
export declare const discover: Rule;
|
|
6
6
|
export declare const deprecated: Rule;
|
|
7
|
+
/**
|
|
8
|
+
* serverInfo { name, version } is required in the handshake (both protocol
|
|
9
|
+
* generations). Without them reports say "unknown server", and clients and logs
|
|
10
|
+
* can't tell which server or release answered (Netlify sends neither).
|
|
11
|
+
*/
|
|
12
|
+
export declare const serverInfo: Rule;
|
|
7
13
|
export declare const cacheHints: Rule;
|
package/dist/rules/spec.js
CHANGED
|
@@ -130,6 +130,29 @@ export const deprecated = {
|
|
|
130
130
|
return [];
|
|
131
131
|
},
|
|
132
132
|
};
|
|
133
|
+
/**
|
|
134
|
+
* serverInfo { name, version } is required in the handshake (both protocol
|
|
135
|
+
* generations). Without them reports say "unknown server", and clients and logs
|
|
136
|
+
* can't tell which server or release answered (Netlify sends neither).
|
|
137
|
+
*/
|
|
138
|
+
export const serverInfo = {
|
|
139
|
+
id: 'spec/server-info',
|
|
140
|
+
severity: 'warn',
|
|
141
|
+
summary: 'The handshake names the server and its version',
|
|
142
|
+
run(ctx) {
|
|
143
|
+
if (!ctx.serverInfo)
|
|
144
|
+
return [];
|
|
145
|
+
const missing = [!ctx.serverInfo.name ? 'name' : '', !ctx.serverInfo.version ? 'version' : ''].filter(Boolean);
|
|
146
|
+
if (!missing.length)
|
|
147
|
+
return [];
|
|
148
|
+
return [
|
|
149
|
+
{
|
|
150
|
+
message: `The server's handshake has no serverInfo ${missing.join(' or ')}, which the spec requires. Clients, logs and this report can't tell which ${missing.length === 2 ? 'server or release' : missing[0] === 'name' ? 'server' : 'release'} answered${ctx.serverInfo.name ? ` (${ctx.serverInfo.name})` : ''}.`,
|
|
151
|
+
fix: 'Set serverInfo { name, version } where the server is created, the version from its package.',
|
|
152
|
+
},
|
|
153
|
+
];
|
|
154
|
+
},
|
|
155
|
+
};
|
|
133
156
|
export const cacheHints = {
|
|
134
157
|
id: 'spec/cache-hints',
|
|
135
158
|
severity: 'info',
|
package/dist/rules/write.js
CHANGED
|
@@ -56,11 +56,16 @@ export const noDryRun = {
|
|
|
56
56
|
lesson: '02',
|
|
57
57
|
summary: 'Destructive tools offer a way to look before leaping',
|
|
58
58
|
run(ctx) {
|
|
59
|
+
// Annotated destructive (destructiveHint: true), or destructive only because the
|
|
60
|
+
// spec says so when a tool says readOnlyHint: false and nothing more: told apart,
|
|
61
|
+
// since "hover" counted as destructive is a missing annotation, not a missing dry run.
|
|
59
62
|
const missing = [];
|
|
63
|
+
const byDefault = [];
|
|
60
64
|
for (const tool of kept(ctx.menu.tools, ctx)) {
|
|
61
65
|
const a = tool.annotations ?? {};
|
|
62
|
-
const
|
|
63
|
-
|
|
66
|
+
const annotated = a.destructiveHint === true;
|
|
67
|
+
const defaulted = !annotated && a.readOnlyHint === false && a.destructiveHint === undefined;
|
|
68
|
+
if (!annotated && !defaulted)
|
|
64
69
|
continue;
|
|
65
70
|
const hasParam = Object.keys(tool.inputSchema?.properties ?? {}).some((p) => DRY_RUN_PARAMS.test(p));
|
|
66
71
|
const subject = new Set(nouns(tool.name));
|
|
@@ -68,19 +73,32 @@ export const noDryRun = {
|
|
|
68
73
|
words(other.name).some((w) => PREVIEW_WORDS.has(w)) &&
|
|
69
74
|
nouns(other.name).some((n) => subject.has(n)));
|
|
70
75
|
if (!hasParam && !hasPreviewTool)
|
|
71
|
-
missing.push(tool.name);
|
|
76
|
+
(annotated ? missing : byDefault).push(tool.name);
|
|
72
77
|
}
|
|
73
|
-
if (missing.length === 0)
|
|
78
|
+
if (missing.length === 0 && byDefault.length === 0)
|
|
74
79
|
return [];
|
|
80
|
+
const defaultLine = `Destructive only by default (readOnlyHint: false and no destructiveHint): ${byDefault.join(', ')}`;
|
|
81
|
+
if (missing.length === 0) {
|
|
82
|
+
const one = byDefault.length === 1;
|
|
83
|
+
return [
|
|
84
|
+
{
|
|
85
|
+
...(one ? { tool: byDefault[0] } : {}),
|
|
86
|
+
confidence: 'unsure',
|
|
87
|
+
message: `${one ? `${byDefault[0]} is` : `${count(byDefault.length, 'tool')} are`} destructive only by default: ${one ? 'it says' : 'they say'} readOnlyHint: false and nothing about destructiveHint, which the spec reads as destructive, and toolmenu found no dry_run/preview-style parameter or preview tool. Clients then confirm ${one ? 'it' : 'them'} like a delete.`,
|
|
88
|
+
...(one ? {} : { detail: [byDefault.join(', ')] }),
|
|
89
|
+
fix: `Set destructiveHint on each: false where it only reads, adds or changes something cheap to undo, true where it deletes or overwrites (and then consider a dry_run parameter).`,
|
|
90
|
+
},
|
|
91
|
+
];
|
|
92
|
+
}
|
|
75
93
|
const one = missing.length === 1;
|
|
76
94
|
// One finding, not one per tool: it's a design note, not a list of defects.
|
|
77
95
|
return [
|
|
78
96
|
{
|
|
79
|
-
...(one ? { tool: missing[0] } : {}),
|
|
97
|
+
...(one && !byDefault.length ? { tool: missing[0] } : {}),
|
|
80
98
|
confidence: 'unsure',
|
|
81
99
|
message: `${one ? `${missing[0]} is` : `${count(missing.length, 'tool')} are`} annotated as destructive, and toolmenu found no dry_run/preview-style parameter and no preview tool for the same thing. Where a mistake is costly to undo, a dry run lets the agent see what would change while it can still back out.`,
|
|
82
|
-
...(one ? {} : { detail: [missing.join(', ')] }),
|
|
83
|
-
fix: `Where the effect is hard to undo, add a dry_run parameter${one ? ` to ${missing[0]}` : ''} that returns what would change without changing it. If it's cheap to undo, or a preview exists under another name, ignore this
|
|
100
|
+
...(one && !byDefault.length ? {} : { detail: [...(one ? [] : [missing.join(', ')]), ...(byDefault.length ? [defaultLine] : [])] }),
|
|
101
|
+
fix: `Where the effect is hard to undo, add a dry_run parameter${one ? ` to ${missing[0]}` : ''} that returns what would change without changing it. If it's cheap to undo, or a preview exists under another name, ignore this.${byDefault.length ? ` For the tools destructive only by default, set destructiveHint: false on the ones that don't delete or overwrite.` : ''}`,
|
|
84
102
|
},
|
|
85
103
|
];
|
|
86
104
|
},
|
package/dist/session.d.ts
CHANGED
|
@@ -43,6 +43,11 @@ export interface StepRecord {
|
|
|
43
43
|
note?: string;
|
|
44
44
|
/** Why the call failed, when it did (a tool error or a failed request). */
|
|
45
45
|
failure?: FailureClass;
|
|
46
|
+
/**
|
|
47
|
+
* What a call step's call did: the tool answered, answered with an error
|
|
48
|
+
* (isError), the request failed, or it wasn't sent (refused, not in the menu).
|
|
49
|
+
*/
|
|
50
|
+
outcome?: 'answered' | 'tool-error' | 'failed' | 'not-sent';
|
|
46
51
|
changed: boolean;
|
|
47
52
|
listChanged: number;
|
|
48
53
|
scope?: Scope;
|
|
@@ -76,6 +81,11 @@ export interface SessionResult {
|
|
|
76
81
|
}
|
|
77
82
|
export interface SessionOptions {
|
|
78
83
|
timeoutMs?: number;
|
|
84
|
+
/**
|
|
85
|
+
* --auto: steps for tools that appear mid-session, run right after the step
|
|
86
|
+
* that brought them. Called once per change, with only the new tools.
|
|
87
|
+
*/
|
|
88
|
+
replan?: (added: MenuTool[]) => Step[];
|
|
79
89
|
/** How long to wait for a list_changed notification after a change. */
|
|
80
90
|
noticeGraceMs?: number;
|
|
81
91
|
rules?: Record<string, Severity | 'off'>;
|
|
@@ -91,12 +101,16 @@ export interface SessionOptions {
|
|
|
91
101
|
rateLimitWaitsMs?: number[];
|
|
92
102
|
}
|
|
93
103
|
/** `each`: this step's line when findings of one rule are said once for several steps. */
|
|
94
|
-
|
|
104
|
+
/** `each`: this step's line when repeats merge. `cause`: what makes two edits one cause. Neither is reported. */
|
|
105
|
+
export type Raw = Omit<Finding, 'severity'> & {
|
|
95
106
|
severity: Severity;
|
|
96
107
|
each?: string;
|
|
108
|
+
cause?: string;
|
|
97
109
|
};
|
|
98
110
|
/** Observe the menu while a scripted session runs. No LLM: the scenario is the agent. */
|
|
99
111
|
export declare function session(target: Target, scenario: Scenario, options?: SessionOptions): Promise<SessionResult>;
|
|
112
|
+
/** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
|
|
113
|
+
export declare function stepsText(steps: number[]): string;
|
|
100
114
|
/** Turn a menu change into findings, with the cost of each. */
|
|
101
115
|
export declare function changeFindings(before: MenuTool[], after: MenuTool[], changes: ToolChange[], step: number, origin?: string): Raw[];
|
|
102
116
|
/**
|
|
@@ -107,6 +121,7 @@ export declare function changeFindings(before: MenuTool[], after: MenuTool[], ch
|
|
|
107
121
|
* a fresh process starts there anyway (an unlock undone reads as "seen" by it).
|
|
108
122
|
*/
|
|
109
123
|
export declare function scopeOf(changes: ToolChange[], next: MenuTool[], probe: MenuTool[], transport: Target['kind'], baseline?: MenuTool[]): Scope;
|
|
124
|
+
export declare function mergeRepeats(raw: Raw[]): Raw[];
|
|
110
125
|
export type { Era };
|
|
111
126
|
interface Unlocker {
|
|
112
127
|
tool: MenuTool;
|
package/dist/session.js
CHANGED
|
@@ -5,10 +5,10 @@ import { listTools } from './connect.js';
|
|
|
5
5
|
import { buildMenu, toolDefinition } from './menu.js';
|
|
6
6
|
import { connectPatiently, isContainerWrapper, MAIN_SEED, probeMenu, probeVariance, seeded } from './probe.js';
|
|
7
7
|
import { varianceFinding } from './rules/determinism.js';
|
|
8
|
-
import { classifyFailure, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
8
|
+
import { classifyFailure, errorWords, FAILURE_LABELS, httpStatus, patiently, RATE_LIMIT_WAITS_MS, quotedSentence, RATE_LIMITED, serverWords, SETUP_FAILURES, setupFailureAdvice, tooMany, waitedFor, whyNot } from './failures.js';
|
|
9
9
|
import { SEVERITY_RANK } from './types.js';
|
|
10
10
|
import { leadingJson } from './catalog.js';
|
|
11
|
-
import { describeToolDifference } from './difference.js';
|
|
11
|
+
import { allDifferences, describeToolDifference } from './difference.js';
|
|
12
12
|
import { autoNextStep, neededValues, writeSign } from './auto.js';
|
|
13
13
|
import { LOOKUP_VERBS, singular, VERBS, verbOf, words, WRITE_VERBS } from './words.js';
|
|
14
14
|
const LIST_CHANGED = 'notifications/tools/list_changed';
|
|
@@ -182,7 +182,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
182
182
|
};
|
|
183
183
|
const callWaited = { ms: 0 };
|
|
184
184
|
const listWaited = { ms: 0 };
|
|
185
|
-
|
|
185
|
+
// Read live: --auto vouches for tools that appear mid-session too.
|
|
186
|
+
const assumed = { has: (tool) => (scenario.assumeReadOnly ?? []).includes(tool) };
|
|
186
187
|
// Called on the user's word (assume_read_only), by step.
|
|
187
188
|
const onWord = [];
|
|
188
189
|
// The same failure from the same tool at several steps is one finding that lists them.
|
|
@@ -230,6 +231,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
230
231
|
if (!tool) {
|
|
231
232
|
record.status = 'failed';
|
|
232
233
|
record.reason = 'missing';
|
|
234
|
+
record.outcome = 'not-sent';
|
|
233
235
|
record.note = `${step.tool} isn't in the menu at this point`;
|
|
234
236
|
const squash = (n) => n.toLowerCase().replace(/[-_.\s]/g, '');
|
|
235
237
|
const close = current.tools.find((t) => squash(t.name) === squash(step.tool));
|
|
@@ -245,6 +247,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
245
247
|
else if (!scenario.allowWrites && tool.annotations?.readOnlyHint !== true && !word) {
|
|
246
248
|
record.status = 'refused';
|
|
247
249
|
record.reason = 'refused';
|
|
250
|
+
record.outcome = 'not-sent';
|
|
248
251
|
record.note = 'not marked readOnlyHint';
|
|
249
252
|
raw.push({
|
|
250
253
|
rule: 'session/refused',
|
|
@@ -275,6 +278,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
275
278
|
error: (e) => httpStatus(e) === 429 || (readOnly && tooMany(e)),
|
|
276
279
|
result: (r) => readOnly && r.isError === true && RATE_LIMITED.test(errorText(r)),
|
|
277
280
|
});
|
|
281
|
+
record.outcome = result.isError ? 'tool-error' : 'answered';
|
|
278
282
|
if (result.isError) {
|
|
279
283
|
const text = errorText(result);
|
|
280
284
|
if (RATE_LIMITED.test(text)) {
|
|
@@ -294,6 +298,7 @@ export async function session(target, scenario, options = {}) {
|
|
|
294
298
|
}
|
|
295
299
|
catch (error) {
|
|
296
300
|
record.status = 'failed';
|
|
301
|
+
record.outcome = 'failed';
|
|
297
302
|
record.note = serverWords(error, 200);
|
|
298
303
|
if (lost(record.note, index)) {
|
|
299
304
|
done();
|
|
@@ -406,6 +411,10 @@ export async function session(target, scenario, options = {}) {
|
|
|
406
411
|
}
|
|
407
412
|
}
|
|
408
413
|
}
|
|
414
|
+
// --auto: the tools this step brought get their calls now, while they're listed.
|
|
415
|
+
const added = new Set(changes.filter((c) => c.kind === 'added').map((c) => c.tool));
|
|
416
|
+
if (options.replan && added.size)
|
|
417
|
+
scenario.steps.splice(i + 1, 0, ...options.replan(next.tools.filter((t) => added.has(t.name))));
|
|
409
418
|
current = next;
|
|
410
419
|
see(next);
|
|
411
420
|
}
|
|
@@ -449,7 +458,8 @@ export async function session(target, scenario, options = {}) {
|
|
|
449
458
|
steps,
|
|
450
459
|
findings: settle(mergeRepeats(raw), options),
|
|
451
460
|
...(options.auto ? { auto: { ...options.auto, ...(leftOut.length ? { stopped: leftOut } : {}) } } : {}),
|
|
452
|
-
|
|
461
|
+
// Marked: its order is the order tools were first seen (unlock order), so diff doesn't compare it.
|
|
462
|
+
union: { ...buildMenu([...seen.values()].map(toolDefinition), baseline.server), from: 'session' },
|
|
453
463
|
};
|
|
454
464
|
}
|
|
455
465
|
finally {
|
|
@@ -562,11 +572,11 @@ function nothingCalled(auto) {
|
|
|
562
572
|
const unmarked = by('unmarked');
|
|
563
573
|
const writes = by('not read-only');
|
|
564
574
|
if (needs.length)
|
|
565
|
-
parts.push(`${needs.length} need values the schema doesn't give`);
|
|
575
|
+
parts.push(`${needs.length} ${needs.length === 1 ? 'needs' : 'need'} values the schema doesn't give`);
|
|
566
576
|
if (unmarked.length)
|
|
567
|
-
parts.push(`${unmarked.length} aren't marked readOnlyHint`);
|
|
577
|
+
parts.push(`${unmarked.length} ${unmarked.length === 1 ? "isn't" : "aren't"} marked readOnlyHint`);
|
|
568
578
|
if (openWorld.length)
|
|
569
|
-
parts.push(`${openWorld.length} marked openWorldHint (they may cost API credits)`);
|
|
579
|
+
parts.push(`${openWorld.length} marked openWorldHint (${openWorld.length === 1 ? 'it' : 'they'} may cost API credits)`);
|
|
570
580
|
if (writes.length)
|
|
571
581
|
parts.push(`${writes.length} marked or named as writes`);
|
|
572
582
|
const list = (names) => names.slice(0, 8).join(', ') + (names.length > 8 ? `, and ${names.length - 8} more` : '');
|
|
@@ -607,9 +617,12 @@ function sameError(failures) {
|
|
|
607
617
|
}
|
|
608
618
|
// The server says the feature isn't there, or isn't for this account: no argument
|
|
609
619
|
// fixes that. A guess from its words.
|
|
610
|
-
|
|
620
|
+
// Only definite statements: a hedge in the server's own advice ("the account may lack
|
|
621
|
+
// permission") says nothing about this call, and a bare "permission" or "role" turns up in
|
|
622
|
+
// ordinary validation messages.
|
|
623
|
+
const UNAVAILABLE = /\b(?:isn't|is not|not|aren't|are not)\s+(?:set up|enabled|configured|available|activated|licensed|provisioned)\b|\bdisabled\b|\b(?:permission|access)s? denied\b|\binsufficient (?:permissions?|privileges?)\b|\b(?:don't|doesn't|do not|does not) have (?:the )?(?:required |necessary )?(?:permissions?|access)\b|\bforbidden\b|\bnot allowed\b/i;
|
|
611
624
|
/** "step 2", "steps 2 and 3", "steps 2, 3 and 5", "steps 1–7 and 9". */
|
|
612
|
-
function stepsText(steps) {
|
|
625
|
+
export function stepsText(steps) {
|
|
613
626
|
if (steps.length === 1)
|
|
614
627
|
return `step ${steps[0]}`;
|
|
615
628
|
const runs = [];
|
|
@@ -637,7 +650,9 @@ function failureFindings(failures, auto) {
|
|
|
637
650
|
const calls = [f, ...(f.others ?? [])];
|
|
638
651
|
const tools = [...new Set(calls.map((c) => c.tool))];
|
|
639
652
|
const who = tools.length === 1 ? f.tool : `${tools.slice(0, -1).join(', ')} and ${tools[tools.length - 1]}`;
|
|
640
|
-
const
|
|
653
|
+
const words = errorWords(f.text);
|
|
654
|
+
const shown = clip(words, 160);
|
|
655
|
+
const said = f.text ? quotedSentence(shown) : 'no error text.';
|
|
641
656
|
const firstArg = Object.keys(f.args)[0];
|
|
642
657
|
const argsCan = f.failure === 'invalid-arguments' || f.failure === 'other';
|
|
643
658
|
// No argument can fix a call to a tool that takes none, or a feature the server
|
|
@@ -671,6 +686,8 @@ function failureFindings(failures, auto) {
|
|
|
671
686
|
step: f.steps[0],
|
|
672
687
|
...(many ? { steps: f.steps } : {}),
|
|
673
688
|
tool: f.tool,
|
|
689
|
+
// The server's whole text, when the message above shows less of it.
|
|
690
|
+
...(f.text && shown !== f.text ? { serverText: f.text } : {}),
|
|
674
691
|
...(!f.timedOutMs && (unavailable || (noParams && argsCan)) ? { confidence: 'unsure' } : {}),
|
|
675
692
|
message: f.rule === 'session/tool-error'
|
|
676
693
|
? tools.length > 1
|
|
@@ -806,7 +823,7 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
806
823
|
const cost = !brk
|
|
807
824
|
? []
|
|
808
825
|
: [
|
|
809
|
-
`
|
|
826
|
+
`cost: the whole tool list (this server's part: ~${total.toLocaleString('en-US')} tokens, estimate) and the conversation after it; the change starts at position ${brk.position}${brk.position >= after.length ? ' (tools removed from the end)' : ''}`,
|
|
810
827
|
];
|
|
811
828
|
const why = origin ? [origin] : [];
|
|
812
829
|
const added = changes.filter((c) => c.kind === 'added');
|
|
@@ -870,11 +887,17 @@ export function changeFindings(before, after, changes, step, origin) {
|
|
|
870
887
|
const b = after.find((t) => t.name === tool);
|
|
871
888
|
// What differs, down to the value: `inputSchema.properties.end_date.default: "…21.5Z" vs "…40.2Z"`.
|
|
872
889
|
const seen = a && b ? [describeToolDifference(a, b).replace(/^[^:]*: /, '')] : [];
|
|
890
|
+
// One cause: the same tool, the same changed values and the same origin (a
|
|
891
|
+
// timestamp in a description, rewritten at every call). A change that touches
|
|
892
|
+
// anything more is a change of its own.
|
|
893
|
+
const paths = a && b ? allDifferences(toolDefinition(a), toolDefinition(b)).map((d) => d.path).sort().join(',') : '';
|
|
873
894
|
out.push({
|
|
874
895
|
rule: 'session/edit',
|
|
875
896
|
severity: 'error',
|
|
876
897
|
step,
|
|
877
898
|
tool,
|
|
899
|
+
cause: `${tool}|${fields.join(',')}|${paths}|${origin ?? ''}`,
|
|
900
|
+
each: `step ${step}: position ${brk?.position ?? '?'} · ~${total.toLocaleString('en-US')} tokens (estimate)${seen[0] ? ` · ${seen[0]}` : ''}`,
|
|
878
901
|
message: `${tool}: ${fields.join(', ')} changed mid-session. Invalidates the cached prompt: the tool list and the conversation after it are processed again.`,
|
|
879
902
|
detail: [...seen, ...cost, ...why],
|
|
880
903
|
fix: origin === 'no tool call in between' ? `Make ${tool}'s definition the same on every tools/list: nothing built from the clock, a random value, or a set's order.` : `Keep ${tool}'s definition fixed for the session; if it has to change, change it between conversations.`,
|
|
@@ -944,11 +967,12 @@ function scopeFindings(scope, modern, step) {
|
|
|
944
967
|
* Findings that say the same thing at every step are said once, naming the steps:
|
|
945
968
|
* stdio's side-effect warning, and an unlock per domain, where each value appends
|
|
946
969
|
* tools for this connection only (9 steps made 18 near-identical blocks). What
|
|
947
|
-
* differs per step, the tools and their cost, goes in the detail. Mid-inserts
|
|
948
|
-
*
|
|
970
|
+
* differs per step, the tools and their cost, goes in the detail. Mid-inserts
|
|
971
|
+
* and removals stay per step: each is its own change. Edits merge only when the
|
|
972
|
+
* tool, the changed values and their origin are the same: one cause.
|
|
949
973
|
*/
|
|
950
974
|
const REPEATS = ['session/side-effect', 'session/append', 'session/connection-local'];
|
|
951
|
-
function mergeRepeats(raw) {
|
|
975
|
+
export function mergeRepeats(raw) {
|
|
952
976
|
let out = raw;
|
|
953
977
|
for (const rule of REPEATS) {
|
|
954
978
|
const same = out.filter((f) => f.rule === rule);
|
|
@@ -968,6 +992,21 @@ function mergeRepeats(raw) {
|
|
|
968
992
|
const merged = { ...same[0], message, steps, ...(detail ? { detail } : {}) };
|
|
969
993
|
out = out.filter((f) => f.rule !== rule || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
970
994
|
}
|
|
995
|
+
// The same edit at several steps is one cause: one finding naming the steps,
|
|
996
|
+
// with each step's position and cost.
|
|
997
|
+
const byCause = new Map();
|
|
998
|
+
for (const f of out)
|
|
999
|
+
if (f.rule === 'session/edit' && f.cause)
|
|
1000
|
+
byCause.set(f.cause, [...(byCause.get(f.cause) ?? []), f]);
|
|
1001
|
+
for (const same of byCause.values()) {
|
|
1002
|
+
if (same.length < 2)
|
|
1003
|
+
continue;
|
|
1004
|
+
const steps = same.map((f) => f.step);
|
|
1005
|
+
// The per-step value and cost are in each step's line; keep the rest (the origin) once.
|
|
1006
|
+
const why = [...new Set(same.flatMap((f) => (f.detail ?? []).filter((d) => !d.startsWith('cost: ') && !f.each.endsWith(` · ${d}`))))];
|
|
1007
|
+
const merged = { ...same[0], steps, message: `${stepsText(steps).replace(/^s/, 'S')}: ${same[0].message}`, detail: [...same.map((f) => f.each), ...why] };
|
|
1008
|
+
out = out.filter((f) => !same.includes(f) || f === same[0]).map((f) => (f === same[0] ? merged : f));
|
|
1009
|
+
}
|
|
971
1010
|
return out;
|
|
972
1011
|
}
|
|
973
1012
|
function describe(changes) {
|
|
@@ -977,7 +1016,7 @@ function settle(raw, options) {
|
|
|
977
1016
|
const ignore = (options.ignore ?? []).map((g) => new RegExp('^' + g.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$'));
|
|
978
1017
|
return raw
|
|
979
1018
|
.filter((f) => options.rules?.[f.rule] !== 'off' && !(f.tool && ignore.some((re) => re.test(f.tool))))
|
|
980
|
-
.map(({ each: _each, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
1019
|
+
.map(({ each: _each, cause: _cause, ...f }) => ({ ...f, severity: options.rules?.[f.rule] ?? f.severity }))
|
|
981
1020
|
.sort((a, b) => (a.step ?? 0) - (b.step ?? 0) || SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity]);
|
|
982
1021
|
}
|
|
983
1022
|
/*
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -22,3 +22,9 @@ export interface SnapshotResult {
|
|
|
22
22
|
}
|
|
23
23
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
24
24
|
export declare function snapshot(target: Target, options?: SnapshotOptions): Promise<SnapshotResult>;
|
|
25
|
+
/**
|
|
26
|
+
* How this run reached the server, as the end of a command line, for next steps:
|
|
27
|
+
* the options that got it past the door (names only; values are the user's) and
|
|
28
|
+
* then `-- command args` or the URL.
|
|
29
|
+
*/
|
|
30
|
+
export declare function serverArgs(target: Target): string;
|
package/dist/snapshot.js
CHANGED
|
@@ -6,6 +6,7 @@ import { connectPatiently, isContainerWrapper, MAIN_SEED, probeVariance, seeded
|
|
|
6
6
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
7
7
|
import { catalogFindings, readCatalog } from './catalog.js';
|
|
8
8
|
import { SEVERITY_RANK } from './types.js';
|
|
9
|
+
import { shellWord } from './init.js';
|
|
9
10
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
10
11
|
export async function snapshot(target, options = {}) {
|
|
11
12
|
// Connecting and listing are read-only: a refusal for being too many is waited
|
|
@@ -57,6 +58,8 @@ export async function snapshot(target, options = {}) {
|
|
|
57
58
|
transport: target.kind,
|
|
58
59
|
probes,
|
|
59
60
|
wrapper: isContainerWrapper(target),
|
|
61
|
+
serverInfo: connection.server,
|
|
62
|
+
server: serverArgs(target),
|
|
60
63
|
}, options);
|
|
61
64
|
// What --catalog called and what it got, under the same settings as the rules
|
|
62
65
|
// (rules: { "catalog/read": "off" }, ignore).
|
|
@@ -72,3 +75,16 @@ export async function snapshot(target, options = {}) {
|
|
|
72
75
|
}
|
|
73
76
|
return { menu, findings };
|
|
74
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* How this run reached the server, as the end of a command line, for next steps:
|
|
80
|
+
* the options that got it past the door (names only; values are the user's) and
|
|
81
|
+
* then `-- command args` or the URL.
|
|
82
|
+
*/
|
|
83
|
+
export function serverArgs(target) {
|
|
84
|
+
if (target.kind === 'stdio') {
|
|
85
|
+
const env = Object.keys(target.env ?? {}).map((k) => `--env ${k}=<value>`);
|
|
86
|
+
return [...env, `-- ${[target.command, ...target.args].map(shellWord).join(' ')}`].join(' ');
|
|
87
|
+
}
|
|
88
|
+
const headers = Object.keys(target.headers ?? {}).map((k) => `--header "${k}: <value>"`);
|
|
89
|
+
return [...headers, ...(target.noAuth ? ['--no-auth'] : []), shellWord(target.url)].join(' ');
|
|
90
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -24,6 +24,8 @@ export interface Finding {
|
|
|
24
24
|
* where, not what it means. Absent: toolmenu observed it directly.
|
|
25
25
|
*/
|
|
26
26
|
confidence?: 'unsure';
|
|
27
|
+
/** For `session` tool errors: the tool's whole error text, when the message shows only part of it. */
|
|
28
|
+
serverText?: string;
|
|
27
29
|
}
|
|
28
30
|
/** A tool exactly as the server returned it, plus toolmenu's token estimate. */
|
|
29
31
|
export interface MenuTool {
|
|
@@ -60,6 +62,11 @@ export interface Menu {
|
|
|
60
62
|
ttlMs?: number;
|
|
61
63
|
cacheScope?: string;
|
|
62
64
|
};
|
|
65
|
+
/**
|
|
66
|
+
* 'session': every tool a session saw (session --union-out), in the order first
|
|
67
|
+
* seen, which is the unlock order, not an order any client was served.
|
|
68
|
+
*/
|
|
69
|
+
from?: 'session';
|
|
63
70
|
/** Operations behind a search tool (snapshot --catalog). Not part of the menu or its tokens. */
|
|
64
71
|
catalog?: import('./catalog.js').Catalog;
|
|
65
72
|
}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "toolmenu",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.1",
|
|
4
4
|
"description": "Lint your MCP server's tool menu for changes that confuse agents or break caches.",
|
|
5
|
-
"author": "
|
|
5
|
+
"author": "Nikša Perović (https://niksa.me)",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"build": "tsc -p tsconfig.json",
|