toolmenu 0.14.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/breakdown.js +25 -7
- package/dist/catalog.js +4 -0
- package/dist/cli.js +10 -3
- package/dist/connect.d.ts +5 -5
- package/dist/connect.js +26 -1
- package/dist/diff.js +20 -0
- package/dist/explain.d.ts +8 -0
- package/dist/explain.js +51 -7
- package/dist/failures.d.ts +1 -1
- package/dist/failures.js +19 -6
- package/dist/help.js +2 -1
- package/dist/mrtr.d.ts +41 -0
- package/dist/mrtr.js +95 -0
- package/dist/options.d.ts +3 -0
- package/dist/options.js +1 -0
- package/dist/rules/gated.js +4 -1
- package/dist/rules/headers.d.ts +14 -0
- package/dist/rules/headers.js +116 -0
- package/dist/rules/ids.js +2 -1
- package/dist/rules/index.js +6 -0
- package/dist/rules/refs.d.ts +15 -0
- package/dist/rules/refs.js +125 -0
- package/dist/rules/search.d.ts +2 -0
- package/dist/rules/search.js +70 -0
- package/dist/search.d.ts +24 -0
- package/dist/search.js +86 -0
- package/dist/session.d.ts +9 -1
- package/dist/session.js +184 -24
- package/package.json +1 -1
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { count, names } from './rule.js';
|
|
2
|
+
/*
|
|
3
|
+
* x-mcp-header (2026-07-28, SEP-2243): a tool parameter mirrored into an
|
|
4
|
+
* `Mcp-Param-{name}` HTTP header. "Clients using the Streamable HTTP transport
|
|
5
|
+
* MUST reject tool definitions where any x-mcp-header value violates these
|
|
6
|
+
* constraints ... the client MUST exclude the invalid tool from the result of
|
|
7
|
+
* tools/list" (server/tools.mdx). So a bad annotation hides the tool, and the
|
|
8
|
+
* server never hears about it: the official SDK client only logs a warning.
|
|
9
|
+
*
|
|
10
|
+
* The checks follow the spec and the SDK client's own scan
|
|
11
|
+
* (scanXMcpHeaderDeclarations in @modelcontextprotocol/client), which decides
|
|
12
|
+
* what SDK-based clients actually drop. Where the two differ (`number`: the spec
|
|
13
|
+
* forbids it, the SDK accepts it), the finding says so.
|
|
14
|
+
*/
|
|
15
|
+
const KEY = 'x-mcp-header';
|
|
16
|
+
// RFC 9110 §5.1 token: 1*tchar. No spaces, control characters or delimiters.
|
|
17
|
+
const TOKEN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
|
|
18
|
+
const PRIMITIVE = new Set(['string', 'integer', 'boolean']);
|
|
19
|
+
// Keywords a header's chain of `properties` keys must not pass through.
|
|
20
|
+
const UNREACHABLE = [
|
|
21
|
+
'items', 'prefixItems', 'contains', 'additionalProperties', 'unevaluatedProperties', 'unevaluatedItems',
|
|
22
|
+
'propertyNames', 'patternProperties', 'dependentSchemas', 'oneOf', 'anyOf', 'allOf', 'not', 'if', 'then', 'else',
|
|
23
|
+
'$defs', 'definitions',
|
|
24
|
+
];
|
|
25
|
+
const NAMED = new Set(['patternProperties', 'dependentSchemas', '$defs', 'definitions']);
|
|
26
|
+
/** Every broken x-mcp-header annotation in a tool's input schema; empty when all are valid or there are none. */
|
|
27
|
+
export function headerFaults(tool) {
|
|
28
|
+
const faults = [];
|
|
29
|
+
const seen = new Map();
|
|
30
|
+
const visit = (node, path, reachable) => {
|
|
31
|
+
if (!node || typeof node !== 'object' || Array.isArray(node))
|
|
32
|
+
return;
|
|
33
|
+
const schema = node;
|
|
34
|
+
const where = path.length ? path.join('.') : 'the schema root';
|
|
35
|
+
if (KEY in schema) {
|
|
36
|
+
const raw = schema[KEY];
|
|
37
|
+
const fault = (problem, dropped = true) => faults.push({ path: where, header: raw, problem, dropped });
|
|
38
|
+
if (!reachable || path.length === 0) {
|
|
39
|
+
fault(path.length === 0
|
|
40
|
+
? 'it sits on the schema root, not on a parameter'
|
|
41
|
+
: `it sits under ${path.find((p) => p.startsWith('<'))?.slice(1, -1) ?? 'a keyword'}; it's only allowed on a parameter reached through "properties" alone (not items, additionalProperties, oneOf/anyOf/allOf/not, if/then/else or $defs)`);
|
|
42
|
+
}
|
|
43
|
+
else if (typeof raw !== 'string' || raw.length === 0) {
|
|
44
|
+
fault('the header name must be a non-empty string');
|
|
45
|
+
}
|
|
46
|
+
else if (!TOKEN.test(raw)) {
|
|
47
|
+
fault(`"${raw}" isn't a valid HTTP header name (RFC 9110 token: no spaces, control characters or delimiters such as ( ) / : ; = ? @ [ ] { } " ,)`);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
const type = schema.type;
|
|
51
|
+
if (typeof type !== 'string') {
|
|
52
|
+
fault(type === undefined ? 'the parameter has no "type"; it must be string, integer or boolean' : `the parameter's type is ${JSON.stringify(type)}; it must be one of string, integer or boolean, not a list`);
|
|
53
|
+
}
|
|
54
|
+
else if (type === 'number') {
|
|
55
|
+
fault('the parameter is a number; the spec allows only string, integer or boolean (the official SDK client accepts number for now, other clients may not)', false);
|
|
56
|
+
}
|
|
57
|
+
else if (!PRIMITIVE.has(type)) {
|
|
58
|
+
fault(`the parameter is ${type === 'array' || type === 'object' ? 'an' : 'a'} ${type}; it must be string, integer or boolean`);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
const prior = seen.get(raw.toLowerCase());
|
|
62
|
+
if (prior !== undefined)
|
|
63
|
+
fault(`"${raw}" is already used by ${prior}; header names must differ ignoring case`);
|
|
64
|
+
else
|
|
65
|
+
seen.set(raw.toLowerCase(), `${where} ("${raw}")`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const props = schema.properties;
|
|
70
|
+
if (props && typeof props === 'object' && !Array.isArray(props)) {
|
|
71
|
+
for (const [key, child] of Object.entries(props))
|
|
72
|
+
visit(child, [...path, key], reachable);
|
|
73
|
+
}
|
|
74
|
+
for (const k of UNREACHABLE) {
|
|
75
|
+
const sub = schema[k];
|
|
76
|
+
if (sub === undefined)
|
|
77
|
+
continue;
|
|
78
|
+
const branches = Array.isArray(sub) ? sub : sub && typeof sub === 'object' && NAMED.has(k) ? Object.values(sub) : [sub];
|
|
79
|
+
for (const b of branches)
|
|
80
|
+
visit(b, [...path, `<${k}>`], false);
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
visit(tool.inputSchema, [], true);
|
|
84
|
+
return faults;
|
|
85
|
+
}
|
|
86
|
+
export const xMcpHeader = {
|
|
87
|
+
id: 'spec/x-mcp-header',
|
|
88
|
+
severity: 'error',
|
|
89
|
+
since: '2026-07-28',
|
|
90
|
+
summary: 'x-mcp-header annotations are valid, so HTTP clients keep the tool',
|
|
91
|
+
run(ctx) {
|
|
92
|
+
const findings = [];
|
|
93
|
+
const http = ctx.transport === 'http';
|
|
94
|
+
for (const tool of ctx.menu.tools) {
|
|
95
|
+
const faults = headerFaults(tool);
|
|
96
|
+
if (!faults.length)
|
|
97
|
+
continue;
|
|
98
|
+
const dropped = faults.some((f) => f.dropped);
|
|
99
|
+
const where = faults.map((f) => f.path);
|
|
100
|
+
findings.push({
|
|
101
|
+
tool: tool.name,
|
|
102
|
+
// Over HTTP the tool is gone for every compliant client. On stdio clients may
|
|
103
|
+
// ignore the annotation, but the same server is often deployed over HTTP too.
|
|
104
|
+
severity: dropped ? (http ? 'error' : 'warn') : 'warn',
|
|
105
|
+
message: dropped
|
|
106
|
+
? `${tool.name} has ${faults.length === 1 ? 'an x-mcp-header annotation' : `${count(faults.length, 'x-mcp-header annotation')}`} the spec doesn't allow (on ${names(where, 4)}), and Streamable HTTP clients must then leave the whole tool out of their menu. The official SDK client does, with only a console warning, so the agent never sees ${tool.name} and nothing tells the server.${http ? ' toolmenu reads the list off the wire, which is why it is in this menu.' : ' Over stdio clients may ignore the annotation, so it shows here; served over HTTP, it would be gone.'}`
|
|
107
|
+
: `${tool.name} mirrors a number parameter into a header (${names(where, 4)}). The spec allows only string, integer or boolean there, so a strict Streamable HTTP client must leave the tool out of its menu; the official SDK client keeps it for now (its conformance tests expect number to work, a gap tracked upstream).`,
|
|
108
|
+
detail: faults.map((f) => `${f.path}${typeof f.header === 'string' && f.header ? ` (x-mcp-header: "${f.header}")` : ''}: ${f.problem}`),
|
|
109
|
+
fix: dropped
|
|
110
|
+
? `Fix each annotation listed: on a string, integer or boolean parameter reached through "properties" alone, with a header name of letters, digits and - _ . only, different from the tool's other headers ignoring case. Then list the tools over HTTP with an SDK client and check ${tool.name} is there.`
|
|
111
|
+
: 'Make the parameter a string or an integer (a header carries text anyway), or stop mirroring it into a header.',
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
return findings;
|
|
115
|
+
},
|
|
116
|
+
};
|
package/dist/rules/ids.js
CHANGED
|
@@ -3,7 +3,8 @@ import { kept } from './rule.js';
|
|
|
3
3
|
// form_id, form-id, formId, formID. Not uid or paid: the suffix needs a boundary.
|
|
4
4
|
// Not "key" either: in real menus it's as often a keyboard key as an identifier.
|
|
5
5
|
const ID_SUFFIX = /^(.+?)[_-](id|ids|uuid|uuids)$/i;
|
|
6
|
-
|
|
6
|
+
// UUID before ID: viewUUID is a view's ID, not a viewUU's.
|
|
7
|
+
const CAMEL_ID = /^([a-z][a-zA-Z0-9]*?)(UUIDs?|Uuids?|IDs|ID|Ids|Id)$/;
|
|
7
8
|
const BARE_ID = /^(id|ids|uid|uuid)$/i;
|
|
8
9
|
const CREATE_VERBS = new Set(['create', 'add', 'new', 'insert', 'post', 'upsert', 'start', 'run', 'submit', 'duplicate', 'copy']);
|
|
9
10
|
// Words that join alternatives in a name: projectSlugOrId → project.
|
package/dist/rules/index.js
CHANGED
|
@@ -6,6 +6,9 @@ import { route, sharedWord, vagueId } from './naming.js';
|
|
|
6
6
|
import { cacheHints, deprecated, discover, duplicateName, schema, serverInfo, toolsCapability } from './spec.js';
|
|
7
7
|
import { noDryRun, unannotated } from './write.js';
|
|
8
8
|
import { gated } from './gated.js';
|
|
9
|
+
import { xMcpHeader } from './headers.js';
|
|
10
|
+
import { schemaRef } from './refs.js';
|
|
11
|
+
import { findable } from './search.js';
|
|
9
12
|
export const MENU_RULES = [
|
|
10
13
|
nondeterministic,
|
|
11
14
|
duplicateName,
|
|
@@ -16,6 +19,7 @@ export const MENU_RULES = [
|
|
|
16
19
|
vagueId,
|
|
17
20
|
sharedWord,
|
|
18
21
|
route,
|
|
22
|
+
findable,
|
|
19
23
|
authoredIds,
|
|
20
24
|
buried,
|
|
21
25
|
cut,
|
|
@@ -23,6 +27,8 @@ export const MENU_RULES = [
|
|
|
23
27
|
noDryRun,
|
|
24
28
|
unannotated,
|
|
25
29
|
schema,
|
|
30
|
+
xMcpHeader,
|
|
31
|
+
schemaRef,
|
|
26
32
|
toolsCapability,
|
|
27
33
|
discover,
|
|
28
34
|
deprecated,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { MenuTool } from '../types.js';
|
|
2
|
+
import { type Rule } from './rule.js';
|
|
3
|
+
export type Where = 'inputSchema' | 'outputSchema';
|
|
4
|
+
export interface RefFault {
|
|
5
|
+
where: Where;
|
|
6
|
+
/** Where the $ref sits: `filter.region`, `<items>`, `<$defs>.A`. */
|
|
7
|
+
path: string;
|
|
8
|
+
ref: string;
|
|
9
|
+
/** network: an absolute http(s) URI, which clients must not fetch. missing: nothing in the schema it points to. */
|
|
10
|
+
kind: 'network' | 'missing';
|
|
11
|
+
}
|
|
12
|
+
/** Every $ref in one schema that a client can't resolve from the schema itself. */
|
|
13
|
+
export declare function unresolvedRefs(schema: unknown, where: Where): RefFault[];
|
|
14
|
+
export declare function refFaults(tool: MenuTool): RefFault[];
|
|
15
|
+
export declare const schemaRef: Rule;
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { count, names } from './rule.js';
|
|
2
|
+
/** JSON Pointer within the schema: `#/$defs/A`, `#/properties/a/items`. */
|
|
3
|
+
function pointer(root, ref) {
|
|
4
|
+
if (ref === '#' || ref === '#/')
|
|
5
|
+
return true;
|
|
6
|
+
let node = root;
|
|
7
|
+
for (const raw of ref.slice(2).split('/')) {
|
|
8
|
+
const part = decodeURIComponent(raw).replace(/~1/g, '/').replace(/~0/g, '~');
|
|
9
|
+
if (Array.isArray(node) && /^\d+$/.test(part) && Number(part) < node.length)
|
|
10
|
+
node = node[Number(part)];
|
|
11
|
+
else if (node && typeof node === 'object' && part in node)
|
|
12
|
+
node = node[part];
|
|
13
|
+
else
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
/** Every $id and $anchor in a schema, for refs that name them. */
|
|
19
|
+
function targets(node, ids, anchors, depth = 0) {
|
|
20
|
+
if (!node || typeof node !== 'object' || depth > 64)
|
|
21
|
+
return;
|
|
22
|
+
if (Array.isArray(node)) {
|
|
23
|
+
for (const v of node)
|
|
24
|
+
targets(v, ids, anchors, depth + 1);
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const s = node;
|
|
28
|
+
if (typeof s.$id === 'string')
|
|
29
|
+
ids.add(s.$id.replace(/#$/, ''));
|
|
30
|
+
if (typeof s.$anchor === 'string')
|
|
31
|
+
anchors.add(s.$anchor);
|
|
32
|
+
if (typeof s.$dynamicAnchor === 'string')
|
|
33
|
+
anchors.add(s.$dynamicAnchor);
|
|
34
|
+
for (const [k, v] of Object.entries(s))
|
|
35
|
+
if (k !== 'enum' && k !== 'const' && k !== 'default' && k !== 'examples')
|
|
36
|
+
targets(v, ids, anchors, depth + 1);
|
|
37
|
+
}
|
|
38
|
+
/** Every $ref in one schema that a client can't resolve from the schema itself. */
|
|
39
|
+
export function unresolvedRefs(schema, where) {
|
|
40
|
+
if (!schema || typeof schema !== 'object')
|
|
41
|
+
return [];
|
|
42
|
+
const ids = new Set();
|
|
43
|
+
const anchors = new Set();
|
|
44
|
+
targets(schema, ids, anchors);
|
|
45
|
+
const out = [];
|
|
46
|
+
const visit = (node, path, depth) => {
|
|
47
|
+
if (!node || typeof node !== 'object' || depth > 64)
|
|
48
|
+
return;
|
|
49
|
+
if (Array.isArray(node)) {
|
|
50
|
+
node.forEach((v, i) => visit(v, [...path, `${i}`], depth + 1));
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const s = node;
|
|
54
|
+
for (const key of ['$ref', '$dynamicRef']) {
|
|
55
|
+
const ref = s[key];
|
|
56
|
+
if (typeof ref !== 'string')
|
|
57
|
+
continue;
|
|
58
|
+
const at = path.length ? path.join('.') : 'the schema root';
|
|
59
|
+
if (ref.startsWith('#/') || ref === '#') {
|
|
60
|
+
if (!pointer(schema, ref))
|
|
61
|
+
out.push({ where, path: at, ref, kind: 'missing' });
|
|
62
|
+
}
|
|
63
|
+
else if (ref.startsWith('#')) {
|
|
64
|
+
if (!anchors.has(ref.slice(1)))
|
|
65
|
+
out.push({ where, path: at, ref, kind: 'missing' });
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// A ref to a schema's own $id (or a fragment of it) resolves locally.
|
|
69
|
+
const [base, fragment] = ref.split('#');
|
|
70
|
+
const local = [...ids].some((id) => id === base || id.endsWith(`/${base}`));
|
|
71
|
+
if (local && (!fragment || fragment.startsWith('/') || anchors.has(fragment)))
|
|
72
|
+
continue;
|
|
73
|
+
out.push({ where, path: at, ref, kind: /^https?:\/\//i.test(ref) ? 'network' : 'missing' });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
for (const [k, v] of Object.entries(s)) {
|
|
77
|
+
if (k === 'enum' || k === 'const' || k === 'default' || k === 'examples')
|
|
78
|
+
continue;
|
|
79
|
+
visit(v, k === 'properties' ? path : [...path, k === 'items' || k.startsWith('$') || /^(any|one|all)Of$|^not$|^if$|^then$|^else$|Properties$/.test(k) ? `<${k}>` : k], depth + 1);
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
visit(schema, [], 0);
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
export function refFaults(tool) {
|
|
86
|
+
return [...unresolvedRefs(tool.inputSchema, 'inputSchema'), ...unresolvedRefs(tool.outputSchema, 'outputSchema')];
|
|
87
|
+
}
|
|
88
|
+
export const schemaRef = {
|
|
89
|
+
id: 'spec/ref',
|
|
90
|
+
severity: 'error',
|
|
91
|
+
since: '2026-07-28',
|
|
92
|
+
summary: "Every $ref in a tool's schemas resolves inside the schema",
|
|
93
|
+
run(ctx) {
|
|
94
|
+
const findings = [];
|
|
95
|
+
for (const tool of ctx.menu.tools) {
|
|
96
|
+
const faults = refFaults(tool);
|
|
97
|
+
if (!faults.length)
|
|
98
|
+
continue;
|
|
99
|
+
const output = faults.filter((f) => f.where === 'outputSchema');
|
|
100
|
+
const missingInput = faults.filter((f) => f.where === 'inputSchema' && f.kind === 'missing');
|
|
101
|
+
const detail = faults.map((f) => `${f.where} ${f.path}: $ref "${f.ref}" ${f.kind === 'network' ? 'is a network URI, which clients must not fetch' : 'points to nothing in the schema'}`);
|
|
102
|
+
const refs = names([...new Set(faults.map((f) => `"${f.ref}"`))], 3);
|
|
103
|
+
if (output.length) {
|
|
104
|
+
findings.push({
|
|
105
|
+
tool: tool.name,
|
|
106
|
+
message: `${tool.name}'s outputSchema has ${count(output.length, '$ref')} a client can't resolve (${names([...new Set(output.map((f) => `"${f.ref}"`))], 3)}), so the official SDK client fails every call to it ("has an invalid outputSchema: can't resolve reference"), however well the server answers. Clients must not fetch a network URI, so a $ref has to point inside the schema.`,
|
|
107
|
+
detail,
|
|
108
|
+
fix: `Put what the $ref points to in the outputSchema's own $defs and point the $ref there (#/$defs/…), then call ${tool.name} with an SDK client to confirm.`,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
findings.push({
|
|
113
|
+
tool: tool.name,
|
|
114
|
+
severity: missingInput.length ? 'error' : 'warn',
|
|
115
|
+
message: missingInput.length
|
|
116
|
+
? `${tool.name}'s inputSchema has a $ref that points to nothing in the schema (${refs}), so the schema isn't valid: the spec requires schemas to be, the official SDK's server won't register it, and the model reads a parameter with no definition.`
|
|
117
|
+
: `${tool.name}'s inputSchema points to a network URI (${refs}). Clients must not fetch it and should reject the schema rather than treat the parameter as unconstrained, so a strict client can refuse ${tool.name}, and any other sends the model a parameter with no definition.`,
|
|
118
|
+
detail,
|
|
119
|
+
fix: `Put what the $ref points to in the inputSchema's own $defs and point the $ref there (#/$defs/…).`,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return findings;
|
|
124
|
+
},
|
|
125
|
+
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { buildIndex, placeOf, SEARCH_LIMIT } from '../search.js';
|
|
2
|
+
import { commonWords, VERBS, words } from '../words.js';
|
|
3
|
+
import { clip, count, kept, names, num } from './rule.js';
|
|
4
|
+
/**
|
|
5
|
+
* When a client defers tools and lets the model search for them (Claude's tool
|
|
6
|
+
* search, OpenAI's tool_search, Claude Code's deferred MCP tools), a tool the
|
|
7
|
+
* search doesn't return can't be called. The search returns 5 tools unless the
|
|
8
|
+
* model asks for more, so a tool that a search for its own name doesn't put in
|
|
9
|
+
* the top 5 is one the agent can miss.
|
|
10
|
+
*
|
|
11
|
+
* Only for menus a client would defer: the tool search docs suggest it from 10
|
|
12
|
+
* tools or 10K tokens of definitions.
|
|
13
|
+
*/
|
|
14
|
+
const MIN_TOOLS = 10;
|
|
15
|
+
const MIN_TOKENS = 10_000;
|
|
16
|
+
export const findable = {
|
|
17
|
+
id: 'search/findable',
|
|
18
|
+
severity: 'info',
|
|
19
|
+
summary: 'Each tool comes up when searched for by its own name, for clients that defer tools',
|
|
20
|
+
run(ctx) {
|
|
21
|
+
const tools = ctx.menu.tools;
|
|
22
|
+
if (tools.length < MIN_TOOLS || ((ctx.menu.totalTokens ?? 0) < MIN_TOKENS && tools.length < 30))
|
|
23
|
+
return [];
|
|
24
|
+
const index = buildIndex(tools);
|
|
25
|
+
const findings = [];
|
|
26
|
+
// The words of a name that say something: not a prefix most tools share (firecrawl_, API-).
|
|
27
|
+
const shared = commonWords(tools.map((t) => t.name));
|
|
28
|
+
const missed = [];
|
|
29
|
+
for (const t of kept(tools, ctx)) {
|
|
30
|
+
// A bare verb ("create" from ads_create) isn't how an agent searches: then the
|
|
31
|
+
// prefix stays, since it's the object the tool acts on.
|
|
32
|
+
const all = words(t.name);
|
|
33
|
+
const own = all.filter((w) => !shared.has(w));
|
|
34
|
+
const query = (own.some((w) => !VERBS.has(w)) ? own : all).join(' ');
|
|
35
|
+
if (!query || !all.some((w) => !VERBS.has(w)))
|
|
36
|
+
continue;
|
|
37
|
+
const { place, ahead } = placeOf(index, query, t.name);
|
|
38
|
+
if (place === undefined || place > SEARCH_LIMIT)
|
|
39
|
+
missed.push({ tool: t.name, query, place, ahead });
|
|
40
|
+
}
|
|
41
|
+
if (missed.length) {
|
|
42
|
+
const first = missed[0];
|
|
43
|
+
findings.push({
|
|
44
|
+
...(missed.length === 1 ? { tool: first.tool } : {}),
|
|
45
|
+
confidence: 'unsure',
|
|
46
|
+
message: `${count(missed.length, 'tool')} of ${num(tools.length)} ${missed.length === 1 ? "doesn't" : "don't"} come up in the top ${SEARCH_LIMIT} when searched for by ${missed.length === 1 ? 'its' : 'their'} own name. Clients that defer a menu this size let the model search it, and a search returns ${SEARCH_LIMIT} tools unless the model asks for more, so an agent looking for ${missed.length === 1 ? 'it' : 'them'} can be shown other tools instead. Ranked the way a BM25 tool search ranks them (name, description, argument names and descriptions); a client's own search can rank a little differently.`,
|
|
47
|
+
detail: missed.slice(0, 12).map((m) => `${m.tool}: “${m.query}” puts it ${m.place === undefined ? 'nowhere' : `at ${m.place}`}, behind ${names(m.ahead.slice(0, 4), 4)}${m.ahead.length > 4 ? ` and ${num(m.ahead.length - 4)} more` : ''}`).concat(missed.length > 12 ? [`and ${num(missed.length - 12)} more`] : []),
|
|
48
|
+
fix: 'Say the words of each tool\'s name in its own description, early on (a search ranks the description too, and "Returns the llms.txt" never says "guide"), give it a word the tools ahead of it lack (the object it acts on, not the verb), and cut repeats of other tools\' words from long descriptions: a description that mentions "domain" ten times outranks the tool named for domains.',
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
// routes.yml: a route's tools should be in what a search for it returns.
|
|
52
|
+
for (const [keyword, expectation] of Object.entries(ctx.routes ?? {})) {
|
|
53
|
+
const present = (expectation.must_match ?? []).filter((n) => tools.some((t) => t.name === n));
|
|
54
|
+
const lost = present
|
|
55
|
+
.map((n) => ({ n, ...placeOf(index, keyword, n) }))
|
|
56
|
+
.filter((r) => r.place === undefined || r.place > SEARCH_LIMIT);
|
|
57
|
+
for (const r of lost) {
|
|
58
|
+
findings.push({
|
|
59
|
+
tool: r.n,
|
|
60
|
+
severity: 'warn',
|
|
61
|
+
confidence: 'unsure',
|
|
62
|
+
message: `routes.yml says “${keyword}” should reach ${r.n}, but a tool search for “${keyword}” ${r.place === undefined ? "doesn't return it at all" : `puts it at ${r.place}`}, past the ${SEARCH_LIMIT} a search returns by default. A client that defers tools may never show it.`,
|
|
63
|
+
detail: [`ahead of it: ${names(r.ahead.slice(0, 5), 5)}`, `${r.n}: ${clip(tools.find((t) => t.name === r.n)?.description ?? '', 120)}`],
|
|
64
|
+
fix: `Put the words of “${keyword}” in ${r.n}'s name or the start of its description, or take them out of the tools ahead of it.`,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return findings;
|
|
69
|
+
},
|
|
70
|
+
};
|
package/dist/search.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { MenuTool } from './types.js';
|
|
2
|
+
/** How many tools a search returns when the model doesn't set a limit. */
|
|
3
|
+
export declare const SEARCH_LIMIT = 5;
|
|
4
|
+
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
5
|
+
export declare function searchText(tool: MenuTool): string[];
|
|
6
|
+
/** Lowercase, singular, without stop words: what a query and a document are compared on. */
|
|
7
|
+
export declare function terms(text: string): string[];
|
|
8
|
+
export interface SearchIndex {
|
|
9
|
+
/** Tools ranked for a query, best first, with their scores. Tools that match no term are left out. */
|
|
10
|
+
rank(query: string): {
|
|
11
|
+
tool: string;
|
|
12
|
+
score: number;
|
|
13
|
+
}[];
|
|
14
|
+
}
|
|
15
|
+
export declare function buildIndex(tools: MenuTool[]): SearchIndex;
|
|
16
|
+
/**
|
|
17
|
+
* Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
|
|
18
|
+
* (only tools that score strictly higher are ahead of it), so a miss is never a
|
|
19
|
+
* coin flip. Undefined when the query matches nothing in the tool.
|
|
20
|
+
*/
|
|
21
|
+
export declare function placeOf(index: SearchIndex, query: string, tool: string): {
|
|
22
|
+
place?: number;
|
|
23
|
+
ahead: string[];
|
|
24
|
+
};
|
package/dist/search.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { singular, words } from './words.js';
|
|
2
|
+
/** How many tools a search returns when the model doesn't set a limit. */
|
|
3
|
+
export const SEARCH_LIMIT = 5;
|
|
4
|
+
// Words that carry no meaning in a query or a description.
|
|
5
|
+
const STOP = new Set([
|
|
6
|
+
'a', 'an', 'the', 'of', 'for', 'to', 'in', 'on', 'at', 'by', 'from', 'with', 'and', 'or', 'is', 'are', 'be',
|
|
7
|
+
'this', 'that', 'it', 'its', 'as', 'use', 'when', 'if', 'you', 'your', 'can', 'will', 'not', 'any', 'all',
|
|
8
|
+
]);
|
|
9
|
+
const K1 = 1.2;
|
|
10
|
+
const B = 0.75;
|
|
11
|
+
/** The words a search sees in a tool: its name, description, and every argument's name and description. */
|
|
12
|
+
export function searchText(tool) {
|
|
13
|
+
const parts = [tool.name, tool.description ?? ''];
|
|
14
|
+
const walk = (schema, depth) => {
|
|
15
|
+
if (!schema || typeof schema !== 'object' || depth > 6)
|
|
16
|
+
return;
|
|
17
|
+
const s = schema;
|
|
18
|
+
for (const [name, prop] of Object.entries(s.properties ?? {})) {
|
|
19
|
+
parts.push(name);
|
|
20
|
+
const d = prop?.description;
|
|
21
|
+
if (typeof d === 'string')
|
|
22
|
+
parts.push(d);
|
|
23
|
+
walk(prop, depth + 1);
|
|
24
|
+
}
|
|
25
|
+
if (s.items)
|
|
26
|
+
walk(s.items, depth + 1);
|
|
27
|
+
};
|
|
28
|
+
walk(tool.inputSchema, 0);
|
|
29
|
+
return terms(parts.join(' '));
|
|
30
|
+
}
|
|
31
|
+
/** Lowercase, singular, without stop words: what a query and a document are compared on. */
|
|
32
|
+
export function terms(text) {
|
|
33
|
+
return words(text)
|
|
34
|
+
.map(singular)
|
|
35
|
+
.filter((w) => w.length > 1 && !STOP.has(w));
|
|
36
|
+
}
|
|
37
|
+
export function buildIndex(tools) {
|
|
38
|
+
const docs = tools.map((t) => {
|
|
39
|
+
const tf = new Map();
|
|
40
|
+
const text = searchText(t);
|
|
41
|
+
for (const w of text)
|
|
42
|
+
tf.set(w, (tf.get(w) ?? 0) + 1);
|
|
43
|
+
return { name: t.name, tf, length: text.length };
|
|
44
|
+
});
|
|
45
|
+
const avg = docs.reduce((n, d) => n + d.length, 0) / Math.max(docs.length, 1);
|
|
46
|
+
const df = new Map();
|
|
47
|
+
for (const d of docs)
|
|
48
|
+
for (const w of d.tf.keys())
|
|
49
|
+
df.set(w, (df.get(w) ?? 0) + 1);
|
|
50
|
+
const idf = (w) => {
|
|
51
|
+
const n = df.get(w) ?? 0;
|
|
52
|
+
return Math.log(1 + (docs.length - n + 0.5) / (n + 0.5));
|
|
53
|
+
};
|
|
54
|
+
return {
|
|
55
|
+
rank(query) {
|
|
56
|
+
const q = [...new Set(terms(query))];
|
|
57
|
+
const out = [];
|
|
58
|
+
for (const d of docs) {
|
|
59
|
+
let score = 0;
|
|
60
|
+
for (const w of q) {
|
|
61
|
+
const f = d.tf.get(w);
|
|
62
|
+
if (!f)
|
|
63
|
+
continue;
|
|
64
|
+
score += idf(w) * ((f * (K1 + 1)) / (f + K1 * (1 - B + (B * d.length) / (avg || 1))));
|
|
65
|
+
}
|
|
66
|
+
if (score > 0)
|
|
67
|
+
out.push({ tool: d.name, score });
|
|
68
|
+
}
|
|
69
|
+
// Ties keep the menu's order, as a stable search would.
|
|
70
|
+
return out.sort((a, b) => b.score - a.score);
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Where a tool lands for a query: 1 is first. A tie counts in the tool's favour
|
|
76
|
+
* (only tools that score strictly higher are ahead of it), so a miss is never a
|
|
77
|
+
* coin flip. Undefined when the query matches nothing in the tool.
|
|
78
|
+
*/
|
|
79
|
+
export function placeOf(index, query, tool) {
|
|
80
|
+
const ranked = index.rank(query);
|
|
81
|
+
const mine = ranked.find((r) => r.tool === tool);
|
|
82
|
+
if (!mine)
|
|
83
|
+
return { ahead: ranked.slice(0, SEARCH_LIMIT).map((r) => r.tool) };
|
|
84
|
+
const ahead = ranked.filter((r) => r.score > mine.score + 1e-9).map((r) => r.tool);
|
|
85
|
+
return { place: ahead.length + 1, ahead };
|
|
86
|
+
}
|
package/dist/session.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ToolChange } from './compare.js';
|
|
2
2
|
import { type Target } from './connect.js';
|
|
3
|
+
import { type InputTrace } from './mrtr.js';
|
|
3
4
|
import { type FailureClass } from './failures.js';
|
|
4
5
|
import type { Era, Finding, Menu, MenuTool, Severity } from './types.js';
|
|
5
6
|
import { type AutoSummary } from './auto.js';
|
|
@@ -47,7 +48,9 @@ export interface StepRecord {
|
|
|
47
48
|
* What a call step's call did: the tool answered, answered with an error
|
|
48
49
|
* (isError), the request failed, or it wasn't sent (refused, not in the menu).
|
|
49
50
|
*/
|
|
50
|
-
outcome?: 'answered' | 'tool-error' | 'failed' | 'not-sent';
|
|
51
|
+
outcome?: 'answered' | 'tool-error' | 'failed' | 'not-sent' | 'needs-input';
|
|
52
|
+
/** 2026-07-28: what the tool asked for and what came of the decline (mrtr.ts). */
|
|
53
|
+
input?: InputTrace;
|
|
51
54
|
changed: boolean;
|
|
52
55
|
listChanged: number;
|
|
53
56
|
scope?: Scope;
|
|
@@ -138,6 +141,11 @@ interface Unlocker {
|
|
|
138
141
|
* character. Drops U+FFFD left over from a server that cut its own message by bytes.
|
|
139
142
|
*/
|
|
140
143
|
export declare function clip(text: string, max: number): string;
|
|
144
|
+
/**
|
|
145
|
+
* Absolute paths longer than 40 characters, kept to their last two parts:
|
|
146
|
+
* /tmp/a/b/node_modules/pkg/dist/bin/core-linux-x64 → …/bin/core-linux-x64.
|
|
147
|
+
*/
|
|
148
|
+
export declare function shortenPaths(text: string): string;
|
|
141
149
|
export declare function unlockers(tools: MenuTool[]): Unlocker[];
|
|
142
150
|
/**
|
|
143
151
|
* Unlocks whose values the schema doesn't give (no enum), each with a read-only
|