toolmenu 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/diff.js +39 -6
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/menu.d.ts +7 -0
- package/dist/menu.js +14 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -67,7 +67,7 @@ jobs:
|
|
|
67
67
|
- uses: actions/setup-node@v7
|
|
68
68
|
with: { node-version: 22 }
|
|
69
69
|
- run: npm ci && npm run build
|
|
70
|
-
- uses: niksa90/toolmenu@v0.7.
|
|
70
|
+
- uses: niksa90/toolmenu@v0.7.1
|
|
71
71
|
with:
|
|
72
72
|
command: node dist/server.js
|
|
73
73
|
baseline: menu.json # your committed snapshot
|
|
@@ -93,6 +93,7 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
93
93
|
| `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
|
|
94
94
|
|
|
95
95
|
Server behind auth, or PRs from forks and Dependabot? See [docs/github-action.md](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md).
|
|
96
|
+
Server that unlocks tools on request? Snapshot the full menu for the baseline and run the session on the default one: [servers that unlock tools](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#servers-that-unlock-tools).
|
|
96
97
|
PRs without secrets are skipped, and a skipped check still passes; the `skipped` output tells you.
|
|
97
98
|
|
|
98
99
|
## `session`: the menu changing while the agent works
|
|
@@ -142,7 +143,7 @@ npx toolmenu diff menu.json new-menu.json
|
|
|
142
143
|
```
|
|
143
144
|
$ npx toolmenu diff --release 2026.1.14..2026.8.31 fs-2026.1.14.json fs-2026.8.31.json
|
|
144
145
|
toolmenu diff secure-filesystem-server 2026.1.14 → 2026.8.31
|
|
145
|
-
14 → 14 tools · ~
|
|
146
|
+
14 → 14 tools · ~1,640 → ~1,664 tokens (+24, estimate): this release adds ~24 tokens to every conversation that loads the menu
|
|
146
147
|
1 breaking · 0 minor · 15 notice · suggested bump: major · actual: not checked (calendar version)
|
|
147
148
|
|
|
148
149
|
ERROR diff/safety-hint
|
|
@@ -290,7 +291,9 @@ or bad usage. `toolmenu --help` lists the `session` and `history` options.
|
|
|
290
291
|
- **Server order isn't final client order.** Clients merge tools from several servers
|
|
291
292
|
and may sort or filter them. A stable server menu is necessary for cache hits, not
|
|
292
293
|
sufficient. toolmenu measures what the server controls.
|
|
293
|
-
- **Token counts are estimates** (`o200k_base`)
|
|
294
|
+
- **Token counts are estimates** (`o200k_base`) of what the model reads: each tool's
|
|
295
|
+
name, description and input schema. Output schemas, annotations, icons and `_meta`
|
|
296
|
+
stay with the client and aren't counted. Vendors tokenize, cache and bill
|
|
294
297
|
differently.
|
|
295
298
|
- **Heuristic rules say so** in their messages.
|
|
296
299
|
- Not a security scanner or a full conformance suite. Other tools do those well.
|
package/dist/diff.js
CHANGED
|
@@ -147,12 +147,12 @@ function compareTool(old, t) {
|
|
|
147
147
|
const before = oldProps[p];
|
|
148
148
|
if (!before) {
|
|
149
149
|
out.push(newReq.has(p)
|
|
150
|
-
? { rule: 'diff/param-required', tool: name, message: `${name}.${p} is new and required. Existing calls don't send it.` }
|
|
150
|
+
? { rule: 'diff/param-required', tool: name, message: `${name}.${p} is new and required. Existing calls don't send it.${hasDefault(schema) ? ` It has a default (${show(schema.default)}), so the server may still accept calls without it, but clients that validate arguments won't.` : ''}` }
|
|
151
151
|
: { rule: 'diff/param-added', tool: name, message: `${name}.${p} is a new optional parameter.` });
|
|
152
152
|
continue;
|
|
153
153
|
}
|
|
154
154
|
if (!oldReq.has(p) && newReq.has(p)) {
|
|
155
|
-
out.push({ rule: 'diff/param-required', tool: name, message: `${name}.${p} was optional and is now required.` });
|
|
155
|
+
out.push({ rule: 'diff/param-required', tool: name, message: `${name}.${p} was optional and is now required.${hasDefault(schema) ? ` It has a default (${show(schema.default)}), so the server may still accept calls without it, but clients that validate arguments won't.` : ''}` });
|
|
156
156
|
}
|
|
157
157
|
else if (oldReq.has(p) && !newReq.has(p)) {
|
|
158
158
|
out.push({ rule: 'diff/param-relaxed', tool: name, message: `${name}.${p} was required and is now optional.` });
|
|
@@ -172,7 +172,7 @@ function compareTool(old, t) {
|
|
|
172
172
|
}
|
|
173
173
|
// Array parameters: the element type and allowed values count like the
|
|
174
174
|
// parameter's own (no items schema = any element).
|
|
175
|
-
const isArray = (s) => (
|
|
175
|
+
const isArray = (s) => typesOf(s)?.includes('array') ?? false;
|
|
176
176
|
if (isArray(before) && isArray(schema)) {
|
|
177
177
|
const oldItems = before.items ?? {};
|
|
178
178
|
const newItems = schema.items ?? {};
|
|
@@ -247,6 +247,9 @@ function readOnly(a) {
|
|
|
247
247
|
function destructive(a) {
|
|
248
248
|
return !readOnly(a) && a.destructiveHint !== false;
|
|
249
249
|
}
|
|
250
|
+
function hasDefault(schema) {
|
|
251
|
+
return 'default' in schema;
|
|
252
|
+
}
|
|
250
253
|
function show(value) {
|
|
251
254
|
return value === undefined ? '(unset)' : JSON.stringify(value);
|
|
252
255
|
}
|
|
@@ -277,7 +280,10 @@ function paramKey(t) {
|
|
|
277
280
|
}
|
|
278
281
|
/** Does `next` accept every value type that `prev` accepted? */
|
|
279
282
|
function accepts(next, prev) {
|
|
280
|
-
const types = (s) =>
|
|
283
|
+
const types = (s) => {
|
|
284
|
+
const t = typesOf(s);
|
|
285
|
+
return t && new Set(t);
|
|
286
|
+
};
|
|
281
287
|
const n = types(next);
|
|
282
288
|
const p = types(prev);
|
|
283
289
|
if (!n)
|
|
@@ -289,6 +295,9 @@ function accepts(next, prev) {
|
|
|
289
295
|
/** A parameter's schema without the parts diff classifies itself. */
|
|
290
296
|
function residual(schema) {
|
|
291
297
|
const { type: _t, enum: _e, description: _d, items, ...rest } = schema;
|
|
298
|
+
// A type written as anyOf/oneOf alternatives is compared as the type.
|
|
299
|
+
if (typeAlternatives(schema))
|
|
300
|
+
delete rest[schema.anyOf ? 'anyOf' : 'oneOf'];
|
|
292
301
|
if (items && typeof items === 'object') {
|
|
293
302
|
const { enum: _ie, type: _it, ...itemRest } = items;
|
|
294
303
|
return Object.keys(itemRest).length ? { ...rest, items: itemRest } : rest;
|
|
@@ -296,8 +305,32 @@ function residual(schema) {
|
|
|
296
305
|
return rest;
|
|
297
306
|
}
|
|
298
307
|
function typeOf(schema) {
|
|
299
|
-
|
|
300
|
-
|
|
308
|
+
return [...new Set(typesOf(schema) ?? [])].sort().join('|');
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* The types a schema accepts, undefined for any. `anyOf: [{type: string}, {type:
|
|
312
|
+
* null}]` is the same as `type: [string, null]`: schema generators switch between
|
|
313
|
+
* the two (zod did), and that isn't a change.
|
|
314
|
+
*/
|
|
315
|
+
function typesOf(schema) {
|
|
316
|
+
if (schema.type !== undefined)
|
|
317
|
+
return Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
318
|
+
return typeAlternatives(schema);
|
|
319
|
+
}
|
|
320
|
+
/** anyOf/oneOf whose branches only name a type (a description or title aside). */
|
|
321
|
+
function typeAlternatives(schema) {
|
|
322
|
+
const branches = (schema.anyOf ?? schema.oneOf);
|
|
323
|
+
if (!Array.isArray(branches) || branches.length === 0)
|
|
324
|
+
return undefined;
|
|
325
|
+
const types = [];
|
|
326
|
+
for (const b of branches) {
|
|
327
|
+
if (!b || typeof b !== 'object' || b.type === undefined)
|
|
328
|
+
return undefined;
|
|
329
|
+
if (Object.keys(b).some((k) => !['type', 'description', 'title'].includes(k)))
|
|
330
|
+
return undefined;
|
|
331
|
+
types.push(...(Array.isArray(b.type) ? b.type : [b.type]));
|
|
332
|
+
}
|
|
333
|
+
return types;
|
|
301
334
|
}
|
|
302
335
|
function enumChange(before, after) {
|
|
303
336
|
const a = before.enum;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { snapshot, type SnapshotOptions, type SnapshotResult } from './snapshot.js';
|
|
2
2
|
export { connect, listTools, type Target, type Connection } from './connect.js';
|
|
3
|
-
export { buildMenu, countTokens, loadMenu } from './menu.js';
|
|
3
|
+
export { buildMenu, countTokens, loadMenu, toolTokens } from './menu.js';
|
|
4
4
|
export { diffMenus, versionBump, DIFF_RULES, type DiffResult, type DiffOptions, type DiffFinding, type ChangeClass, type Bump, type TokenChange } from './diff.js';
|
|
5
5
|
export { history, historyCsv, npmSource, classifyFailure, WATCHED_DEPENDENCIES, type HistoryResult, type HistoryRow, type HistoryOptions, type PackageSource, type PublishedVersion, type FailureReason } from './history.js';
|
|
6
6
|
export { session, loadScenario, parseScenario, changeFindings, starterScenario, type Scenario, type Step, type SessionResult, type SessionOptions, type StepRecord, type Scope } from './session.js';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { snapshot } from './snapshot.js';
|
|
2
2
|
export { connect, listTools } from './connect.js';
|
|
3
|
-
export { buildMenu, countTokens, loadMenu } from './menu.js';
|
|
3
|
+
export { buildMenu, countTokens, loadMenu, toolTokens } from './menu.js';
|
|
4
4
|
export { diffMenus, versionBump, DIFF_RULES } from './diff.js';
|
|
5
5
|
export { history, historyCsv, npmSource, classifyFailure, WATCHED_DEPENDENCIES } from './history.js';
|
|
6
6
|
export { session, loadScenario, parseScenario, changeFindings, starterScenario } from './session.js';
|
package/dist/menu.d.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import type { Era, Menu, MenuTool } from './types.js';
|
|
2
2
|
/** Estimated tokens for a piece of text. An estimate: vendors tokenize differently. */
|
|
3
3
|
export declare function countTokens(text: string): number;
|
|
4
|
+
/**
|
|
5
|
+
* Estimated tokens the model reads for a tool: its name, description and input
|
|
6
|
+
* schema, which is what clients send (Claude's Messages API takes exactly those).
|
|
7
|
+
* outputSchema, annotations, icons, title and _meta stay with the client: GitHub's
|
|
8
|
+
* server embeds icons that would otherwise count five times its real menu.
|
|
9
|
+
*/
|
|
10
|
+
export declare function toolTokens(tool: Record<string, unknown>): number;
|
|
4
11
|
/** The tool definition without toolmenu's own `tokens` field. */
|
|
5
12
|
export declare function toolDefinition(tool: MenuTool): Record<string, unknown>;
|
|
6
13
|
export declare function buildMenu(rawTools: Record<string, unknown>[], server: Menu['server'], listMeta?: Menu['listMeta']): Menu;
|
package/dist/menu.js
CHANGED
|
@@ -6,6 +6,15 @@ export function countTokens(text) {
|
|
|
6
6
|
encoder ??= new Tiktoken(o200k);
|
|
7
7
|
return encoder.encode(text).length;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Estimated tokens the model reads for a tool: its name, description and input
|
|
11
|
+
* schema, which is what clients send (Claude's Messages API takes exactly those).
|
|
12
|
+
* outputSchema, annotations, icons, title and _meta stay with the client: GitHub's
|
|
13
|
+
* server embeds icons that would otherwise count five times its real menu.
|
|
14
|
+
*/
|
|
15
|
+
export function toolTokens(tool) {
|
|
16
|
+
return countTokens(JSON.stringify({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
|
|
17
|
+
}
|
|
9
18
|
/** The tool definition without toolmenu's own `tokens` field. */
|
|
10
19
|
export function toolDefinition(tool) {
|
|
11
20
|
const { tokens: _tokens, ...definition } = tool;
|
|
@@ -14,7 +23,7 @@ export function toolDefinition(tool) {
|
|
|
14
23
|
export function buildMenu(rawTools, server, listMeta) {
|
|
15
24
|
const tools = rawTools.map((raw) => ({
|
|
16
25
|
...raw,
|
|
17
|
-
tokens:
|
|
26
|
+
tokens: toolTokens(raw),
|
|
18
27
|
}));
|
|
19
28
|
return {
|
|
20
29
|
toolmenu: 1,
|
|
@@ -44,14 +53,14 @@ export async function loadMenu(path) {
|
|
|
44
53
|
if (menu?.toolmenu !== 1 || !Array.isArray(menu.tools)) {
|
|
45
54
|
throw new Error(`${path} isn't a toolmenu menu file (expected "toolmenu": 1 and a "tools" list). Write one with \`toolmenu snapshot\`.`);
|
|
46
55
|
}
|
|
56
|
+
// Always recounted, never read from the file: a baseline written by an older
|
|
57
|
+
// toolmenu (which counted every field) still compares fairly with a new snapshot.
|
|
47
58
|
for (const t of menu.tools) {
|
|
48
59
|
if (typeof t?.name !== 'string')
|
|
49
60
|
throw new Error(`${path}: every tool needs a name`);
|
|
50
|
-
|
|
51
|
-
t.tokens = countTokens(JSON.stringify(t));
|
|
61
|
+
t.tokens = toolTokens(t);
|
|
52
62
|
}
|
|
53
|
-
|
|
54
|
-
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
63
|
+
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
55
64
|
menu.server ??= {};
|
|
56
65
|
return menu;
|
|
57
66
|
}
|