toolmenu 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -7
- package/dist/breakdown.d.ts +30 -0
- package/dist/breakdown.js +63 -0
- package/dist/catalog.d.ts +70 -0
- package/dist/catalog.js +251 -0
- package/dist/cli.js +44 -5
- package/dist/config.d.ts +9 -0
- package/dist/diff.js +43 -0
- package/dist/init.d.ts +22 -0
- package/dist/init.js +105 -0
- package/dist/menu.js +2 -0
- package/dist/report.js +12 -0
- package/dist/snapshot.d.ts +3 -0
- package/dist/snapshot.js +4 -0
- package/dist/types.d.ts +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ No LLM anywhere: the same inputs give the same answer, so it can sit in CI.
|
|
|
29
29
|
(`history` installs old versions with today's dependencies, so it records what they
|
|
30
30
|
serve now, which can differ from what they shipped with: FINDINGS F4.)
|
|
31
31
|
|
|
32
|
-
> **Status: 0.
|
|
32
|
+
> **Status: 0.11, early.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md). What it has found on
|
|
33
33
|
> real servers: [docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
34
34
|
|
|
35
35
|
## Start here
|
|
@@ -38,13 +38,23 @@ Needs Node 22 or later. The GitHub Action brings its own if the runner's Node is
|
|
|
38
38
|
for toolmenu only: your server and the job's later steps keep the job's Node.
|
|
39
39
|
Your server can be in any language: toolmenu starts it (stdio) or connects to it (HTTP).
|
|
40
40
|
|
|
41
|
+
One command sets up CI: it snapshots your server into `menu.json` (the baseline) and
|
|
42
|
+
writes `.github/workflows/toolmenu.yml` for your project (Node, Python, Go or Rust).
|
|
43
|
+
Commit both, open a pull request, and toolmenu comments on it.
|
|
44
|
+
|
|
41
45
|
```sh
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
npx toolmenu init -- node dist/server.js # add --with-session to also run session --auto
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Credentials you pass with `--env` go into the workflow as `${{ secrets.… }}`, never as
|
|
50
|
+
values; `init` says which secrets to add. It never overwrites a file.
|
|
45
51
|
|
|
46
|
-
|
|
47
|
-
|
|
52
|
+
By hand:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
npx toolmenu snapshot -- node dist/server.js # the menu, its findings, menu.json
|
|
56
|
+
npx toolmenu diff menu.json new-menu.json # what changed between two releases
|
|
57
|
+
npx toolmenu session --auto -- node dist/server.js # call read-only tools, watch the menu
|
|
48
58
|
```
|
|
49
59
|
|
|
50
60
|
HTTP servers work the same way: `npx toolmenu snapshot https://example.com/mcp --header "Authorization: Bearer $TOKEN"`.
|
|
@@ -80,7 +90,7 @@ jobs:
|
|
|
80
90
|
- uses: actions/setup-node@v7
|
|
81
91
|
with: { node-version: 22 }
|
|
82
92
|
- run: npm ci && npm run build
|
|
83
|
-
- uses: niksa90/toolmenu@v0.
|
|
93
|
+
- uses: niksa90/toolmenu@v0.11.0
|
|
84
94
|
with:
|
|
85
95
|
command: node dist/server.js
|
|
86
96
|
baseline: menu.json # your committed snapshot
|
|
@@ -100,6 +110,7 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
100
110
|
| `env` | | Environment variables for a `command` server, one `KEY=value` per line |
|
|
101
111
|
| `baseline` | `menu.json` | The committed snapshot to diff against |
|
|
102
112
|
| `scenario` | | A scenario to run with `session`, or `auto` to build the steps from the menu |
|
|
113
|
+
| `catalog` | `false` | `true` also diffs the operations behind a search tool (`snapshot --catalog`) |
|
|
103
114
|
| `baseline-from` | `snapshot` | `session` diffs every tool the scenario saw (the union menu), for servers whose tools appear after an unlock |
|
|
104
115
|
| `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
|
|
105
116
|
| `fail-on` | `error` | Fail the job on findings at or above this level |
|
|
@@ -198,6 +209,29 @@ with `--server-version-is-release`. `diff` also enforces an optional `tokenBudge
|
|
|
198
209
|
(`diff/token-budget`). Order changes are only a notice here: between releases, prompt caches rebuild. Order
|
|
199
210
|
matters *within* a session.
|
|
200
211
|
|
|
212
|
+
### Operations behind a search tool (`--catalog`)
|
|
213
|
+
|
|
214
|
+
Big servers increasingly keep most operations out of the menu: a small fixed menu, a
|
|
215
|
+
search tool, and an execute tool (Sentry's `search_sentry_tools` + `execute_sentry_tool`,
|
|
216
|
+
Atlassian's `discover` + `executeRead`). A breaking change to one of those operations
|
|
217
|
+
never shows in the menu. `snapshot --catalog` asks the search tool a fixed set of
|
|
218
|
+
queries and keeps the operations it returns in `menu.json`; `diff` then compares
|
|
219
|
+
operations found in both snapshots with the same rules as tools.
|
|
220
|
+
|
|
221
|
+
```sh
|
|
222
|
+
npx toolmenu snapshot --catalog -- node dist/server.js
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
A search returns only its top matches, so toolmenu crawls the catalog by its own
|
|
226
|
+
words: it starts from the search tool's own example phrases and the nouns in your tool
|
|
227
|
+
names, then searches for every operation it finds or sees mentioned, until 20 queries
|
|
228
|
+
in a row turn up nothing new (at most 200). On Atlassian's hosted server that finds all
|
|
229
|
+
154 operations behind its 21 tools; on Sentry's, 65 behind 9, the same on every run.
|
|
230
|
+
Queries are paced and a rate limit is waited out. An operation not found this time is a
|
|
231
|
+
notice ("not returned by the same queries"), never "removed". To steer it, set
|
|
232
|
+
`"catalog": { "queries": [...], "maxQueries": 200, "crawl": true }` in
|
|
233
|
+
`toolmenu.config.json` (FINDINGS F15).
|
|
234
|
+
|
|
201
235
|
## `history`: a package's releases, researched
|
|
202
236
|
|
|
203
237
|
```sh
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Menu } from './types.js';
|
|
2
|
+
export interface Breakdown {
|
|
3
|
+
total: number;
|
|
4
|
+
/** The biggest tools, with their share of the menu and what their tokens are spent on. */
|
|
5
|
+
top: {
|
|
6
|
+
name: string;
|
|
7
|
+
tokens: number;
|
|
8
|
+
share: number;
|
|
9
|
+
description: number;
|
|
10
|
+
schema: number;
|
|
11
|
+
}[];
|
|
12
|
+
/** Enums with many values. */
|
|
13
|
+
enums: {
|
|
14
|
+
tool: string;
|
|
15
|
+
param: string;
|
|
16
|
+
values: number;
|
|
17
|
+
tokens: number;
|
|
18
|
+
}[];
|
|
19
|
+
/** The same parameter, byte for byte, in several tools: paid once per tool. */
|
|
20
|
+
repeated: {
|
|
21
|
+
param: string;
|
|
22
|
+
tokens: number;
|
|
23
|
+
tools: number;
|
|
24
|
+
total: number;
|
|
25
|
+
}[];
|
|
26
|
+
}
|
|
27
|
+
/** Where a menu's tokens go. Estimates, like every count in toolmenu. */
|
|
28
|
+
export declare function breakdown(menu: Menu): Breakdown;
|
|
29
|
+
/** The breakdown as report lines, or none for a menu too small to need one. */
|
|
30
|
+
export declare function breakdownLines(b: Breakdown, tools: number): string[];
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { canonical } from './compare.js';
|
|
2
|
+
import { countTokens } from './menu.js';
|
|
3
|
+
const TOP = 5;
|
|
4
|
+
const BIG_ENUM = 20;
|
|
5
|
+
const REPEATED_MIN_TOKENS = 30;
|
|
6
|
+
const REPEATED_MIN_TOOLS = 3;
|
|
7
|
+
/** Where a menu's tokens go. Estimates, like every count in toolmenu. */
|
|
8
|
+
export function breakdown(menu) {
|
|
9
|
+
const total = menu.totalTokens || 1;
|
|
10
|
+
const top = [...menu.tools]
|
|
11
|
+
.sort((a, b) => b.tokens - a.tokens)
|
|
12
|
+
.slice(0, TOP)
|
|
13
|
+
.map((t) => ({
|
|
14
|
+
name: t.name,
|
|
15
|
+
tokens: t.tokens,
|
|
16
|
+
share: t.tokens / total,
|
|
17
|
+
description: t.description ? countTokens(JSON.stringify(t.description)) : 0,
|
|
18
|
+
schema: t.inputSchema ? countTokens(JSON.stringify(t.inputSchema)) : 0,
|
|
19
|
+
}));
|
|
20
|
+
const enums = [];
|
|
21
|
+
const seen = new Map();
|
|
22
|
+
for (const tool of menu.tools) {
|
|
23
|
+
for (const [param, schema] of Object.entries(tool.inputSchema?.properties ?? {})) {
|
|
24
|
+
const values = enumValues(schema);
|
|
25
|
+
if (values.length >= BIG_ENUM)
|
|
26
|
+
enums.push({ tool: tool.name, param, values: values.length, tokens: countTokens(JSON.stringify(values)) });
|
|
27
|
+
const key = `${param}\u0000${canonical(schema)}`;
|
|
28
|
+
const entry = seen.get(key) ?? { param, tokens: countTokens(JSON.stringify({ [param]: schema })), tools: new Set() };
|
|
29
|
+
entry.tools.add(tool.name);
|
|
30
|
+
seen.set(key, entry);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const repeated = [...seen.values()]
|
|
34
|
+
.filter((e) => e.tokens >= REPEATED_MIN_TOKENS && e.tools.size >= REPEATED_MIN_TOOLS)
|
|
35
|
+
.map((e) => ({ param: e.param, tokens: e.tokens, tools: e.tools.size, total: e.tokens * e.tools.size }))
|
|
36
|
+
.sort((a, b) => b.total - a.total)
|
|
37
|
+
.slice(0, 5);
|
|
38
|
+
enums.sort((a, b) => b.tokens - a.tokens);
|
|
39
|
+
return { total: menu.totalTokens, top, enums: enums.slice(0, 5), repeated };
|
|
40
|
+
}
|
|
41
|
+
function enumValues(schema) {
|
|
42
|
+
if (Array.isArray(schema.enum))
|
|
43
|
+
return schema.enum;
|
|
44
|
+
if (Array.isArray(schema.items?.enum))
|
|
45
|
+
return schema.items.enum;
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
/** The breakdown as report lines, or none for a menu too small to need one. */
|
|
49
|
+
export function breakdownLines(b, tools) {
|
|
50
|
+
if (tools < 5)
|
|
51
|
+
return [];
|
|
52
|
+
const n = (x) => `~${x.toLocaleString('en-US')}`;
|
|
53
|
+
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
54
|
+
const topShare = b.top.reduce((s, t) => s + t.share, 0);
|
|
55
|
+
const lines = [`Where the tokens go (estimate): the top ${b.top.length} tools are ${pct(topShare)} of the menu`];
|
|
56
|
+
for (const t of b.top)
|
|
57
|
+
lines.push(` ${pct(t.share).padStart(4)} ${t.name} ${n(t.tokens)} (description ${n(t.description)}, schema ${n(t.schema)})`);
|
|
58
|
+
for (const e of b.enums)
|
|
59
|
+
lines.push(` enum: ${e.tool}.${e.param} has ${e.values} values, ${n(e.tokens)} tokens`);
|
|
60
|
+
for (const r of b.repeated)
|
|
61
|
+
lines.push(` repeated: ${r.param} (${n(r.tokens)}) in ${r.tools} tools, ${n(r.total)} in all`);
|
|
62
|
+
return lines;
|
|
63
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { Connection } from './connect.js';
|
|
2
|
+
import type { MenuTool } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Operations a server keeps behind a search tool instead of in its menu (search
|
|
5
|
+
* and execute: Sentry's search_sentry_tools, Atlassian's discover). `diff` can't
|
|
6
|
+
* see them in the menu, so snapshot --catalog asks the search tool a fixed set of
|
|
7
|
+
* queries and keeps what comes back.
|
|
8
|
+
*/
|
|
9
|
+
export interface Catalog {
|
|
10
|
+
/** The search tool asked. */
|
|
11
|
+
tool: string;
|
|
12
|
+
/** The queries asked, in order: the same ones every run, so runs compare. */
|
|
13
|
+
queries: string[];
|
|
14
|
+
/** Every operation found, by name. Not complete: a search returns its top matches. */
|
|
15
|
+
operations: MenuTool[];
|
|
16
|
+
/** Queries that failed (rate limits, errors): the catalog is partial where they would have looked. */
|
|
17
|
+
failed?: {
|
|
18
|
+
query: string;
|
|
19
|
+
error: string;
|
|
20
|
+
}[];
|
|
21
|
+
}
|
|
22
|
+
export interface CatalogOptions {
|
|
23
|
+
/** The search tool, when detection picks the wrong one. */
|
|
24
|
+
tool?: string;
|
|
25
|
+
/** The queries to ask. Default: derived from the menu. */
|
|
26
|
+
queries?: string[];
|
|
27
|
+
timeoutMs?: number;
|
|
28
|
+
/** Pause between queries, ms (default 300): hosted servers rate-limit searches (Sentry did). */
|
|
29
|
+
pauseMs?: number;
|
|
30
|
+
/** Follow the catalog's own words: search for each operation found or mentioned (default true). */
|
|
31
|
+
crawl?: boolean;
|
|
32
|
+
/** At most this many queries (default 200). */
|
|
33
|
+
maxQueries?: number;
|
|
34
|
+
/** Stop after this many queries in a row find nothing new (default 20): the catalog is exhausted. */
|
|
35
|
+
stopAfter?: number;
|
|
36
|
+
}
|
|
37
|
+
/** The menu's catalog search tool: read-only, one required query, named like one. */
|
|
38
|
+
export declare function findCatalogTool(tools: MenuTool[]): MenuTool | undefined;
|
|
39
|
+
/**
|
|
40
|
+
* Queries derived from the menu, the same for the same menu: the nouns in its tool
|
|
41
|
+
* names ("issue", "release", "page"), each asked as "list", "get", "create",
|
|
42
|
+
* "update" and "delete".
|
|
43
|
+
*/
|
|
44
|
+
export declare function defaultQueries(tools: MenuTool[], max?: number): string[];
|
|
45
|
+
/** Ask the search tool every query and collect the operations it returns. */
|
|
46
|
+
export declare function readCatalog(conn: Connection, tools: MenuTool[], options?: CatalogOptions): Promise<Catalog>;
|
|
47
|
+
/**
|
|
48
|
+
* Operation definitions in a search result: objects with a `name` and either a
|
|
49
|
+
* JSON Schema (`inputSchema`, `input_schema`, `parameters`) or a list of
|
|
50
|
+
* `inputs` (Atlassian), found anywhere in structuredContent or in text that
|
|
51
|
+
* parses as JSON.
|
|
52
|
+
*/
|
|
53
|
+
export declare function operationsIn(result: {
|
|
54
|
+
structuredContent?: unknown;
|
|
55
|
+
content?: unknown;
|
|
56
|
+
}): MenuTool[];
|
|
57
|
+
/**
|
|
58
|
+
* The JSON value a text starts with, ignoring what follows: Atlassian's discover
|
|
59
|
+
* returns its results as JSON and then a prose list of related operations, which
|
|
60
|
+
* makes the whole text invalid JSON.
|
|
61
|
+
*/
|
|
62
|
+
export declare function leadingJson(text: string): unknown;
|
|
63
|
+
/**
|
|
64
|
+
* Operation names a result mentions without defining them: lines like
|
|
65
|
+
* "createJiraBoard — Create a new company-managed…" after Atlassian's JSON. Each
|
|
66
|
+
* is worth a search of its own.
|
|
67
|
+
*/
|
|
68
|
+
export declare function mentionedNames(result: {
|
|
69
|
+
content?: unknown;
|
|
70
|
+
}): string[];
|
package/dist/catalog.js
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { toolTokens } from './menu.js';
|
|
2
|
+
import { nouns, singular } from './words.js';
|
|
3
|
+
const QUERY_PARAM = /^(q|query|search|keywords?|text)$/i;
|
|
4
|
+
const SEARCH_NAME = /(search|find|list|discover)[\w-]*(tools?|operations?|actions?|capabilit)|^discover$|(tools?|operations?)[\w-]*(search|discover)/i;
|
|
5
|
+
/** The menu's catalog search tool: read-only, one required query, named like one. */
|
|
6
|
+
export function findCatalogTool(tools) {
|
|
7
|
+
return tools.find((t) => {
|
|
8
|
+
if (t.annotations?.readOnlyHint !== true)
|
|
9
|
+
return false;
|
|
10
|
+
const required = t.inputSchema?.required ?? [];
|
|
11
|
+
return SEARCH_NAME.test(t.name) && required.length === 1 && QUERY_PARAM.test(required[0]);
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Queries derived from the menu, the same for the same menu: the nouns in its tool
|
|
16
|
+
* names ("issue", "release", "page"), each asked as "list", "get", "create",
|
|
17
|
+
* "update" and "delete".
|
|
18
|
+
*/
|
|
19
|
+
export function defaultQueries(tools, max = 25) {
|
|
20
|
+
const counts = new Map();
|
|
21
|
+
for (const t of tools)
|
|
22
|
+
for (const n of new Set(nouns(t.name).map(singular)))
|
|
23
|
+
counts.set(n, (counts.get(n) ?? 0) + 1);
|
|
24
|
+
const subjects = [...counts].filter(([n]) => n.length > 2).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([n]) => n);
|
|
25
|
+
const queries = [];
|
|
26
|
+
for (const noun of subjects)
|
|
27
|
+
for (const verb of ['list', 'get', 'create', 'update', 'delete'])
|
|
28
|
+
queries.push(`${verb} ${noun}`);
|
|
29
|
+
return queries.slice(0, max);
|
|
30
|
+
}
|
|
31
|
+
/** Ask the search tool every query and collect the operations it returns. */
|
|
32
|
+
export async function readCatalog(conn, tools, options = {}) {
|
|
33
|
+
const tool = options.tool ? tools.find((t) => t.name === options.tool) : findCatalogTool(tools);
|
|
34
|
+
if (!tool)
|
|
35
|
+
throw new Error(options.tool ? `--catalog: no tool named ${options.tool} in the menu.` : '--catalog: no catalog search tool in the menu (read-only, one required query, named like search_*_tools or discover). Name it with catalog.tool in the config.');
|
|
36
|
+
const queryParam = tool.inputSchema.required[0];
|
|
37
|
+
const limit = limitArg(tool.inputSchema);
|
|
38
|
+
// Seeds, then the catalog's own words: every operation found or mentioned is
|
|
39
|
+
// searched for by name, which finds it and its neighbours. Breadth first, in the
|
|
40
|
+
// order found, so the same catalog gives the same queries.
|
|
41
|
+
const queue = [...new Set(options.queries ?? [...describedExamples(tool), ...defaultQueries(tools)])];
|
|
42
|
+
const crawl = options.crawl ?? true;
|
|
43
|
+
// Crawl until the catalog stops yielding: a small one ends early, a big one
|
|
44
|
+
// (Atlassian's ~300 operations) goes on, up to the cap.
|
|
45
|
+
const maxQueries = options.maxQueries ?? 200;
|
|
46
|
+
const stopAfter = options.stopAfter ?? 20;
|
|
47
|
+
let dry = 0;
|
|
48
|
+
const asked = [];
|
|
49
|
+
const queued = new Set(queue);
|
|
50
|
+
const enqueue = (name) => {
|
|
51
|
+
const q = phrase(name);
|
|
52
|
+
if (crawl && q && !queued.has(q)) {
|
|
53
|
+
queued.add(q);
|
|
54
|
+
queue.push(q);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
const found = new Map();
|
|
58
|
+
const failed = [];
|
|
59
|
+
const pause = options.pauseMs ?? 300;
|
|
60
|
+
while (queue.length && asked.length < maxQueries && dry < stopAfter) {
|
|
61
|
+
const query = queue.shift();
|
|
62
|
+
if (asked.length > 0)
|
|
63
|
+
await sleep(pause);
|
|
64
|
+
asked.push(query);
|
|
65
|
+
// A rate limit is waited out (2 s, 4 s, 8 s); anything else, or a limit that
|
|
66
|
+
// persists, leaves this query out and the catalog partial, never the snapshot
|
|
67
|
+
// failed: diff treats operations not found as notices.
|
|
68
|
+
for (let attempt = 0;; attempt++) {
|
|
69
|
+
try {
|
|
70
|
+
const result = await conn.client.callTool({ name: tool.name, arguments: { [queryParam]: query, ...limit } }, { timeout: options.timeoutMs ?? 30_000 });
|
|
71
|
+
const text = result.isError ? resultText(result) : '';
|
|
72
|
+
if (result.isError && RATE_LIMIT.test(text) && attempt < 3) {
|
|
73
|
+
await sleep(2000 * 2 ** attempt);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (result.isError)
|
|
77
|
+
failed.push({ query, error: text.slice(0, 200) || 'the tool returned an error' });
|
|
78
|
+
else {
|
|
79
|
+
const before = found.size;
|
|
80
|
+
for (const op of operationsIn(result)) {
|
|
81
|
+
if (found.has(op.name))
|
|
82
|
+
continue;
|
|
83
|
+
found.set(op.name, op);
|
|
84
|
+
enqueue(op.name);
|
|
85
|
+
}
|
|
86
|
+
for (const name of mentionedNames(result))
|
|
87
|
+
if (!found.has(name))
|
|
88
|
+
enqueue(name);
|
|
89
|
+
dry = found.size > before ? 0 : dry + 1;
|
|
90
|
+
}
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
const message = error instanceof Error ? error.message.split('\n')[0] : String(error);
|
|
95
|
+
if (RATE_LIMIT.test(message) && attempt < 3) {
|
|
96
|
+
await sleep(2000 * 2 ** attempt);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
failed.push({ query, error: message.slice(0, 200) });
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
const operations = [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
105
|
+
return { tool: tool.name, queries: asked, operations, ...(failed.length ? { failed } : {}) };
|
|
106
|
+
}
|
|
107
|
+
/** An operation name as a search phrase: listJiraIssueWorklogs → "list jira issue worklogs". */
|
|
108
|
+
function phrase(name) {
|
|
109
|
+
return name
|
|
110
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
111
|
+
.split(/[^A-Za-z0-9]+/)
|
|
112
|
+
.filter(Boolean)
|
|
113
|
+
.map((w) => w.toLowerCase())
|
|
114
|
+
.join(' ');
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Example queries the search tool's own schema suggests: Atlassian's discover
|
|
118
|
+
* documents its query with "transition a jira issue", "list confluence spaces"…
|
|
119
|
+
*/
|
|
120
|
+
function describedExamples(tool) {
|
|
121
|
+
const text = [tool.description ?? '', ...Object.values(tool.inputSchema?.properties ?? {}).map((p) => String(p.description ?? ''))].join(' ');
|
|
122
|
+
return [...text.matchAll(/["“]([a-z][a-z0-9 ,'-]{5,60})["”]/g)].map((m) => m[1]).filter((q) => q.includes(' '));
|
|
123
|
+
}
|
|
124
|
+
const RATE_LIMIT = /rate.?limit|too many requests|\b429\b/i;
|
|
125
|
+
function sleep(ms) {
|
|
126
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
127
|
+
}
|
|
128
|
+
function resultText(result) {
|
|
129
|
+
return (Array.isArray(result.content) ? result.content : []).map((c) => c.text ?? '').join(' ').trim();
|
|
130
|
+
}
|
|
131
|
+
/** Ask for as many results as the search allows (Sentry: limit up to 20). */
|
|
132
|
+
function limitArg(schema) {
|
|
133
|
+
for (const [name, s] of Object.entries(schema?.properties ?? {})) {
|
|
134
|
+
if (!/^(limit|max_?results|top_?k|count)$/i.test(name))
|
|
135
|
+
continue;
|
|
136
|
+
const branches = [s, ...(s.anyOf ?? s.oneOf ?? [])];
|
|
137
|
+
const max = branches.map((b) => b.maximum).find((m) => typeof m === 'number');
|
|
138
|
+
if (max)
|
|
139
|
+
return { [name]: max };
|
|
140
|
+
}
|
|
141
|
+
return {};
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Operation definitions in a search result: objects with a `name` and either a
|
|
145
|
+
* JSON Schema (`inputSchema`, `input_schema`, `parameters`) or a list of
|
|
146
|
+
* `inputs` (Atlassian), found anywhere in structuredContent or in text that
|
|
147
|
+
* parses as JSON.
|
|
148
|
+
*/
|
|
149
|
+
export function operationsIn(result) {
|
|
150
|
+
const roots = [];
|
|
151
|
+
if (result.structuredContent)
|
|
152
|
+
roots.push(result.structuredContent);
|
|
153
|
+
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
154
|
+
const text = part.text;
|
|
155
|
+
if (typeof text !== 'string')
|
|
156
|
+
continue;
|
|
157
|
+
const value = leadingJson(text);
|
|
158
|
+
if (value !== undefined)
|
|
159
|
+
roots.push(value);
|
|
160
|
+
}
|
|
161
|
+
const out = new Map();
|
|
162
|
+
const walk = (v, depth) => {
|
|
163
|
+
if (depth > 6 || !v || typeof v !== 'object')
|
|
164
|
+
return;
|
|
165
|
+
if (Array.isArray(v))
|
|
166
|
+
return v.forEach((x) => walk(x, depth + 1));
|
|
167
|
+
const op = asOperation(v);
|
|
168
|
+
if (op) {
|
|
169
|
+
out.set(op.name, op);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
for (const x of Object.values(v))
|
|
173
|
+
walk(x, depth + 1);
|
|
174
|
+
};
|
|
175
|
+
roots.forEach((r) => walk(r, 0));
|
|
176
|
+
return [...out.values()];
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The JSON value a text starts with, ignoring what follows: Atlassian's discover
|
|
180
|
+
* returns its results as JSON and then a prose list of related operations, which
|
|
181
|
+
* makes the whole text invalid JSON.
|
|
182
|
+
*/
|
|
183
|
+
export function leadingJson(text) {
|
|
184
|
+
const trimmed = text.trimStart();
|
|
185
|
+
if (!/^[[{]/.test(trimmed))
|
|
186
|
+
return undefined;
|
|
187
|
+
try {
|
|
188
|
+
return JSON.parse(trimmed);
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
const at = /position (\d+)/.exec(error instanceof Error ? error.message : '');
|
|
192
|
+
if (!at)
|
|
193
|
+
return undefined;
|
|
194
|
+
try {
|
|
195
|
+
return JSON.parse(trimmed.slice(0, Number(at[1])));
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return undefined;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Operation names a result mentions without defining them: lines like
|
|
204
|
+
* "createJiraBoard — Create a new company-managed…" after Atlassian's JSON. Each
|
|
205
|
+
* is worth a search of its own.
|
|
206
|
+
*/
|
|
207
|
+
export function mentionedNames(result) {
|
|
208
|
+
const names = new Set();
|
|
209
|
+
for (const part of Array.isArray(result.content) ? result.content : []) {
|
|
210
|
+
const text = part.text;
|
|
211
|
+
if (typeof text !== 'string')
|
|
212
|
+
continue;
|
|
213
|
+
for (const m of text.matchAll(/^[ \t]*[-*]?[ \t]*`?([A-Za-z][A-Za-z0-9_.-]{2,80})`?[ \t]+(?:—|–|-{1,2}|:)[ \t]/gm))
|
|
214
|
+
names.add(m[1]);
|
|
215
|
+
}
|
|
216
|
+
return [...names];
|
|
217
|
+
}
|
|
218
|
+
function asOperation(o) {
|
|
219
|
+
if (typeof o.name !== 'string' || !o.name)
|
|
220
|
+
return undefined;
|
|
221
|
+
const schema = (o.inputSchema ?? o.input_schema ?? o.parameters);
|
|
222
|
+
let inputSchema;
|
|
223
|
+
if (schema && typeof schema === 'object' && !Array.isArray(schema))
|
|
224
|
+
inputSchema = schema;
|
|
225
|
+
else if (Array.isArray(o.inputs))
|
|
226
|
+
inputSchema = fromInputs(o.inputs);
|
|
227
|
+
else
|
|
228
|
+
return undefined;
|
|
229
|
+
const def = { name: o.name, ...(typeof o.description === 'string' ? { description: o.description } : {}), inputSchema };
|
|
230
|
+
if (o.annotations && typeof o.annotations === 'object')
|
|
231
|
+
def.annotations = o.annotations;
|
|
232
|
+
return { ...def, tokens: toolTokens(def) };
|
|
233
|
+
}
|
|
234
|
+
/** Atlassian's `inputs: [{ name, type, required, description, minimum, … }]` as a JSON Schema. */
|
|
235
|
+
function fromInputs(inputs) {
|
|
236
|
+
const properties = {};
|
|
237
|
+
const required = [];
|
|
238
|
+
for (const i of inputs) {
|
|
239
|
+
if (typeof i.name !== 'string')
|
|
240
|
+
continue;
|
|
241
|
+
// repairHint stays: it's text the agent reads, so a change to it is a change.
|
|
242
|
+
const { name, required: req, integer, ...rest } = i;
|
|
243
|
+
const p = { ...rest };
|
|
244
|
+
if (integer === true && p.type === 'number')
|
|
245
|
+
p.type = 'integer';
|
|
246
|
+
properties[name] = p;
|
|
247
|
+
if (req === true)
|
|
248
|
+
required.push(name);
|
|
249
|
+
}
|
|
250
|
+
return { type: 'object', properties, ...(required.length ? { required } : {}) };
|
|
251
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { writeFile } from 'node:fs/promises';
|
|
3
|
-
import { join } from 'node:path';
|
|
2
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
4
|
import { parseArgs } from 'node:util';
|
|
5
5
|
import { loadConfig } from './config.js';
|
|
6
6
|
import { diffMenus } from './diff.js';
|
|
7
7
|
import { loadMenu } from './menu.js';
|
|
8
8
|
import { history, historyCsv } from './history.js';
|
|
9
|
-
import { formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
|
|
9
|
+
import { counts, formatDiff, formatHistory, formatPlan, formatSession, formatSnapshot } from './report.js';
|
|
10
10
|
import { loadScenario, session, starterScenario } from './session.js';
|
|
11
11
|
import { connect, listTools } from './connect.js';
|
|
12
12
|
import { buildMenu } from './menu.js';
|
|
@@ -16,12 +16,14 @@ import { snapshot } from './snapshot.js';
|
|
|
16
16
|
import { MAIN_SEED, probeMenu, seeded } from './probe.js';
|
|
17
17
|
import { autoScenario, scenarioYaml } from './auto.js';
|
|
18
18
|
import { authDir, listLogins, login, logout } from './auth.js';
|
|
19
|
+
import { detectProject, secretsNeeded, workflowYaml } from './init.js';
|
|
19
20
|
import { SEVERITY_RANK } from './types.js';
|
|
20
21
|
import { VERSION } from './version.js';
|
|
21
22
|
const HELP = `toolmenu ${VERSION}
|
|
22
23
|
Lint your MCP server's tool menu for changes that confuse agents or break caches.
|
|
23
24
|
|
|
24
25
|
Usage:
|
|
26
|
+
toolmenu init [options] -- <command> | <url> set up CI: a baseline and a workflow
|
|
25
27
|
toolmenu snapshot [options] -- <command> [args...] stdio server
|
|
26
28
|
toolmenu snapshot [options] <url> Streamable HTTP server
|
|
27
29
|
toolmenu diff [options] <old.json> <new.json> compare two snapshots
|
|
@@ -33,6 +35,7 @@ Usage:
|
|
|
33
35
|
the same, with steps built from the menu
|
|
34
36
|
|
|
35
37
|
Commands:
|
|
38
|
+
init snapshot the server into menu.json and write .github/workflows/toolmenu.yml
|
|
36
39
|
snapshot establish the menu: list tools twice, write menu.json, run the menu rules
|
|
37
40
|
diff compare releases: breaking changes, token change, semver bump
|
|
38
41
|
session observe the menu changing while a scripted session runs
|
|
@@ -52,6 +55,8 @@ Options:
|
|
|
52
55
|
only gets a minimal environment (PATH, HOME, ...) plus these
|
|
53
56
|
--no-auth don't use a stored OAuth login for this server
|
|
54
57
|
--timeout <ms> per-request timeout (default: 30000)
|
|
58
|
+
--catalog also read the operations behind a search tool (search and execute:
|
|
59
|
+
discover, search_*_tools) and keep them in the menu file for diff
|
|
55
60
|
--processes <n> server processes (stdio) or connections (HTTP) to compare,
|
|
56
61
|
the main one included (default: 2; 1 turns the check off)
|
|
57
62
|
-h, --help show this help
|
|
@@ -90,6 +95,9 @@ history options (best effort; installs and runs third-party code, so use a conta
|
|
|
90
95
|
--keep-installs keep each version's install directory
|
|
91
96
|
--out <dir> where menus and history.json go (default: toolmenu-history/<package>)
|
|
92
97
|
|
|
98
|
+
init options:
|
|
99
|
+
--with-session the workflow also runs session --auto (calls read-only tools)
|
|
100
|
+
|
|
93
101
|
auth options:
|
|
94
102
|
--port <n> loopback port for the login redirect (default: 33418)
|
|
95
103
|
--scope <scopes> scopes to ask for (default: the server's)
|
|
@@ -140,6 +148,8 @@ export async function main(argv) {
|
|
|
140
148
|
init: { type: 'boolean' },
|
|
141
149
|
'union-out': { type: 'string' },
|
|
142
150
|
auto: { type: 'boolean' },
|
|
151
|
+
'with-session': { type: 'boolean' },
|
|
152
|
+
catalog: { type: 'boolean' },
|
|
143
153
|
'open-world': { type: 'boolean' },
|
|
144
154
|
'max-calls': { type: 'string' },
|
|
145
155
|
'save-scenario': { type: 'string' },
|
|
@@ -156,7 +166,7 @@ export async function main(argv) {
|
|
|
156
166
|
process.stdout.write(HELP);
|
|
157
167
|
return sub || values.help ? 0 : 2;
|
|
158
168
|
}
|
|
159
|
-
if (!['snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
169
|
+
if (!['init', 'snapshot', 'diff', 'history', 'session', 'auth'].includes(sub))
|
|
160
170
|
throw new UsageError(`Unknown command "${sub}". Try --help.`);
|
|
161
171
|
const format = (values.json ? 'json' : values.format ?? 'text');
|
|
162
172
|
if (!['text', 'json', 'github', 'markdown'].includes(format))
|
|
@@ -240,6 +250,35 @@ export async function main(argv) {
|
|
|
240
250
|
}
|
|
241
251
|
return 0;
|
|
242
252
|
}
|
|
253
|
+
if (sub === 'init') {
|
|
254
|
+
const workflow = join('.github', 'workflows', 'toolmenu.yml');
|
|
255
|
+
const baseline = values.out ?? 'menu.json';
|
|
256
|
+
for (const f of [workflow, baseline])
|
|
257
|
+
if (existsSync(f))
|
|
258
|
+
throw new UsageError(`${f} already exists; init never overwrites. Remove it, or set things up by hand (README: "In CI").`);
|
|
259
|
+
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
260
|
+
const { menu, findings } = await snapshot(target, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
261
|
+
await writeFile(baseline, JSON.stringify(menu, null, 2) + '\n');
|
|
262
|
+
await mkdir(dirname(workflow), { recursive: true });
|
|
263
|
+
const project = detectProject('.');
|
|
264
|
+
await writeFile(workflow, workflowYaml({ target, project, session: values['with-session'], baseline }));
|
|
265
|
+
const c = counts(findings);
|
|
266
|
+
const secrets = secretsNeeded(target);
|
|
267
|
+
const lines = [
|
|
268
|
+
`toolmenu init ${menu.server.name ?? 'server'} · ${menu.tools.length} tools · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
269
|
+
` wrote ${baseline}: the baseline every pull request is compared with`,
|
|
270
|
+
` wrote ${workflow}${project.kind === 'unknown' ? ' (fill in the TODO: how CI builds your server)' : ` (${project.kind} project)`}`,
|
|
271
|
+
` today's menu: ${c.error} errors, ${c.warn} warnings, ${c.info} info (run toolmenu snapshot to see them)`,
|
|
272
|
+
...(secrets.length ? ['', `Add these repository secrets (Settings → Secrets and variables → Actions): ${secrets.join(', ')}`] : []),
|
|
273
|
+
'',
|
|
274
|
+
'Next:',
|
|
275
|
+
` git add ${baseline} ${workflow} && git commit -m "Check the MCP tool menu on every PR"`,
|
|
276
|
+
' Then open a pull request: toolmenu comments on it. Refresh the baseline when a change is intended:',
|
|
277
|
+
` toolmenu snapshot --out ${baseline} ${target.kind === 'stdio' ? '-- ' + [target.command, ...target.args].join(' ') : target.url}`,
|
|
278
|
+
];
|
|
279
|
+
process.stdout.write(lines.join('\n') + '\n');
|
|
280
|
+
return 0;
|
|
281
|
+
}
|
|
243
282
|
if (sub === 'session' && values.auto) {
|
|
244
283
|
if (values.scenario)
|
|
245
284
|
throw new UsageError('--auto builds the steps itself; drop --scenario (or drop --auto).');
|
|
@@ -293,7 +332,7 @@ export async function main(argv) {
|
|
|
293
332
|
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth']);
|
|
294
333
|
const routesPath = values.routes ?? config.routes;
|
|
295
334
|
const routes = routesPath ? await loadRoutes(routesPath) : undefined;
|
|
296
|
-
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
335
|
+
const { menu, findings } = await snapshot(target, { routes, timeoutMs, processes, catalog: values.catalog ? (config.catalog ?? true) : undefined, rules: config.rules, ignore: config.ignore, descriptionLimit: config.descriptionLimit, fullDescriptions: config.fullDescriptions });
|
|
297
336
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
298
337
|
if (outPath)
|
|
299
338
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
package/dist/config.d.ts
CHANGED
|
@@ -16,6 +16,15 @@ export interface Config {
|
|
|
16
16
|
fullDescriptions?: string[];
|
|
17
17
|
/** Server processes (stdio) or connections (HTTP) to compare, the main one included (default 2). */
|
|
18
18
|
processes?: number;
|
|
19
|
+
/** snapshot --catalog: which search tool and which queries (default: detected, derived from the menu). */
|
|
20
|
+
catalog?: {
|
|
21
|
+
tool?: string;
|
|
22
|
+
queries?: string[];
|
|
23
|
+
pauseMs?: number;
|
|
24
|
+
crawl?: boolean;
|
|
25
|
+
maxQueries?: number;
|
|
26
|
+
stopAfter?: number;
|
|
27
|
+
};
|
|
19
28
|
}
|
|
20
29
|
/** Optional: no config file is fine. */
|
|
21
30
|
export declare function loadConfig(path?: string): Promise<Config>;
|
package/dist/diff.js
CHANGED
|
@@ -24,6 +24,12 @@ export const DIFF_RULES = {
|
|
|
24
24
|
'diff/version-bump': { severity: 'warn' },
|
|
25
25
|
'diff/version-backwards': { severity: 'warn' },
|
|
26
26
|
'diff/token-budget': { severity: 'error' },
|
|
27
|
+
// Operations behind a search tool (snapshot --catalog). A search returns its top
|
|
28
|
+
// matches, so one not found this time may be ranked out, not removed: a notice,
|
|
29
|
+
// no bump. Changes to operations found in both runs use the rules above.
|
|
30
|
+
'diff/catalog-missing': { severity: 'info' },
|
|
31
|
+
'diff/catalog-added': { severity: 'info' },
|
|
32
|
+
'diff/catalog-queries': { severity: 'info' },
|
|
27
33
|
};
|
|
28
34
|
export function diffMenus(before, after, options = {}) {
|
|
29
35
|
const raw = [];
|
|
@@ -62,6 +68,7 @@ export function diffMenus(before, after, options = {}) {
|
|
|
62
68
|
message: `Tool order changed (${moved.map((m) => m.tool).join(', ')}). Only a notice between releases: prompt caches are short-lived, so a deploy costs roughly one rewrite. It matters within a session.`,
|
|
63
69
|
});
|
|
64
70
|
}
|
|
71
|
+
raw.push(...compareCatalogs(before.catalog, after.catalog, ignored));
|
|
65
72
|
// Ignored tools leave the token counts and the budget too.
|
|
66
73
|
const tokens = tokenChange(oldTools, newTools);
|
|
67
74
|
if (options.tokenBudget !== undefined && tokens.after > options.tokenBudget) {
|
|
@@ -207,6 +214,42 @@ function compareTool(old, t) {
|
|
|
207
214
|
}
|
|
208
215
|
return out.concat(compareRest(old, t));
|
|
209
216
|
}
|
|
217
|
+
/** Operations behind a search tool, compared like tools where both runs found them. */
|
|
218
|
+
function compareCatalogs(before, after, ignored) {
|
|
219
|
+
if (!before && !after)
|
|
220
|
+
return [];
|
|
221
|
+
if (!before || !after) {
|
|
222
|
+
return [{ rule: 'diff/catalog-queries', message: `Only the ${before ? 'older' : 'newer'} snapshot has a catalog (snapshot --catalog), so the operations behind ${(before ?? after).tool} weren't compared.` }];
|
|
223
|
+
}
|
|
224
|
+
const out = [];
|
|
225
|
+
const where = `behind ${after.tool}`;
|
|
226
|
+
if (JSON.stringify(before.queries) !== JSON.stringify(after.queries)) {
|
|
227
|
+
out.push({ rule: 'diff/catalog-queries', message: `The two catalogs were read with different queries, so operations found on one side only say little. Pin them with catalog.queries in the config.` });
|
|
228
|
+
}
|
|
229
|
+
const partial = [before, after].filter((c) => c.failed?.length);
|
|
230
|
+
if (partial.length) {
|
|
231
|
+
out.push({ rule: 'diff/catalog-queries', message: `${partial.length === 2 ? 'Both catalogs are' : `The ${partial[0] === before ? 'older' : 'newer'} catalog is`} partial: some queries failed (rate limits or errors), so operations found on one side only say even less.` });
|
|
232
|
+
}
|
|
233
|
+
const oldOps = new Map(before.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
|
|
234
|
+
const newOps = new Map(after.operations.filter((o) => !ignored(o.name)).map((o) => [o.name, o]));
|
|
235
|
+
for (const [name, op] of newOps) {
|
|
236
|
+
const old = oldOps.get(name);
|
|
237
|
+
if (!old) {
|
|
238
|
+
out.push({ rule: 'diff/catalog-added', tool: name, message: `${name} (${where}) is new, or newly found by the same queries.` });
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
for (const r of compareTool(old, op))
|
|
242
|
+
out.push({ ...r, message: `${r.message.replace(/\.$/, '')} (an operation ${where}).` });
|
|
243
|
+
}
|
|
244
|
+
const missing = [...oldOps.keys()].filter((n) => !newOps.has(n));
|
|
245
|
+
if (missing.length) {
|
|
246
|
+
out.push({
|
|
247
|
+
rule: 'diff/catalog-missing',
|
|
248
|
+
message: `${missing.length} operation${missing.length === 1 ? '' : 's'} ${where} weren't returned by the same queries this time: removed, renamed, or ranked lower. Check before relying on them: ${missing.slice(0, 8).join(', ')}${missing.length > 8 ? ', …' : ''}.`,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return out;
|
|
252
|
+
}
|
|
210
253
|
const TOOL_FIELDS = new Set(['name', 'description', 'inputSchema', 'outputSchema', 'annotations', 'tokens']);
|
|
211
254
|
/** Output schema, annotations and any other fields (title, icons, _meta…). */
|
|
212
255
|
function compareRest(old, t) {
|
package/dist/init.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Target } from './connect.js';
|
|
2
|
+
export type ProjectKind = 'node' | 'python' | 'go' | 'rust' | 'unknown';
|
|
3
|
+
export interface Project {
|
|
4
|
+
kind: ProjectKind;
|
|
5
|
+
/** The steps that install and build the server in CI, as YAML list items. */
|
|
6
|
+
setup: string[];
|
|
7
|
+
}
|
|
8
|
+
/** What the repository in `dir` looks like, and how CI gets it ready to run. */
|
|
9
|
+
export declare function detectProject(dir: string): Project;
|
|
10
|
+
/** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
|
|
11
|
+
export declare function looksSecret(key: string): boolean;
|
|
12
|
+
export interface WorkflowOptions {
|
|
13
|
+
target: Target;
|
|
14
|
+
project: Project;
|
|
15
|
+
/** Also run `session --auto` (calls read-only tools with the job's credentials). */
|
|
16
|
+
session?: boolean;
|
|
17
|
+
baseline?: string;
|
|
18
|
+
}
|
|
19
|
+
/** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
|
|
20
|
+
export declare function workflowYaml(opts: WorkflowOptions): string;
|
|
21
|
+
/** The secret names the workflow refers to, to tell the user what to add. */
|
|
22
|
+
export declare function secretsNeeded(target: Target): string[];
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { VERSION } from './version.js';
|
|
4
|
+
/** What the repository in `dir` looks like, and how CI gets it ready to run. */
|
|
5
|
+
export function detectProject(dir) {
|
|
6
|
+
const has = (f) => existsSync(join(dir, f));
|
|
7
|
+
if (has('package.json')) {
|
|
8
|
+
let scripts = {};
|
|
9
|
+
try {
|
|
10
|
+
scripts = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')).scripts ?? {};
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
// an unreadable package.json still means node
|
|
14
|
+
}
|
|
15
|
+
const pm = has('pnpm-lock.yaml') ? 'pnpm' : has('yarn.lock') ? 'yarn' : 'npm';
|
|
16
|
+
const install = pm === 'pnpm' ? 'pnpm install --frozen-lockfile' : pm === 'yarn' ? 'yarn install --frozen-lockfile' : has('package-lock.json') ? 'npm ci' : 'npm install';
|
|
17
|
+
const run = pm === 'npm' ? 'npm run' : pm;
|
|
18
|
+
return {
|
|
19
|
+
kind: 'node',
|
|
20
|
+
setup: [
|
|
21
|
+
...(pm === 'pnpm' ? ['- uses: pnpm/action-setup@v6'] : []),
|
|
22
|
+
'- uses: actions/setup-node@v7',
|
|
23
|
+
` with: { node-version: 22${pm === 'npm' ? '' : `, cache: ${pm}`} }`,
|
|
24
|
+
`- run: ${install}${scripts.build ? ` && ${run} build` : ''}`,
|
|
25
|
+
],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (has('pyproject.toml') || has('requirements.txt')) {
|
|
29
|
+
const uv = has('uv.lock');
|
|
30
|
+
return {
|
|
31
|
+
kind: 'python',
|
|
32
|
+
setup: uv
|
|
33
|
+
? ['- uses: astral-sh/setup-uv@v10', '- run: uv sync']
|
|
34
|
+
: ['- uses: actions/setup-python@v7', " with: { python-version: '3.12' }", `- run: pip install ${has('pyproject.toml') ? '.' : '-r requirements.txt'}`],
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
if (has('go.mod'))
|
|
38
|
+
return { kind: 'go', setup: ['- uses: actions/setup-go@v7', " with: { go-version-file: go.mod }", '- run: go build ./...'] };
|
|
39
|
+
if (has('Cargo.toml'))
|
|
40
|
+
return { kind: 'rust', setup: ['- run: cargo build --release'] };
|
|
41
|
+
return { kind: 'unknown', setup: ['# TODO: install and build your server here'] };
|
|
42
|
+
}
|
|
43
|
+
/** An env var whose value is a credential: goes into the workflow as a secret, never as a value. */
|
|
44
|
+
export function looksSecret(key) {
|
|
45
|
+
return /token|secret|password|passwd|api[_-]?key|(^|_)key$|credential|auth/i.test(key);
|
|
46
|
+
}
|
|
47
|
+
/** The workflow file, as text. Credentials appear only as ${{ secrets.NAME }}. */
|
|
48
|
+
export function workflowYaml(opts) {
|
|
49
|
+
const t = opts.target;
|
|
50
|
+
const lines = [
|
|
51
|
+
'# Written by `toolmenu init`. On every pull request: snapshot the tool menu this',
|
|
52
|
+
'# change produces, diff it against the committed baseline, and post one comment.',
|
|
53
|
+
'# https://github.com/niksa90/toolmenu',
|
|
54
|
+
'name: toolmenu',
|
|
55
|
+
'on: pull_request',
|
|
56
|
+
'permissions:',
|
|
57
|
+
' contents: read',
|
|
58
|
+
' pull-requests: write',
|
|
59
|
+
'jobs:',
|
|
60
|
+
' toolmenu:',
|
|
61
|
+
' runs-on: ubuntu-latest',
|
|
62
|
+
' steps:',
|
|
63
|
+
' - uses: actions/checkout@v7',
|
|
64
|
+
...(t.kind === 'stdio' ? opts.project.setup.map((l) => ` ${l}`) : []),
|
|
65
|
+
` - uses: niksa90/toolmenu@v${VERSION}`,
|
|
66
|
+
' with:',
|
|
67
|
+
];
|
|
68
|
+
if (t.kind === 'stdio') {
|
|
69
|
+
lines.push(` command: ${yamlString([t.command, ...t.args].map(shellWord).join(' '))}`);
|
|
70
|
+
const env = Object.keys(t.env ?? {});
|
|
71
|
+
if (env.length) {
|
|
72
|
+
lines.push(' env: |');
|
|
73
|
+
for (const k of env)
|
|
74
|
+
lines.push(` ${k}=${looksSecret(k) ? `\${{ secrets.${k} }}` : t.env[k]}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
lines.push(` url: ${yamlString(t.url)} # checks the server at this URL, not this PR's code: see docs/github-action.md`);
|
|
79
|
+
const headers = Object.keys(t.headers ?? {});
|
|
80
|
+
if (headers.length) {
|
|
81
|
+
lines.push(' headers: |');
|
|
82
|
+
for (const h of headers)
|
|
83
|
+
lines.push(` ${h}: \${{ secrets.${secretName(h)} }}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
lines.push(` baseline: ${opts.baseline ?? 'menu.json'}`);
|
|
87
|
+
if (opts.session)
|
|
88
|
+
lines.push(' scenario: auto # also call read-only tools and watch the menu');
|
|
89
|
+
return lines.join('\n') + '\n';
|
|
90
|
+
}
|
|
91
|
+
/** The secret names the workflow refers to, to tell the user what to add. */
|
|
92
|
+
export function secretsNeeded(target) {
|
|
93
|
+
if (target.kind === 'stdio')
|
|
94
|
+
return Object.keys(target.env ?? {}).filter(looksSecret);
|
|
95
|
+
return Object.keys(target.headers ?? {}).map(secretName);
|
|
96
|
+
}
|
|
97
|
+
function secretName(header) {
|
|
98
|
+
return 'MCP_' + header.toUpperCase().replace(/[^A-Z0-9]+/g, '_');
|
|
99
|
+
}
|
|
100
|
+
function shellWord(w) {
|
|
101
|
+
return /^[\w./:=@%+-]+$/.test(w) ? w : `'${w.replace(/'/g, `'\\''`)}'`;
|
|
102
|
+
}
|
|
103
|
+
function yamlString(s) {
|
|
104
|
+
return /^[\w./:@%+ =-]+$/.test(s) && !/^[-?:,[\]{}#&*!|>'"%@`]/.test(s) && !/: |\s#/.test(s) ? s : JSON.stringify(s);
|
|
105
|
+
}
|
package/dist/menu.js
CHANGED
|
@@ -61,6 +61,8 @@ export async function loadMenu(path) {
|
|
|
61
61
|
t.tokens = toolTokens(t);
|
|
62
62
|
}
|
|
63
63
|
menu.totalTokens = menu.tools.reduce((s, t) => s + t.tokens, 0);
|
|
64
|
+
for (const op of menu.catalog?.operations ?? [])
|
|
65
|
+
op.tokens = toolTokens(op);
|
|
64
66
|
menu.server ??= {};
|
|
65
67
|
return menu;
|
|
66
68
|
}
|
package/dist/report.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { stepLabel } from './session.js';
|
|
2
|
+
import { breakdown, breakdownLines } from './breakdown.js';
|
|
2
3
|
const LABEL = { error: 'ERROR', warn: 'WARN ', info: 'INFO ' };
|
|
3
4
|
export function counts(findings) {
|
|
4
5
|
const c = { error: 0, warn: 0, info: 0 };
|
|
@@ -17,6 +18,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
17
18
|
tools: menu.tools.length,
|
|
18
19
|
totalTokens: menu.totalTokens,
|
|
19
20
|
listMeta: menu.listMeta,
|
|
21
|
+
breakdown: breakdown(menu),
|
|
20
22
|
written: outPath ?? null,
|
|
21
23
|
counts: counts(findings),
|
|
22
24
|
findings,
|
|
@@ -32,6 +34,7 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
32
34
|
`Protocol ${s.protocolVersion ?? 'unknown'} · ${plural(menu.tools.length, 'tool')} · ~${menu.totalTokens.toLocaleString('en-US')} tokens (estimate)`,
|
|
33
35
|
'',
|
|
34
36
|
...mdFindings(findings),
|
|
37
|
+
...mdBreakdown(menu),
|
|
35
38
|
].join('\n');
|
|
36
39
|
}
|
|
37
40
|
const s = menu.server;
|
|
@@ -42,10 +45,19 @@ export function formatSnapshot(menu, findings, format, outPath) {
|
|
|
42
45
|
if (outPath)
|
|
43
46
|
lines.push(` wrote ${outPath}`);
|
|
44
47
|
lines.push('');
|
|
48
|
+
const where = breakdownLines(breakdown(menu), menu.tools.length);
|
|
49
|
+
if (where.length)
|
|
50
|
+
lines.push(...where, '');
|
|
45
51
|
lines.push(...findingLines(findings));
|
|
46
52
|
lines.push(summaryLine(findings));
|
|
47
53
|
return lines.join('\n');
|
|
48
54
|
}
|
|
55
|
+
function mdBreakdown(menu) {
|
|
56
|
+
const where = breakdownLines(breakdown(menu), menu.tools.length);
|
|
57
|
+
if (!where.length)
|
|
58
|
+
return [];
|
|
59
|
+
return ['', `<details><summary>${where[0]}</summary>`, '', '```', ...where.slice(1).map((l) => l.trimStart()), '```', '', '</details>'];
|
|
60
|
+
}
|
|
49
61
|
function findingLines(findings) {
|
|
50
62
|
const lines = [];
|
|
51
63
|
for (const f of findings) {
|
package/dist/snapshot.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type Target } from './connect.js';
|
|
2
2
|
import { type RuleSettings } from './rules/index.js';
|
|
3
|
+
import { type CatalogOptions } from './catalog.js';
|
|
3
4
|
import type { Routes } from './routes.js';
|
|
4
5
|
import type { Finding, Menu } from './types.js';
|
|
5
6
|
export interface SnapshotOptions extends RuleSettings {
|
|
@@ -12,6 +13,8 @@ export interface SnapshotOptions extends RuleSettings {
|
|
|
12
13
|
* included. Default 2; 1 turns the check off.
|
|
13
14
|
*/
|
|
14
15
|
processes?: number;
|
|
16
|
+
/** Also read the operations behind a catalog search tool (true, or which tool and queries). */
|
|
17
|
+
catalog?: boolean | CatalogOptions;
|
|
15
18
|
}
|
|
16
19
|
export interface SnapshotResult {
|
|
17
20
|
menu: Menu;
|
package/dist/snapshot.js
CHANGED
|
@@ -2,6 +2,7 @@ import { connect, listTools } from './connect.js';
|
|
|
2
2
|
import { buildMenu } from './menu.js';
|
|
3
3
|
import { isContainerWrapper, MAIN_SEED, probeVariance, seeded } from './probe.js';
|
|
4
4
|
import { MENU_RULES, runRules } from './rules/index.js';
|
|
5
|
+
import { readCatalog } from './catalog.js';
|
|
5
6
|
/** Establish the menu: connect, list the tools twice, run the menu rules. */
|
|
6
7
|
export async function snapshot(target, options = {}) {
|
|
7
8
|
const connection = await connect(seeded(target, MAIN_SEED), { timeoutMs: options.timeoutMs });
|
|
@@ -19,6 +20,9 @@ export async function snapshot(target, options = {}) {
|
|
|
19
20
|
};
|
|
20
21
|
menu = buildMenu(first.tools, server, first.listMeta);
|
|
21
22
|
secondList = buildMenu(second.tools, server, second.listMeta).tools;
|
|
23
|
+
if (options.catalog) {
|
|
24
|
+
menu.catalog = await readCatalog(connection, menu.tools, { ...(typeof options.catalog === 'object' ? options.catalog : {}), timeoutMs: options.timeoutMs });
|
|
25
|
+
}
|
|
22
26
|
}
|
|
23
27
|
catch (error) {
|
|
24
28
|
const stderr = connection.stderr().trim();
|
package/dist/types.d.ts
CHANGED