toolmenu 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -3
- package/dist/badge.d.ts +20 -0
- package/dist/badge.js +59 -0
- package/dist/cli.js +33 -13
- package/dist/help.js +7 -4
- package/dist/options.d.ts +9 -0
- package/dist/options.js +3 -0
- package/dist/sarif.d.ts +14 -0
- package/dist/sarif.js +138 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
[](https://github.com/niksa90/toolmenu/actions/workflows/ci.yml)
|
|
5
5
|
[](https://github.com/niksa90/toolmenu/blob/main/LICENSE)
|
|
6
6
|
[](https://modelcontextprotocol.io)
|
|
7
|
+
[](https://github.com/niksa90/toolmenu)
|
|
7
8
|
|
|
8
9
|
**Checks for MCP servers' tool lists: changes that confuse agents, break prompt caches or break clients, caught before your users hit them.**
|
|
9
10
|
|
|
@@ -36,7 +37,7 @@ It runs on the 22-server [bench](bench/README.md) and on hosted servers (GitHub,
|
|
|
36
37
|
Notion, Supabase, Vercel…). What it has found on real servers is in
|
|
37
38
|
[docs/FINDINGS.md](https://github.com/niksa90/toolmenu/blob/main/docs/FINDINGS.md).
|
|
38
39
|
|
|
39
|
-
> **Status: 0.
|
|
40
|
+
> **Status: 0.25.** Spec in [docs/SPEC.md](https://github.com/niksa90/toolmenu/blob/main/docs/SPEC.md),
|
|
40
41
|
> every rule in [docs/RULES.md](docs/RULES.md), changes in [CHANGELOG.md](CHANGELOG.md).
|
|
41
42
|
|
|
42
43
|
<details>
|
|
@@ -164,7 +165,7 @@ written unless `--out-dir` names a folder for each server's menu file.
|
|
|
164
165
|
|
|
165
166
|
`toolmenu mcp` is toolmenu as an MCP server (stdio), so Claude Code, Cursor or any
|
|
166
167
|
other client can check the server it's working on while it works. In Claude Code, as a
|
|
167
|
-
plugin (the server, and
|
|
168
|
+
plugin (the server, and skills for checking a change, checking a release, and setting up CI):
|
|
168
169
|
|
|
169
170
|
```
|
|
170
171
|
/plugin marketplace add niksa90/toolmenu
|
|
@@ -204,7 +205,7 @@ jobs:
|
|
|
204
205
|
- uses: actions/setup-node@v7
|
|
205
206
|
with: { node-version: 22 }
|
|
206
207
|
- run: npm ci && npm run build
|
|
207
|
-
- uses: niksa90/toolmenu@v0.
|
|
208
|
+
- uses: niksa90/toolmenu@v0.25.0
|
|
208
209
|
with:
|
|
209
210
|
command: node dist/server.js
|
|
210
211
|
baseline: menu.json # your committed snapshot
|
|
@@ -229,8 +230,55 @@ and anything `session` caught. The same report goes to the job summary.
|
|
|
229
230
|
| `release` | `auto` | Release versions for the bump check. `auto` reads `package.json`, `pyproject.toml` or `Cargo.toml` on the base branch and the head, or compares the tag with the previous one on a tag push. Set `"1.4.0..1.5.0"` yourself, or `off` |
|
|
230
231
|
| `fail-on` | `error` | Fail the job on findings at or above this level |
|
|
231
232
|
| `comment` | `true` | Post and update the PR comment |
|
|
233
|
+
| `sarif` | | A folder to also write the findings to as SARIF, for code scanning (below) |
|
|
232
234
|
| `version` | the Action's own | toolmenu version from npm: by default the one matching the Action tag you pinned (`latest` for the newest). The Action installs toolmenu from npm, so it needs a published version |
|
|
233
235
|
|
|
236
|
+
### In the Security tab: code scanning
|
|
237
|
+
|
|
238
|
+
With `sarif`, findings also go to GitHub code scanning: alerts in the Security tab,
|
|
239
|
+
each on the line of its tool in your `menu.json`, and a code scanning check on the pull
|
|
240
|
+
request. GitHub shows an alert inline in the pull request only when that line changed in
|
|
241
|
+
it, so most stay in the Security tab and the check. Run the workflow on pushes to your
|
|
242
|
+
default branch too, so GitHub has a base to compare a pull request's alerts with. Code scanning is free for public repositories; private ones need GitHub Advanced
|
|
243
|
+
Security.
|
|
244
|
+
|
|
245
|
+
```yaml
|
|
246
|
+
on:
|
|
247
|
+
pull_request:
|
|
248
|
+
push:
|
|
249
|
+
branches: [main] # the base GitHub compares a pull request's alerts with
|
|
250
|
+
permissions:
|
|
251
|
+
contents: read
|
|
252
|
+
pull-requests: write
|
|
253
|
+
security-events: write # to upload SARIF
|
|
254
|
+
steps:
|
|
255
|
+
# … checkout, build, as above
|
|
256
|
+
- uses: niksa90/toolmenu@v0.25.0
|
|
257
|
+
with:
|
|
258
|
+
command: node dist/server.js
|
|
259
|
+
sarif: toolmenu-sarif
|
|
260
|
+
- uses: github/codeql-action/upload-sarif@v4
|
|
261
|
+
if: always() # findings fail the toolmenu step; upload them anyway
|
|
262
|
+
with:
|
|
263
|
+
sarif_file: toolmenu-sarif
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Outside the Action: `toolmenu snapshot --sarif toolmenu.sarif -- node dist/server.js`
|
|
267
|
+
writes the file beside the usual report (`diff` and `session` take `--sarif` too), or
|
|
268
|
+
`--format sarif` prints it instead.
|
|
269
|
+
|
|
270
|
+
### A badge for your README
|
|
271
|
+
|
|
272
|
+
```sh
|
|
273
|
+
npx toolmenu snapshot --badge toolmenu-badge.json -- node dist/server.js
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
writes a [shields.io endpoint](https://shields.io/badges/endpoint-badge) file (tools,
|
|
277
|
+
tokens, and errors or warnings, in red, yellow or green) and prints the README line
|
|
278
|
+
for it. Commit it with `menu.json`: it changes when the baseline does, so nothing
|
|
279
|
+
has to commit from CI. The badge at the top of this README is toolmenu's own MCP
|
|
280
|
+
server (`toolmenu mcp`), checked by itself.
|
|
281
|
+
|
|
234
282
|
Server behind auth, or PRs from forks and Dependabot? See [docs/github-action.md](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md).
|
|
235
283
|
Server that unlocks tools on request? Snapshot the full menu for the baseline and run the session on the default one: [servers that unlock tools](https://github.com/niksa90/toolmenu/blob/main/docs/github-action.md#servers-that-unlock-tools).
|
|
236
284
|
PRs without secrets are skipped, and a skipped check still passes; the `skipped` output tells you.
|
package/dist/badge.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Finding, Menu } from './types.js';
|
|
2
|
+
export interface Badge {
|
|
3
|
+
schemaVersion: 1;
|
|
4
|
+
label: string;
|
|
5
|
+
message: string;
|
|
6
|
+
color: string;
|
|
7
|
+
cacheSeconds: number;
|
|
8
|
+
}
|
|
9
|
+
/** "~980", "~4.1K", "~12K": the size people compare, not the exact estimate. */
|
|
10
|
+
export declare function compactTokens(n: number): string;
|
|
11
|
+
/** The endpoint JSON: tools and tokens, and the findings that matter (errors, else warnings). */
|
|
12
|
+
export declare function badge(menu: Menu, findings: Finding[]): Badge;
|
|
13
|
+
/** `owner/repo` of a GitHub remote URL (https or ssh), or undefined. */
|
|
14
|
+
export declare function githubRepo(remote: string): string | undefined;
|
|
15
|
+
/**
|
|
16
|
+
* The README line for a badge file: a shields endpoint badge reading the file from
|
|
17
|
+
* GitHub on the default branch, linking to toolmenu. With no GitHub remote, a
|
|
18
|
+
* template to fill in.
|
|
19
|
+
*/
|
|
20
|
+
export declare function badgeMarkdown(badgePath: string, cwd?: string): string;
|
package/dist/badge.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* The README badge: a shields.io endpoint file (https://shields.io/badges/endpoint-badge)
|
|
3
|
+
* that `snapshot --badge <path>` writes beside the menu file. Committed with the
|
|
4
|
+
* baseline, it changes when the baseline does, so no CI has to commit anything:
|
|
5
|
+
* shields reads it from the repository.
|
|
6
|
+
*/
|
|
7
|
+
import { execFileSync } from 'node:child_process';
|
|
8
|
+
import { relative, resolve, sep } from 'node:path';
|
|
9
|
+
import { counts } from './report.js';
|
|
10
|
+
import { repoRoot } from './sarif.js';
|
|
11
|
+
/** "~980", "~4.1K", "~12K": the size people compare, not the exact estimate. */
|
|
12
|
+
export function compactTokens(n) {
|
|
13
|
+
if (n < 1000)
|
|
14
|
+
return `~${n}`;
|
|
15
|
+
const k = n / 1000;
|
|
16
|
+
return `~${k < 10 ? k.toFixed(1).replace(/\.0$/, '') : Math.round(k)}K`;
|
|
17
|
+
}
|
|
18
|
+
/** The endpoint JSON: tools and tokens, and the findings that matter (errors, else warnings). */
|
|
19
|
+
export function badge(menu, findings) {
|
|
20
|
+
const c = counts(findings);
|
|
21
|
+
const tools = `${menu.tools.length} tool${menu.tools.length === 1 ? '' : 's'}`;
|
|
22
|
+
const found = c.error ? ` · ${c.error} error${c.error === 1 ? '' : 's'}` : c.warn ? ` · ${c.warn} warning${c.warn === 1 ? '' : 's'}` : '';
|
|
23
|
+
return {
|
|
24
|
+
schemaVersion: 1,
|
|
25
|
+
label: 'toolmenu',
|
|
26
|
+
message: `${tools} · ${compactTokens(menu.totalTokens)} tokens${found}`,
|
|
27
|
+
color: c.error ? 'red' : c.warn ? 'yellow' : 'brightgreen',
|
|
28
|
+
// shields caches for at least 300 s; an hour is plenty for a file that changes with releases.
|
|
29
|
+
cacheSeconds: 3600,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/** `owner/repo` of a GitHub remote URL (https or ssh), or undefined. */
|
|
33
|
+
export function githubRepo(remote) {
|
|
34
|
+
const m = /github\.com[/:]([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(remote.trim());
|
|
35
|
+
return m ? `${m[1]}/${m[2]}` : undefined;
|
|
36
|
+
}
|
|
37
|
+
function git(args, cwd) {
|
|
38
|
+
try {
|
|
39
|
+
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || undefined;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The README line for a badge file: a shields endpoint badge reading the file from
|
|
47
|
+
* GitHub on the default branch, linking to toolmenu. With no GitHub remote, a
|
|
48
|
+
* template to fill in.
|
|
49
|
+
*/
|
|
50
|
+
export function badgeMarkdown(badgePath, cwd = process.cwd()) {
|
|
51
|
+
const root = repoRoot(cwd);
|
|
52
|
+
const path = relative(root, resolve(cwd, badgePath)).split(sep).join('/');
|
|
53
|
+
const repo = githubRepo(git(['remote', 'get-url', 'origin'], cwd) ?? '');
|
|
54
|
+
// origin/HEAD names the default branch when the clone recorded it; else the current one.
|
|
55
|
+
const branch = git(['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], cwd)?.replace(/^origin\//, '') ?? git(['symbolic-ref', '--short', 'HEAD'], cwd) ?? 'main';
|
|
56
|
+
const raw = `https://raw.githubusercontent.com/${repo ?? '<owner>/<repo>'}/${branch}/${path}`;
|
|
57
|
+
// The badge links to toolmenu: what it is, for whoever clicks it on the README.
|
|
58
|
+
return `[})](https://github.com/niksa90/toolmenu)`;
|
|
59
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -23,6 +23,8 @@ import { autoScenario, loadValuesFile, parseAssumeReadOnly, parseValueFlag, scen
|
|
|
23
23
|
import { authDir, listLogins, login, logout } from './auth.js';
|
|
24
24
|
import { existingMessage, initReport, planInit, shellWord, writeInit } from './init.js';
|
|
25
25
|
import { SEVERITY_RANK } from './types.js';
|
|
26
|
+
import { toSarif } from './sarif.js';
|
|
27
|
+
import { badge, badgeMarkdown } from './badge.js';
|
|
26
28
|
import { VERSION } from './version.js';
|
|
27
29
|
import { SECRET_ARGS_ENV } from './mcp-env.js';
|
|
28
30
|
import { CLI_OPTIONS } from './options.js';
|
|
@@ -78,9 +80,28 @@ export async function main(given) {
|
|
|
78
80
|
const one = stray.length === 1;
|
|
79
81
|
process.stderr.write(`toolmenu: ${sub} ignores ${stray.join(', ')}: ${one ? 'it is' : 'they are'} for other commands.\n → Next: Leave ${one ? 'it' : 'them'} out to silence this; toolmenu ${sub} --help lists the options ${sub} takes.\n`);
|
|
80
82
|
}
|
|
81
|
-
const
|
|
82
|
-
if (!['text', 'json', 'github', 'markdown'].includes(
|
|
83
|
-
throw new UsageError(`--format must be text, json, github or
|
|
83
|
+
const asked = values.json ? 'json' : values.format ?? 'text';
|
|
84
|
+
if (!['text', 'json', 'github', 'markdown', 'sarif'].includes(asked))
|
|
85
|
+
throw new UsageError(`--format must be text, json, github, markdown or sarif`);
|
|
86
|
+
// SARIF is a findings file for code scanning: --format sarif prints it instead of the
|
|
87
|
+
// report, --sarif <path> writes it beside the report.
|
|
88
|
+
const sarifCommands = ['snapshot', 'diff', 'session'];
|
|
89
|
+
if ((asked === 'sarif' || values.sarif !== undefined || values['sarif-menu'] !== undefined) && !sarifCommands.includes(sub))
|
|
90
|
+
throw new UsageError(`SARIF output (--format sarif, --sarif, --sarif-menu) goes with snapshot, diff or session; ${sub} has none.`);
|
|
91
|
+
if (sub === 'session' && (values.plan || values.init) && (asked === 'sarif' || values.sarif !== undefined))
|
|
92
|
+
throw new UsageError('SARIF needs findings: --plan and --init run nothing.');
|
|
93
|
+
const sarifToStdout = asked === 'sarif';
|
|
94
|
+
const format = (sarifToStdout ? 'text' : asked);
|
|
95
|
+
/** Print the report, or the SARIF log with --format sarif; write --sarif's file too. */
|
|
96
|
+
const emit = async (report, findings, command, menuPath) => {
|
|
97
|
+
const menu = values['sarif-menu'] ?? menuPath;
|
|
98
|
+
if (values.sarif !== undefined)
|
|
99
|
+
await writeFile(values.sarif, toSarif(findings, { command, menuPath: menu }) + '\n');
|
|
100
|
+
if (sarifToStdout)
|
|
101
|
+
process.stdout.write(toSarif(findings, { command, menuPath: menu }) + '\n');
|
|
102
|
+
else if (report)
|
|
103
|
+
process.stdout.write(show(report) + '\n');
|
|
104
|
+
};
|
|
84
105
|
// Colour for a report read in a terminal (colour.ts); piped, CI and other formats stay plain.
|
|
85
106
|
const show = (report) => (format === 'text' && colourFor(process.stdout) ? colourise(report) : report);
|
|
86
107
|
const failOn = (values['fail-on'] ?? 'error');
|
|
@@ -200,7 +221,7 @@ export async function main(given) {
|
|
|
200
221
|
release,
|
|
201
222
|
serverVersionIsRelease: values['server-version-is-release'] ?? config.serverVersionIsRelease,
|
|
202
223
|
});
|
|
203
|
-
|
|
224
|
+
await emit(formatDiff(result, before.tools.length, after.tools.length, format), result.findings, 'diff', rest[1]);
|
|
204
225
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
205
226
|
}
|
|
206
227
|
if (sub === 'history') {
|
|
@@ -361,9 +382,7 @@ export async function main(given) {
|
|
|
361
382
|
result.auto = { ...result.auto, ...auto };
|
|
362
383
|
if (values['union-out'])
|
|
363
384
|
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
364
|
-
|
|
365
|
-
if (output)
|
|
366
|
-
process.stdout.write(show(output) + '\n');
|
|
385
|
+
await emit(formatSession(result, format), result.findings, 'session', values['union-out'] ?? (existsSync('menu.json') ? 'menu.json' : undefined));
|
|
367
386
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
368
387
|
}
|
|
369
388
|
if (sub === 'session') {
|
|
@@ -386,9 +405,7 @@ export async function main(given) {
|
|
|
386
405
|
const result = await session(sessionTarget, scenario, { timeoutMs, processes, rules: config.rules, ignore: config.ignore, scenarioName: values.scenario, unionOut: !!values['union-out'] });
|
|
387
406
|
if (values['union-out'])
|
|
388
407
|
await writeFile(values['union-out'], JSON.stringify(result.union, null, 2) + '\n');
|
|
389
|
-
|
|
390
|
-
if (output)
|
|
391
|
-
process.stdout.write(show(output) + '\n');
|
|
408
|
+
await emit(formatSession(result, format), result.findings, 'session', values['union-out'] ?? (existsSync('menu.json') ? 'menu.json' : undefined));
|
|
392
409
|
return result.findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
393
410
|
}
|
|
394
411
|
const target = parseTarget(rest, command, values.header ?? [], values.env ?? [], values['no-auth'], prefix, protocol);
|
|
@@ -403,9 +420,12 @@ export async function main(given) {
|
|
|
403
420
|
const outPath = values['no-write'] ? undefined : values.out ?? 'menu.json';
|
|
404
421
|
if (outPath)
|
|
405
422
|
await writeFile(outPath, JSON.stringify(menu, null, 2) + '\n');
|
|
406
|
-
|
|
407
|
-
if (
|
|
408
|
-
|
|
423
|
+
await emit(formatSnapshot(menu, findings, format, outPath, cost, notOffered), findings, 'snapshot', outPath);
|
|
424
|
+
if (values.badge !== undefined) {
|
|
425
|
+
await writeFile(values.badge, JSON.stringify(badge(menu, findings), null, 2) + '\n');
|
|
426
|
+
// stderr: stdout stays the report (or the JSON/SARIF a script parses).
|
|
427
|
+
process.stderr.write(`toolmenu: wrote the badge to ${values.badge}. Commit it with the menu file; in the README:\n ${badgeMarkdown(values.badge)}\n`);
|
|
428
|
+
}
|
|
409
429
|
return findings.some((f) => SEVERITY_RANK[f.severity] >= SEVERITY_RANK[failOn]) ? 1 : 0;
|
|
410
430
|
}
|
|
411
431
|
function parseTarget(positionals, command, headers, env, noAuth, prefix, protocol) {
|
package/dist/help.js
CHANGED
|
@@ -12,7 +12,10 @@ const OPTIONS = {
|
|
|
12
12
|
routes: ['--routes <path>', ['routes.yml: pin which words route to which tools']],
|
|
13
13
|
catalog: ['--catalog', ['also read the operations behind a search tool (search and execute:', 'discover, search_*_tools) or command routers (listed with learn: true,', 'never run) and keep them in the menu file for diff']],
|
|
14
14
|
config: ['--config <path>', ['config file (default: toolmenu.config.json, if present)']],
|
|
15
|
-
format: ['--format <fmt>', ['text (default), json, github (annotations)
|
|
15
|
+
format: ['--format <fmt>', ['text (default), json, github (annotations), markdown (PR comments)', 'or sarif (code scanning; snapshot, diff and session)']],
|
|
16
|
+
badge: ['--badge <path>', ['also write a README badge there (a shields.io endpoint file: tools, tokens,', 'errors or warnings). Commit it with the menu file; it prints the README line']],
|
|
17
|
+
sarif: ['--sarif <path>', ['also write the findings as SARIF 2.1.0 to this file, for GitHub code', 'scanning (upload it with github/codeql-action/upload-sarif)']],
|
|
18
|
+
'sarif-menu': ['--sarif-menu <path>', ['the committed menu file SARIF results point at, at each tool\'s line', '(default: the menu file this command wrote, or diff\'s newer one)']],
|
|
16
19
|
json: ['--json', ['same as --format json']],
|
|
17
20
|
'fail-on': ['--fail-on <level>', ['exit 1 on findings at this level or above: error (default), warn, info']],
|
|
18
21
|
header: ['--header "<k>: <v>"', ['HTTP header, repeatable (e.g. "Authorization: Bearer ...")']],
|
|
@@ -85,7 +88,7 @@ const HELP = {
|
|
|
85
88
|
'reports what in the menu confuses agents or breaks prompt caches.',
|
|
86
89
|
SERVER_NOTE,
|
|
87
90
|
],
|
|
88
|
-
own: ['out', 'no-write', 'routes', 'catalog', 'as', 'client', 'lists', 'offline-check'],
|
|
91
|
+
own: ['out', 'no-write', 'routes', 'catalog', 'as', 'client', 'lists', 'offline-check', 'sarif', 'sarif-menu', 'badge'],
|
|
89
92
|
shared: ['config', 'format', 'json', 'fail-on', 'header', 'env', 'no-auth', 'protocol', 'timeout', 'processes'],
|
|
90
93
|
examples: [
|
|
91
94
|
'toolmenu snapshot -- npx -y @modelcontextprotocol/server-memory',
|
|
@@ -97,7 +100,7 @@ const HELP = {
|
|
|
97
100
|
summary: 'compare two menu files: breaking changes, token change, semver bump',
|
|
98
101
|
usage: ['toolmenu diff [options] <old.json> <new.json>'],
|
|
99
102
|
about: ['Compares two menus written by snapshot (or session --union-out). Starts no server.'],
|
|
100
|
-
own: ['release', 'server-version-is-release'],
|
|
103
|
+
own: ['release', 'server-version-is-release', 'sarif', 'sarif-menu'],
|
|
101
104
|
shared: ['config', 'format', 'json', 'fail-on'],
|
|
102
105
|
examples: ['toolmenu diff menu.json new-menu.json', 'toolmenu diff --release 1.4.0..1.5.0 --format markdown old.json new.json'],
|
|
103
106
|
exits: ['0 no findings at or above --fail-on', '1 findings at or above --fail-on', "2 a menu file can't be read, or bad usage"],
|
|
@@ -115,7 +118,7 @@ const HELP = {
|
|
|
115
118
|
'prompt cache, a different menu in a fresh process.',
|
|
116
119
|
SERVER_NOTE,
|
|
117
120
|
],
|
|
118
|
-
own: ['scenario', 'auto', 'init', 'plan', 'value', 'values-file', 'assume-read-only', 'open-world', 'create-handles', 'max-calls', 'max-calls-per-unlock', 'save-scenario', 'union-out', 'elicitation'],
|
|
121
|
+
own: ['scenario', 'auto', 'init', 'plan', 'value', 'values-file', 'assume-read-only', 'open-world', 'create-handles', 'max-calls', 'max-calls-per-unlock', 'save-scenario', 'union-out', 'elicitation', 'sarif', 'sarif-menu'],
|
|
119
122
|
shared: ['config', 'format', 'json', 'fail-on', 'header', 'env', 'no-auth', 'protocol', 'timeout', 'processes'],
|
|
120
123
|
examples: [
|
|
121
124
|
'toolmenu session --auto -- node dist/index.js',
|
package/dist/options.d.ts
CHANGED
|
@@ -2,6 +2,15 @@ export declare const CLI_OPTIONS: {
|
|
|
2
2
|
readonly out: {
|
|
3
3
|
readonly type: 'string';
|
|
4
4
|
};
|
|
5
|
+
readonly sarif: {
|
|
6
|
+
readonly type: 'string';
|
|
7
|
+
};
|
|
8
|
+
readonly badge: {
|
|
9
|
+
readonly type: 'string';
|
|
10
|
+
};
|
|
11
|
+
readonly 'sarif-menu': {
|
|
12
|
+
readonly type: 'string';
|
|
13
|
+
};
|
|
5
14
|
readonly 'out-dir': {
|
|
6
15
|
readonly type: 'string';
|
|
7
16
|
};
|
package/dist/options.js
CHANGED
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
*/
|
|
5
5
|
export const CLI_OPTIONS = {
|
|
6
6
|
out: { type: 'string' },
|
|
7
|
+
sarif: { type: 'string' },
|
|
8
|
+
badge: { type: 'string' },
|
|
9
|
+
'sarif-menu': { type: 'string' },
|
|
7
10
|
'out-dir': { type: 'string' },
|
|
8
11
|
'offline-check': { type: 'boolean' },
|
|
9
12
|
concurrency: { type: 'string' },
|
package/dist/sarif.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Finding } from './types.js';
|
|
2
|
+
export interface SarifOptions {
|
|
3
|
+
/** snapshot, diff or session: the run's category, so code scanning keeps them apart. */
|
|
4
|
+
command: string;
|
|
5
|
+
/** The menu file results point at, as a path from the working directory. */
|
|
6
|
+
menuPath?: string;
|
|
7
|
+
/** Where paths are made relative to: the repository root (default: found from the working directory). */
|
|
8
|
+
root?: string;
|
|
9
|
+
}
|
|
10
|
+
/** The repository root: the nearest folder up from `from` with a .git, or `from` itself. */
|
|
11
|
+
export declare function repoRoot(from?: string): string;
|
|
12
|
+
/** The line of each tool's "name" in a menu file written by toolmenu (1-based). */
|
|
13
|
+
export declare function toolLines(text: string): Map<string, number>;
|
|
14
|
+
export declare function toSarif(findings: Finding[], options: SarifOptions): string;
|
package/dist/sarif.js
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* SARIF 2.1.0 output, for GitHub code scanning and other SARIF viewers.
|
|
3
|
+
*
|
|
4
|
+
* toolmenu's findings are about tools in a menu, not lines of source, so each result
|
|
5
|
+
* points at the menu file in the repository (the committed baseline, or the one this
|
|
6
|
+
* command wrote) and, when the finding names a tool, at the line where that tool
|
|
7
|
+
* starts. A tool the file doesn't have (new since the baseline) points at line 1.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
11
|
+
import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
|
|
12
|
+
import { pathToFileURL } from 'node:url';
|
|
13
|
+
import { ruleDocs } from './rule-docs.js';
|
|
14
|
+
import { VERSION } from './version.js';
|
|
15
|
+
const LEVEL = { error: 'error', warn: 'warning', info: 'note' };
|
|
16
|
+
const RULES_URL = 'https://github.com/niksa90/toolmenu/blob/main/docs/RULES.md';
|
|
17
|
+
/** The repository root: the nearest folder up from `from` with a .git, or `from` itself. */
|
|
18
|
+
export function repoRoot(from = process.cwd()) {
|
|
19
|
+
let dir = resolve(from);
|
|
20
|
+
for (;;) {
|
|
21
|
+
if (existsSync(resolve(dir, '.git')))
|
|
22
|
+
return dir;
|
|
23
|
+
const up = dirname(dir);
|
|
24
|
+
if (up === dir)
|
|
25
|
+
return resolve(from);
|
|
26
|
+
dir = up;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** The line of each tool's "name" in a menu file written by toolmenu (1-based). */
|
|
30
|
+
export function toolLines(text) {
|
|
31
|
+
const lines = new Map();
|
|
32
|
+
const all = text.split(/\r?\n/);
|
|
33
|
+
const start = all.findIndex((l) => /^\s*"tools":\s*\[/.test(l));
|
|
34
|
+
if (start === -1)
|
|
35
|
+
return lines;
|
|
36
|
+
// A tool's own "name" is the first key of its object, one level inside the array.
|
|
37
|
+
for (let i = start + 1; i < all.length; i++) {
|
|
38
|
+
const m = /^ {6}"name":\s*("(?:[^"\\]|\\.)*")/.exec(all[i]);
|
|
39
|
+
if (m) {
|
|
40
|
+
const name = JSON.parse(m[1]);
|
|
41
|
+
if (!lines.has(name))
|
|
42
|
+
lines.set(name, i + 1);
|
|
43
|
+
}
|
|
44
|
+
if (/^ {2}\]/.test(all[i]))
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
return lines;
|
|
48
|
+
}
|
|
49
|
+
/** The tools a finding is about: its tool, or each tool of a change found in several. */
|
|
50
|
+
function toolsOf(f) {
|
|
51
|
+
const several = f.tools;
|
|
52
|
+
if (Array.isArray(several))
|
|
53
|
+
return several.filter((t) => typeof t === 'string');
|
|
54
|
+
return f.tool ? [f.tool] : [];
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* A stable identity for a result across runs: rule, tool, and which of that rule's
|
|
58
|
+
* findings on that tool it is (one rule can find two things about one tool). Not the
|
|
59
|
+
* message: it carries counts that change between runs.
|
|
60
|
+
*/
|
|
61
|
+
function fingerprint(rule, tool, nth) {
|
|
62
|
+
return createHash('sha256').update(`${rule}\u0000${tool ?? ''}\u0000${nth}`).digest('hex').slice(0, 32);
|
|
63
|
+
}
|
|
64
|
+
export function toSarif(findings, options) {
|
|
65
|
+
const root = options.root ?? repoRoot();
|
|
66
|
+
let artifact;
|
|
67
|
+
let lines = new Map();
|
|
68
|
+
if (options.menuPath) {
|
|
69
|
+
const abs = resolve(options.menuPath);
|
|
70
|
+
const rel = relative(root, abs);
|
|
71
|
+
// Inside the repository: relative to its root. Outside (or another drive): a file URL.
|
|
72
|
+
artifact = rel && rel !== '..' && !rel.startsWith('..' + sep) && !isAbsolute(rel) ? { uri: rel.split(sep).join('/'), uriBaseId: '%SRCROOT%' } : { uri: pathToFileURL(abs).href };
|
|
73
|
+
try {
|
|
74
|
+
lines = toolLines(readFileSync(abs, 'utf8'));
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
// Not written (--no-write) or unreadable: results point at the file's first line.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const docs = ruleDocs();
|
|
81
|
+
const ruleIds = [...new Set(findings.map((f) => f.rule))].sort();
|
|
82
|
+
const seen = new Map();
|
|
83
|
+
const results = findings.flatMap((f) => {
|
|
84
|
+
const tools = toolsOf(f);
|
|
85
|
+
// One result per tool, so each alert sits on its own tool's line.
|
|
86
|
+
return (tools.length ? tools : [undefined]).map((tool) => {
|
|
87
|
+
const key = `${f.rule}\u0000${tool ?? ''}`;
|
|
88
|
+
const nth = seen.get(key) ?? 0;
|
|
89
|
+
seen.set(key, nth + 1);
|
|
90
|
+
const text = [f.message, ...(f.detail ?? []), ...(f.fix ? [`→ Next: ${f.fix}`] : [])].join('\n');
|
|
91
|
+
return {
|
|
92
|
+
ruleId: f.rule,
|
|
93
|
+
ruleIndex: ruleIds.indexOf(f.rule),
|
|
94
|
+
level: LEVEL[f.severity],
|
|
95
|
+
message: { text: f.confidence === 'unsure' ? `${text}\n(unsure: a heuristic, from names and words)` : text },
|
|
96
|
+
...(artifact ? { locations: [{ physicalLocation: { artifactLocation: artifact, region: { startLine: (tool && lines.get(tool)) || 1 } } }] } : {}),
|
|
97
|
+
partialFingerprints: { 'toolmenuFinding/v1': fingerprint(f.rule, tool, nth) },
|
|
98
|
+
properties: { ...(tool ? { tool } : {}), ...(f.step !== undefined ? { step: f.step } : {}), ...(f.confidence ? { confidence: f.confidence } : {}) },
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
const log = {
|
|
103
|
+
$schema: 'https://json.schemastore.org/sarif-2.1.0.json',
|
|
104
|
+
version: '2.1.0',
|
|
105
|
+
runs: [
|
|
106
|
+
{
|
|
107
|
+
tool: {
|
|
108
|
+
driver: {
|
|
109
|
+
name: 'toolmenu',
|
|
110
|
+
version: VERSION,
|
|
111
|
+
semanticVersion: VERSION,
|
|
112
|
+
informationUri: 'https://github.com/niksa90/toolmenu',
|
|
113
|
+
rules: ruleIds.map((id) => {
|
|
114
|
+
const doc = docs[id];
|
|
115
|
+
// RULES.md gives a default, sometimes with conditions ("warn, error if no call got through"): its first word.
|
|
116
|
+
const documented = /^(error|warn|info)\b/.exec(doc?.severity ?? '')?.[1];
|
|
117
|
+
const severity = documented ?? findings.find((f) => f.rule === id)?.severity ?? 'warn';
|
|
118
|
+
return {
|
|
119
|
+
id,
|
|
120
|
+
name: id.replace(/[^A-Za-z0-9]+(.)?/g, (_, c) => (c ? c.toUpperCase() : '')),
|
|
121
|
+
shortDescription: { text: id },
|
|
122
|
+
...(doc ? { fullDescription: { text: doc.text } } : {}),
|
|
123
|
+
helpUri: RULES_URL,
|
|
124
|
+
help: { text: `toolmenu explain ${id}` },
|
|
125
|
+
defaultConfiguration: { level: LEVEL[severity] },
|
|
126
|
+
properties: { tags: ['mcp', doc?.command ?? options.command] },
|
|
127
|
+
};
|
|
128
|
+
}),
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
automationDetails: { id: `toolmenu/${options.command}/` },
|
|
132
|
+
originalUriBaseIds: { '%SRCROOT%': { uri: pathToFileURL(root + sep).href } },
|
|
133
|
+
results,
|
|
134
|
+
},
|
|
135
|
+
],
|
|
136
|
+
};
|
|
137
|
+
return JSON.stringify(log, null, 2);
|
|
138
|
+
}
|
package/package.json
CHANGED